Compare commits

..

1 Commits

Author SHA1 Message Date
Ben McClelland
e9b41d53b6 feat: implement read only option for posix/scoutfs
Fixes #484. When read only option selected, backend will return
AccessDenied for any operations that would update data or metadata
in the backend. This include creating and deleting objects and
buckets as well as updating object metadata.

This will still allow admin to change bucket owner since read only
is mainly intended for s3 api only.
2024-04-08 10:39:22 -07:00
300 changed files with 7017 additions and 52880 deletions

View File

@@ -1,23 +1,27 @@
---
name: Bug Report
name: Bug report
about: Create a report to help us improve
title: '[Bug] - <Short Description>'
title: ''
labels: bug
assignees: ''
---
**Describe the bug**
<!-- A clear and concise description of what the bug is. -->
A clear and concise description of what the bug is.
**To Reproduce**
<!-- Steps to reproduce the behavior. -->
Steps to reproduce the behavior.
**Expected behavior**
<!-- A clear and concise description of what you expected to happen. -->
A clear and concise description of what you expected to happen.
**Server Version**
<!-- output of: './versitygw -version && uname -a' -->
output of
```
./versitygw -version
uname -a
```
**Additional context**
<!-- Describe s3 client and version if applicable.
Describe s3 client and version if applicable.

View File

@@ -1,14 +1,14 @@
---
name: Feature Request
name: Feature request
about: Suggest an idea for this project
title: '[Feature] - <Short Description>'
title: ''
labels: enhancement
assignees: ''
---
**Describe the solution you'd like**
<!-- A clear and concise description of what you want to happen. -->
A clear and concise description of what you want to happen.
**Additional context**
<!-- Add any other context or screenshots about the feature request here. -->
Add any other context or screenshots about the feature request here.

View File

@@ -1,33 +0,0 @@
---
name: Test Case Request
about: Request new test cases or additional test coverage
title: '[Test Case] - <Short Description>'
labels: 'testcase'
assignees: ''
---
## Description
<!-- Please provide a detailed description of the test case or test coverage request. -->
## Purpose
<!-- Explain why this test case is important and what it aims to achieve. -->
## Scope
<!-- Describe the scope of the test case, including any specific functionalities, features, or modules that should be tested. -->
## Acceptance Criteria
<!-- List the criteria that must be met for the test case to be considered complete. -->
1.
2.
3.
## Additional Context
<!-- Add any other context or screenshots about the feature request here. -->
## Resources
<!-- Provide any resources, documentation, or links that could help in writing the test case. -->
**Thank you for contributing to our project!**

View File

@@ -1,37 +0,0 @@
name: azurite functional tests
on: pull_request
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 'stable'
id: go
- name: Set up Docker Compose
run: |
docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
- name: Wait for Azurite to be ready
run: sleep 40
- name: Get Dependencies
run: |
go mod download
- name: Build and Run
run: |
make
./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
- name: Shut down services
run: |
docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw

View File

@@ -1,29 +0,0 @@
name: docker bats tests
on: pull_request
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build Docker Image
run: |
cp tests/.env.docker.default tests/.env.docker
cp tests/.secrets.default tests/.secrets
# see https://github.com/versity/versitygw/issues/1034
docker build \
--build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
--build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \
--build-arg="MC_FOLDER=linux-amd64" \
--progress=plain \
-f tests/Dockerfile_test_bats \
-t bats_test .
- name: Run Docker Container
run: |
docker compose -f tests/docker-compose-bats.yml --project-directory . \
up --exit-code-from s3api_np_only s3api_np_only

View File

@@ -15,13 +15,6 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
@@ -50,7 +43,6 @@ jobs:
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64,linux/arm64
build-args: |
VERSION=${{ github.event.release.tag_name }}
TIME=${{ github.event.release.published_at }}

View File

@@ -1,8 +1,7 @@
name: functional tests
on: pull_request
jobs:
build:
name: RunTests
runs-on: ubuntu-latest
@@ -19,7 +18,7 @@ jobs:
- name: Get Dependencies
run: |
go mod download
go get -v -t -d ./...
- name: Build and Run
run: |

View File

@@ -1,16 +0,0 @@
name: shellcheck
on: pull_request
jobs:
build:
name: Run shellcheck
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Run checks
run: |
shellcheck --version
shellcheck -e SC1091 tests/*.sh tests/*/*.sh

View File

@@ -4,105 +4,16 @@ jobs:
build:
name: RunTests
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- set: "mc, posix, non-file count, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "mc-non-file-count"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "mc, posix, file count, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "mc-file-count"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "REST, posix, non-static, all, folder IAM"
IAM_TYPE: folder
RUN_SET: "rest"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3, posix, non-file count, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3-non-file-count"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3, posix, file count, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3-file-count"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3api, posix, policy, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-policy"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3api, posix, user, non-static, s3 IAM"
IAM_TYPE: s3
RUN_SET: "s3api-user"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3api, posix, bucket, static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-bucket"
RECREATE_BUCKETS: "false"
BACKEND: "posix"
- set: "s3api, posix, multipart, static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-multipart"
RECREATE_BUCKETS: "false"
BACKEND: "posix"
- set: "s3api, posix, object, static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-object"
RECREATE_BUCKETS: "false"
BACKEND: "posix"
- set: "s3api, posix, policy, static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-policy"
RECREATE_BUCKETS: "false"
BACKEND: "posix"
- set: "s3api, posix, user, static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-user"
RECREATE_BUCKETS: "false"
BACKEND: "posix"
# TODO fix/debug s3 gateway
#- set: "s3api, s3, multipart|object, non-static, folder IAM"
# IAM_TYPE: folder
# RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
# RECREATE_BUCKETS: "true"
# BACKEND: "s3"
#- set: "s3api, s3, policy|user, non-static, folder IAM"
# IAM_TYPE: folder
# RUN_SET: "s3api-policy,s3api-user"
# RECREATE_BUCKETS: "true"
# BACKEND: "s3"
- set: "s3cmd, posix, file count, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3cmd-file-count"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3cmd, posix, non-user, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3cmd-non-user"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
- set: "s3cmd, posix, user, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3cmd-user"
RECREATE_BUCKETS: "true"
BACKEND: "posix"
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Install ShellCheck
run: sudo apt-get install shellcheck
- name: Run ShellCheck
run: shellcheck -S warning ./tests/*.sh
- name: Set up Go
uses: actions/setup-go@v5
with:
@@ -117,8 +28,6 @@ jobs:
run: |
git clone https://github.com/bats-core/bats-core.git
cd bats-core && ./install.sh $HOME
git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
- name: Install s3cmd
run: |
@@ -129,73 +38,35 @@ jobs:
curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
chmod 755 /usr/local/bin/mc
- name: Install xmllint (for rest)
run: |
sudo apt-get install libxml2-utils
# see https://github.com/versity/versitygw/issues/1034
- name: Install AWS cli
run: |
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip"
unzip -o awscliv2.zip
./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
- name: Build and run
env:
IAM_TYPE: ${{ matrix.IAM_TYPE }}
RUN_SET: ${{ matrix.RUN_SET }}
AWS_PROFILE: versity
VERSITY_EXE: ${{ github.workspace }}/versitygw
RUN_VERSITYGW: true
BACKEND: ${{ matrix.BACKEND }}
RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
CERT: ${{ github.workspace }}/cert.pem
KEY: ${{ github.workspace }}/versitygw.pem
LOCAL_FOLDER: /tmp/gw
BUCKET_ONE_NAME: versity-gwtest-bucket-one
BUCKET_TWO_NAME: versity-gwtest-bucket-two
USERS_FOLDER: /tmp/iam
USERS_BUCKET: versity-gwtest-iam
AWS_ENDPOINT_URL: https://127.0.0.1:7070
PORT: 7070
S3CMD_CONFIG: tests/s3cfg.local.default
MC_ALIAS: versity
LOG_LEVEL: 4
GOCOVERDIR: ${{ github.workspace }}/cover
USERNAME_ONE: ABCDEFG
PASSWORD_ONE: 1234567
USERNAME_TWO: HIJKLMN
PASSWORD_TWO: 8901234
TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
REMOVE_TEST_FILE_FOLDER: true
VERSIONING_DIR: ${{ github.workspace }}/versioning
COMMAND_LOG: command.log
TIME_LOG: time.log
PYTHON_ENV_FOLDER: ${{ github.workspace }}/env
- name: Build and run, posix backend
run: |
make testbin
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
export AWS_REGION=us-east-1
export AWS_ACCESS_KEY_ID_TWO=user
export AWS_SECRET_ACCESS_KEY_TWO=pass
export AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
aws configure set aws_region $AWS_REGION --profile versity
mkdir $LOCAL_FOLDER
mkdir /tmp/gw
export WORKSPACE=$GITHUB_WORKSPACE
openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
mkdir $GOCOVERDIR $USERS_FOLDER
if [[ $RECREATE_BUCKETS == "false" ]]; then
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
fi
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
mkdir cover
VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
- name: Time report
run: cat ${{ github.workspace }}/time.log
#- name: Build and run, s3 backend
# run: |
# make testbin
# export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
# export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
# export AWS_REGION=us-east-1
# aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
# aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
# aws configure set aws_region $AWS_REGION --profile versity_s3
# export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
# export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
# export WORKSPACE=$GITHUB_WORKSPACE
# VERSITYGW_TEST_ENV=./tests/.env.s3.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
- name: Coverage report
run: |

22
.gitignore vendored
View File

@@ -45,25 +45,7 @@ tests/.secrets*
# IAM users files often created in testing
users.json
users.json.backup
# env files for testing
**/.env*
**/!.env.default
# s3cmd config files (testing)
tests/s3cfg.local*
tests/!s3cfg.local.default
# keys
*.pem
# patches
*.patch
# grafana's local database (kept on filesystem for survival between instantiations)
metrics-exploration/grafana_data/**
# bats tools
/tests/bats-assert
/tests/bats-support
.env*
!.env.default

View File

@@ -32,4 +32,4 @@ RUN mkdir -p $SETUP_DIR
COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
ENTRYPOINT [ "/app/versitygw" ]
ENTRYPOINT [ "/app/versitygw" ]

View File

@@ -6,7 +6,7 @@ COPY go.mod ./
RUN go mod download
COPY ./ ./
COPY ./tests/certs/* /etc/pki/tls/certs/
COPY certs/* /etc/pki/tls/certs/
ARG IAM_DIR=/tmp/vgw
ARG SETUP_DIR=/tmp/vgw

View File

@@ -1,12 +1,8 @@
FROM ubuntu:latest
FROM --platform=linux/arm64 ubuntu:latest
ARG DEBIAN_FRONTEND=noninteractive
ARG SECRETS_FILE=tests/.secrets
ARG CONFIG_FILE=tests/.env.docker
ARG GO_LIBRARY=go1.21.13.linux-arm64.tar.gz
# see https://github.com/versity/versitygw/issues/1034
ARG AWS_CLI=awscli-exe-linux-aarch64-2.22.35.zip
ARG MC_FOLDER=linux-arm64
ENV TZ=Etc/UTC
RUN apt-get update && \
@@ -20,7 +16,6 @@ RUN apt-get update && \
s3cmd \
jq \
bc \
libxml2-utils \
ca-certificates && \
update-ca-certificates && \
rm -rf /var/lib/apt/lists/*
@@ -29,20 +24,20 @@ RUN apt-get update && \
WORKDIR /tmp
# Install AWS cli
RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
# Install mc
RUN curl https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
RUN curl https://dl.min.io/client/mc/release/linux-arm64/mc \
--create-dirs \
-o /usr/local/minio-binaries/mc && \
chmod -R 755 /usr/local/minio-binaries
ENV PATH=/usr/local/minio-binaries:${PATH}
ENV PATH="/usr/local/minio-binaries":${PATH}
# Download Go 1.21 (adjust the version and platform as needed)
RUN wget https://golang.org/dl/${GO_LIBRARY}
RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
# Extract the downloaded archive
RUN tar -xvf $GO_LIBRARY -C /usr/local
RUN tar -xvf go1.21.7.linux-arm64.tar.gz -C /usr/local
# Set Go environment variables
ENV PATH="/usr/local/go/bin:${PATH}"
@@ -65,11 +60,9 @@ RUN git clone https://github.com/bats-core/bats-core.git && \
USER tester
COPY --chown=tester:tester . /home/tester
# add bats support libraries
RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests
RUN git clone https://github.com/ztombol/bats-assert.git && rm -rf /home/tester/tests/bats-assert && mv bats-assert /home/tester/tests
WORKDIR /home/tester
#RUN cp tests/.env.docker.s3.default tests/.env.docker.s3
RUN cp tests/s3cfg.local.default tests/s3cfg.local
RUN make
RUN . $SECRETS_FILE && \
@@ -86,7 +79,5 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
ENV WORKSPACE=.
ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
#ENV AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
ENTRYPOINT ["tests/run.sh"]
CMD ["s3api,s3,s3cmd,mc,rest"]
CMD ["tests/run_all.sh"]

View File

@@ -18,10 +18,6 @@ GOBUILD=$(GOCMD) build
GOCLEAN=$(GOCMD) clean
GOTEST=$(GOCMD) test
# docker-compose
DCCMD=docker-compose
DOCKERCOMPOSE=$(DCCMD) -f tests/docker-compose.yml --env-file .env.dev --project-directory .
BIN=versitygw
VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
@@ -75,19 +71,19 @@ dist:
# Creates and runs S3 gateway instance in a docker container
.PHONY: up-posix
up-posix:
$(DOCKERCOMPOSE) up posix
docker compose --env-file .env.dev up posix
# Creates and runs S3 gateway proxy instance in a docker container
.PHONY: up-proxy
up-proxy:
$(DOCKERCOMPOSE) up proxy
docker compose --env-file .env.dev up proxy
# Creates and runs S3 gateway to azurite instance in a docker container
.PHONY: up-azurite
up-azurite:
$(DOCKERCOMPOSE) up azurite azuritegw
docker compose --env-file .env.dev up azurite azuritegw
# Creates and runs both S3 gateway and proxy server instances in docker containers
.PHONY: up-app
up-app:
$(DOCKERCOMPOSE) up
docker compose --env-file .env.dev up

View File

@@ -6,7 +6,7 @@
<a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
</picture>
[![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/versity/versitygw)](https://goreportcard.com/report/github.com/versity/versitygw) [![Go Reference](https://pkg.go.dev/badge/github.com/versity/versitygw.svg)](https://pkg.go.dev/github.com/versity/versitygw)
[![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)
### Binary release builds
Download [latest release](https://github.com/versity/versitygw/releases)
@@ -14,15 +14,8 @@ Download [latest release](https://github.com/versity/versitygw/releases)
|:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
| ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
### Use Cases
* Turn your local filesystem into an S3 server with a single command!
* Proxy S3 requests to S3 storage
* Simple to deploy S3 server with a single command
* Protocol compatibility in `posix` allows common access to files via posix or S3
* Simplified interface for adding new storage system support
### News
Check out latest wiki articles: [https://github.com/versity/versitygw/wiki/Articles](https://github.com/versity/versitygw/wiki/Articles)
* New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
### Mailing List
Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).
@@ -35,12 +28,18 @@ Ask questions in the [community discussions](https://github.com/versity/versityg
<br>
Contact [Versity Sales](https://www.versity.com/contact/) to discuss enterprise support.
### Use Cases
* Share filesystem directory via S3 protocol
* Proxy S3 requests to S3 storage
* Simple to deploy S3 server with a single command
* Protocol compatibility in `posix` allows common access to files via posix or S3
### Overview
Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage, Versitys open source ScoutFS filesystem, Azure Blob Storage, and other S3 servers.
The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage and Versitys open source ScoutFS filesystem.
The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateways stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance.
@@ -68,7 +67,7 @@ The command format is
```
versitygw [global options] command [command options] [arguments...]
```
The [global options](https://github.com/versity/versitygw/wiki/Global-Options) are specified before the backend type and the backend options are specified after.
The global options are specified before the backend type and the backend options are specified after.
***

View File

@@ -17,8 +17,6 @@ package auth
import (
"context"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"strings"
@@ -29,156 +27,27 @@ import (
)
type ACL struct {
ACL types.BucketCannedACL
Owner string
Grantees []Grantee
}
type Grantee struct {
Permission Permission
Permission types.Permission
Access string
Type types.Type
}
type GetBucketAclOutput struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy"`
Owner *types.Owner
AccessControlList AccessControlList
}
type PutBucketAclInput struct {
Bucket *string
ACL types.BucketCannedACL
AccessControlPolicy *AccessControlPolicy
GrantFullControl *string
GrantRead *string
GrantReadACP *string
GrantWrite *string
GrantWriteACP *string
type AccessControlList struct {
Grants []types.Grant `xml:"Grant"`
}
type AccessControlPolicy struct {
AccessControlList AccessControlList `xml:"AccessControlList"`
Owner *types.Owner
}
func (acp *AccessControlPolicy) Validate() error {
if !acp.AccessControlList.isValid() {
return s3err.GetAPIError(s3err.ErrMalformedACL)
}
// The Owner can't be nil
if acp.Owner == nil {
return s3err.GetAPIError(s3err.ErrMalformedACL)
}
// The Owner ID can't be empty
if acp.Owner.ID == nil || *acp.Owner.ID == "" {
return s3err.GetAPIError(s3err.ErrMalformedACL)
}
return nil
}
type AccessControlList struct {
Grants []Grant `xml:"Grant"`
}
// Validates the AccessControlList
func (acl *AccessControlList) isValid() bool {
for _, el := range acl.Grants {
if !el.isValid() {
return false
}
}
return true
}
type Permission string
const (
PermissionFullControl Permission = "FULL_CONTROL"
PermissionWrite Permission = "WRITE"
PermissionWriteAcp Permission = "WRITE_ACP"
PermissionRead Permission = "READ"
PermissionReadAcp Permission = "READ_ACP"
)
// Check if the permission is valid
func (p Permission) isValid() bool {
return p == PermissionFullControl ||
p == PermissionRead ||
p == PermissionReadAcp ||
p == PermissionWrite ||
p == PermissionWriteAcp
}
type Grant struct {
Grantee *Grt `xml:"Grantee"`
Permission Permission `xml:"Permission"`
}
// Checks if Grant is valid
func (g *Grant) isValid() bool {
return g.Permission.isValid() && g.Grantee.isValid()
}
type Grt struct {
XMLNS string `xml:"xmlns:xsi,attr"`
Type types.Type `xml:"xsi:type,attr"`
ID string `xml:"ID"`
}
// Custom Unmarshalling for Grt to parse xsi:type properly
func (g *Grt) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Iterate through the XML tokens to process the attributes
for _, attr := range start.Attr {
// Check if the attribute is xsi:type and belongs to the xsi namespace
if attr.Name.Space == "http://www.w3.org/2001/XMLSchema-instance" && attr.Name.Local == "type" {
g.Type = types.Type(attr.Value)
}
// Handle xmlns:xsi
if attr.Name.Local == "xmlns:xsi" {
g.XMLNS = attr.Value
}
}
// Decode the inner XML elements like ID
for {
t, err := d.Token()
if err != nil {
return err
}
switch se := t.(type) {
case xml.StartElement:
if se.Name.Local == "ID" {
if err := d.DecodeElement(&g.ID, &se); err != nil {
return err
}
}
case xml.EndElement:
if se.Name.Local == start.Name.Local {
return nil
}
}
}
}
// Validates Grt
func (g *Grt) isValid() bool {
// Validate the Type
// Only these 2 types are supported in the gateway
if g.Type != types.TypeCanonicalUser && g.Type != types.TypeGroup {
return false
}
// The ID prop shouldn't be empty
if g.ID == "" {
return false
}
return true
Owner types.Owner
}
func ParseACL(data []byte) (ACL, error) {
@@ -193,35 +62,17 @@ func ParseACL(data []byte) (ACL, error) {
return acl, nil
}
func ParseACLOutput(data []byte, owner string) (GetBucketAclOutput, error) {
grants := []Grant{}
if len(data) == 0 {
return GetBucketAclOutput{
Owner: &types.Owner{
ID: &owner,
},
AccessControlList: AccessControlList{
Grants: grants,
},
}, nil
}
func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
var acl ACL
if err := json.Unmarshal(data, &acl); err != nil {
return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
}
grants := []types.Grant{}
for _, elem := range acl.Grantees {
acs := elem.Access
grants = append(grants, Grant{
Grantee: &Grt{
XMLNS: "http://www.w3.org/2001/XMLSchema-instance",
ID: acs,
Type: elem.Type,
},
Permission: elem.Permission,
})
grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
}
return GetBucketAclOutput{
@@ -234,43 +85,20 @@ func ParseACLOutput(data []byte, owner string) (GetBucketAclOutput, error) {
}, nil
}
func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool) ([]byte, error) {
func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
if input == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
defaultGrantees := []Grantee{
{
Permission: PermissionFullControl,
Access: acl.Owner,
Type: types.TypeCanonicalUser,
},
if acl.Owner != *input.AccessControlPolicy.Owner.ID {
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
}
// if the ACL is specified, set the ACL, else replace the grantees
if input.ACL != "" {
switch input.ACL {
case types.BucketCannedACLPublicRead:
defaultGrantees = append(defaultGrantees, Grantee{
Permission: PermissionRead,
Access: "all-users",
Type: types.TypeGroup,
})
case types.BucketCannedACLPublicReadWrite:
defaultGrantees = append(defaultGrantees, []Grantee{
{
Permission: PermissionRead,
Access: "all-users",
Type: types.TypeGroup,
},
{
Permission: PermissionWrite,
Access: "all-users",
Type: types.TypeGroup,
},
}...)
}
acl.ACL = input.ACL
acl.Grantees = []Grantee{}
} else {
grantees := []Grantee{}
accs := []string{}
if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
@@ -279,71 +107,45 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
if input.GrantFullControl != nil && *input.GrantFullControl != "" {
fullControlList = splitUnique(*input.GrantFullControl, ",")
for _, str := range fullControlList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: PermissionFullControl,
Type: types.TypeCanonicalUser,
})
grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
}
}
if input.GrantRead != nil && *input.GrantRead != "" {
readList = splitUnique(*input.GrantRead, ",")
for _, str := range readList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: PermissionRead,
Type: types.TypeCanonicalUser,
})
grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
}
}
if input.GrantReadACP != nil && *input.GrantReadACP != "" {
readACPList = splitUnique(*input.GrantReadACP, ",")
for _, str := range readACPList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: PermissionReadAcp,
Type: types.TypeCanonicalUser,
})
grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
}
}
if input.GrantWrite != nil && *input.GrantWrite != "" {
writeList = splitUnique(*input.GrantWrite, ",")
for _, str := range writeList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: PermissionWrite,
Type: types.TypeCanonicalUser,
})
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
}
}
if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
writeACPList = splitUnique(*input.GrantWriteACP, ",")
for _, str := range writeACPList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: PermissionWriteAcp,
Type: types.TypeCanonicalUser,
})
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
}
}
accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
} else {
cache := make(map[string]bool)
for _, grt := range input.AccessControlPolicy.AccessControlList.Grants {
if grt.Grantee == nil || grt.Grantee.ID == "" || grt.Permission == "" {
for _, grt := range input.AccessControlPolicy.Grants {
if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
access := grt.Grantee.ID
defaultGrantees = append(defaultGrantees, Grantee{
Access: access,
Permission: grt.Permission,
Type: types.TypeCanonicalUser,
})
if _, ok := cache[access]; !ok {
cache[access] = true
accs = append(accs, access)
grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
if _, ok := cache[*grt.Grantee.ID]; !ok {
cache[*grt.Grantee.ID] = true
accs = append(accs, *grt.Grantee.ID)
}
}
}
@@ -356,9 +158,10 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
if len(accList) > 0 {
return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
}
}
acl.Grantees = defaultGrantees
acl.Grantees = grantees
acl.ACL = ""
}
result, err := json.Marshal(acl)
if err != nil {
@@ -378,8 +181,8 @@ func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
result = append(result, acc)
continue
}
if errors.Is(err, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)) {
return nil, err
if err == ErrNotSupported {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
return nil, fmt.Errorf("check user account: %w", err)
}
@@ -402,34 +205,35 @@ func splitUnique(s, divider string) []string {
return result
}
func verifyACL(acl ACL, access string, permission Permission) error {
grantee := Grantee{
Access: access,
Permission: permission,
Type: types.TypeCanonicalUser,
}
granteeFullCtrl := Grantee{
Access: access,
Permission: PermissionFullControl,
Type: types.TypeCanonicalUser,
}
granteeAllUsers := Grantee{
Access: "all-users",
Permission: permission,
Type: types.TypeGroup,
}
isFound := false
for _, grt := range acl.Grantees {
if grt == grantee || grt == granteeFullCtrl || grt == granteeAllUsers {
isFound = true
break
func verifyACL(acl ACL, access string, permission types.Permission) error {
if acl.ACL != "" {
if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
}
if isFound {
return nil
} else {
if len(acl.Grantees) == 0 {
return nil
}
grantee := Grantee{Access: access, Permission: permission}
granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
isFound := false
for _, grt := range acl.Grantees {
if grt == grantee || grt == granteeFullCtrl {
isFound = true
break
}
}
if isFound {
return nil
}
}
return s3err.GetAPIError(s3err.ErrAccessDenied)
@@ -469,37 +273,38 @@ func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
type AccessOptions struct {
Acl ACL
AclPermission Permission
AclPermission types.Permission
IsRoot bool
Acc Account
Bucket string
Object string
Action Action
Readonly bool
}
func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
if opts.Readonly {
if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
}
if opts.IsRoot {
return nil
}
if opts.Acc.Role == RoleAdmin {
return nil
}
policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
if policyErr != nil {
if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
return policyErr
}
} else {
return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
if opts.Acc.Access == opts.Acl.Owner {
return nil
}
policy, err := be.GetBucketPolicy(ctx, opts.Bucket)
if err != nil {
return err
}
// If bucket policy is not set and the ACL is default, only the owner has access
if len(policy) == 0 && opts.Acl.ACL == "" && len(opts.Acl.Grantees) == 0 {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
if err := verifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
return err
}
if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
return err
}
@@ -538,7 +343,7 @@ func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource
if err := VerifyAccess(ctx, be, AccessOptions{
Acl: srcBucketAcl,
AclPermission: PermissionRead,
AclPermission: types.PermissionRead,
IsRoot: opts.IsRoot,
Acc: opts.Acc,
Bucket: srcBucket,

View File

@@ -16,52 +16,16 @@ package auth
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"github.com/versity/versitygw/s3err"
)
type policyErr string
func (p policyErr) Error() string {
return string(p)
}
const (
policyErrResourceMismatch = policyErr("Action does not apply to any resource(s) in statement")
policyErrInvalidResource = policyErr("Policy has invalid resource")
policyErrInvalidPrincipal = policyErr("Invalid principal in policy")
policyErrInvalidAction = policyErr("Policy has invalid action")
policyErrInvalidPolicy = policyErr("This policy contains invalid Json")
policyErrInvalidFirstChar = policyErr("Policies must be valid JSON and the first byte must be '{'")
policyErrEmptyStatement = policyErr("Could not parse the policy: Statement is empty!")
policyErrMissingStatmentField = policyErr("Missing required field Statement")
)
type BucketPolicy struct {
Statement []BucketPolicyItem `json:"Statement"`
}
func (bp *BucketPolicy) UnmarshalJSON(data []byte) error {
var tmp struct {
Statement *[]BucketPolicyItem `json:"Statement"`
}
if err := json.Unmarshal(data, &tmp); err != nil {
return err
}
// If Statement is nil (not present in JSON), return an error
if tmp.Statement == nil {
return policyErrMissingStatmentField
}
// Assign the parsed value to the actual struct
bp.Statement = *tmp.Statement
return nil
}
func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
for _, statement := range bp.Statement {
err := statement.Validate(bucket, iam)
@@ -74,19 +38,18 @@ func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
}
func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
var isAllowed bool
for _, statement := range bp.Statement {
if statement.findMatch(principal, action, resource) {
switch statement.Effect {
case BucketPolicyAccessTypeAllow:
isAllowed = true
return true
case BucketPolicyAccessTypeDeny:
return false
}
}
}
return isAllowed
return false
}
type BucketPolicyItem struct {
@@ -112,14 +75,11 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
for action := range bpi.Actions {
isObjectAction := action.IsObjectAction()
if isObjectAction == nil {
break
if isObjectAction && !containsObjectAction {
return fmt.Errorf("unsupported object action '%v' on the specified resources", action)
}
if *isObjectAction && !containsObjectAction {
return policyErrResourceMismatch
}
if !*isObjectAction && !containsBucketAction {
return policyErrResourceMismatch
if !isObjectAction && !containsBucketAction {
return fmt.Errorf("unsupported bucket action '%v' on the specified resources", action)
}
}
@@ -143,20 +103,9 @@ func getMalformedPolicyError(err error) error {
}
func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) error {
if len(policyBin) == 0 || policyBin[0] != '{' {
return getMalformedPolicyError(policyErrInvalidFirstChar)
}
var policy BucketPolicy
if err := json.Unmarshal(policyBin, &policy); err != nil {
var pe policyErr
if errors.As(err, &pe) {
return getMalformedPolicyError(err)
}
return getMalformedPolicyError(policyErrInvalidPolicy)
}
if len(policy.Statement) == 0 {
return getMalformedPolicyError(policyErrEmptyStatement)
return getMalformedPolicyError(err)
}
if err := policy.Validate(bucket, iam); err != nil {
@@ -166,7 +115,12 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
return nil
}
func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
func verifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
// If bucket policy is not set
if len(policy) == 0 {
return nil
}
var bucketPolicy BucketPolicy
if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
return err
@@ -177,6 +131,7 @@ func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Act
resource += "/" + object
}
fmt.Println(access, action, resource)
if !bucketPolicy.isAllowed(access, action, resource) {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}

View File

@@ -16,120 +16,92 @@ package auth
import (
"encoding/json"
"fmt"
"strings"
)
type Action string
const (
GetBucketAclAction Action = "s3:GetBucketAcl"
CreateBucketAction Action = "s3:CreateBucket"
PutBucketAclAction Action = "s3:PutBucketAcl"
DeleteBucketAction Action = "s3:DeleteBucket"
PutBucketVersioningAction Action = "s3:PutBucketVersioning"
GetBucketVersioningAction Action = "s3:GetBucketVersioning"
PutBucketPolicyAction Action = "s3:PutBucketPolicy"
GetBucketPolicyAction Action = "s3:GetBucketPolicy"
DeleteBucketPolicyAction Action = "s3:DeleteBucketPolicy"
AbortMultipartUploadAction Action = "s3:AbortMultipartUpload"
ListMultipartUploadPartsAction Action = "s3:ListMultipartUploadParts"
ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
PutObjectAction Action = "s3:PutObject"
GetObjectAction Action = "s3:GetObject"
GetObjectVersionAction Action = "s3:GetObjectVersion"
DeleteObjectAction Action = "s3:DeleteObject"
GetObjectAclAction Action = "s3:GetObjectAcl"
GetObjectAttributesAction Action = "s3:GetObjectAttributes"
PutObjectAclAction Action = "s3:PutObjectAcl"
RestoreObjectAction Action = "s3:RestoreObject"
GetBucketTaggingAction Action = "s3:GetBucketTagging"
PutBucketTaggingAction Action = "s3:PutBucketTagging"
GetObjectTaggingAction Action = "s3:GetObjectTagging"
PutObjectTaggingAction Action = "s3:PutObjectTagging"
DeleteObjectTaggingAction Action = "s3:DeleteObjectTagging"
ListBucketVersionsAction Action = "s3:ListBucketVersions"
ListBucketAction Action = "s3:ListBucket"
GetBucketObjectLockConfigurationAction Action = "s3:GetBucketObjectLockConfiguration"
PutBucketObjectLockConfigurationAction Action = "s3:PutBucketObjectLockConfiguration"
GetObjectLegalHoldAction Action = "s3:GetObjectLegalHold"
PutObjectLegalHoldAction Action = "s3:PutObjectLegalHold"
GetObjectRetentionAction Action = "s3:GetObjectRetention"
PutObjectRetentionAction Action = "s3:PutObjectRetention"
BypassGovernanceRetentionAction Action = "s3:BypassGovernanceRetention"
PutBucketOwnershipControlsAction Action = "s3:PutBucketOwnershipControls"
GetBucketOwnershipControlsAction Action = "s3:GetBucketOwnershipControls"
PutBucketCorsAction Action = "s3:PutBucketCORS"
GetBucketCorsAction Action = "s3:GetBucketCORS"
AllActions Action = "s3:*"
GetBucketAclAction Action = "s3:GetBucketAcl"
CreateBucketAction Action = "s3:CreateBucket"
PutBucketAclAction Action = "s3:PutBucketAcl"
DeleteBucketAction Action = "s3:DeleteBucket"
PutBucketVersioningAction Action = "s3:PutBucketVersioning"
GetBucketVersioningAction Action = "s3:GetBucketVersioning"
PutBucketPolicyAction Action = "s3:PutBucketPolicy"
GetBucketPolicyAction Action = "s3:GetBucketPolicy"
DeleteBucketPolicyAction Action = "s3:DeleteBucketPolicy"
AbortMultipartUploadAction Action = "s3:AbortMultipartUpload"
ListMultipartUploadPartsAction Action = "s3:ListMultipartUploadParts"
ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
PutObjectAction Action = "s3:PutObject"
GetObjectAction Action = "s3:GetObject"
DeleteObjectAction Action = "s3:DeleteObject"
GetObjectAclAction Action = "s3:GetObjectAcl"
GetObjectAttributesAction Action = "s3:GetObjectAttributes"
PutObjectAclAction Action = "s3:PutObjectAcl"
RestoreObjectAction Action = "s3:RestoreObject"
GetBucketTaggingAction Action = "s3:GetBucketTagging"
PutBucketTaggingAction Action = "s3:PutBucketTagging"
GetObjectTaggingAction Action = "s3:GetObjectTagging"
PutObjectTaggingAction Action = "s3:PutObjectTagging"
DeleteObjectTaggingAction Action = "s3:DeleteObjectTagging"
ListBucketVersionsAction Action = "s3:ListBucketVersions"
ListBucketAction Action = "s3:ListBucket"
AllActions Action = "s3:*"
)
var supportedActionList = map[Action]struct{}{
GetBucketAclAction: {},
CreateBucketAction: {},
PutBucketAclAction: {},
DeleteBucketAction: {},
PutBucketVersioningAction: {},
GetBucketVersioningAction: {},
PutBucketPolicyAction: {},
GetBucketPolicyAction: {},
DeleteBucketPolicyAction: {},
AbortMultipartUploadAction: {},
ListMultipartUploadPartsAction: {},
ListBucketMultipartUploadsAction: {},
PutObjectAction: {},
GetObjectAction: {},
GetObjectVersionAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
PutObjectAclAction: {},
RestoreObjectAction: {},
GetBucketTaggingAction: {},
PutBucketTaggingAction: {},
GetObjectTaggingAction: {},
PutObjectTaggingAction: {},
DeleteObjectTaggingAction: {},
ListBucketVersionsAction: {},
ListBucketAction: {},
PutBucketObjectLockConfigurationAction: {},
GetObjectLegalHoldAction: {},
PutObjectLegalHoldAction: {},
GetObjectRetentionAction: {},
PutObjectRetentionAction: {},
BypassGovernanceRetentionAction: {},
PutBucketOwnershipControlsAction: {},
GetBucketOwnershipControlsAction: {},
PutBucketCorsAction: {},
GetBucketCorsAction: {},
AllActions: {},
GetBucketAclAction: {},
CreateBucketAction: {},
PutBucketAclAction: {},
DeleteBucketAction: {},
PutBucketVersioningAction: {},
GetBucketVersioningAction: {},
PutBucketPolicyAction: {},
GetBucketPolicyAction: {},
DeleteBucketPolicyAction: {},
AbortMultipartUploadAction: {},
ListMultipartUploadPartsAction: {},
ListBucketMultipartUploadsAction: {},
PutObjectAction: {},
GetObjectAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
PutObjectAclAction: {},
RestoreObjectAction: {},
GetBucketTaggingAction: {},
PutBucketTaggingAction: {},
GetObjectTaggingAction: {},
PutObjectTaggingAction: {},
DeleteObjectTaggingAction: {},
ListBucketVersionsAction: {},
ListBucketAction: {},
AllActions: {},
}
var supportedObjectActionList = map[Action]struct{}{
AbortMultipartUploadAction: {},
ListMultipartUploadPartsAction: {},
PutObjectAction: {},
GetObjectAction: {},
GetObjectVersionAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
PutObjectAclAction: {},
RestoreObjectAction: {},
GetObjectTaggingAction: {},
PutObjectTaggingAction: {},
DeleteObjectTaggingAction: {},
GetObjectLegalHoldAction: {},
PutObjectLegalHoldAction: {},
GetObjectRetentionAction: {},
PutObjectRetentionAction: {},
BypassGovernanceRetentionAction: {},
AllActions: {},
AbortMultipartUploadAction: {},
ListMultipartUploadPartsAction: {},
PutObjectAction: {},
GetObjectAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
PutObjectAclAction: {},
RestoreObjectAction: {},
GetObjectTaggingAction: {},
PutObjectTaggingAction: {},
DeleteObjectTaggingAction: {},
AllActions: {},
}
// Validates Action: it should either wildcard match with supported actions list or be in it
func (a Action) IsValid() error {
if !strings.HasPrefix(string(a), "s3:") {
return policyErrInvalidAction
return fmt.Errorf("invalid action: %v", a)
}
if a == AllActions {
@@ -144,39 +116,31 @@ func (a Action) IsValid() error {
}
}
return policyErrInvalidAction
return fmt.Errorf("invalid wildcard usage: %v prefix is not in the supported actions list", pattern)
}
_, found := supportedActionList[a]
if !found {
return policyErrInvalidAction
return fmt.Errorf("unsupported action: %v", a)
}
return nil
}
func getBoolPtr(bl bool) *bool {
return &bl
}
// Checks if the action is object action
// nil points to 's3:*'
func (a Action) IsObjectAction() *bool {
if a == AllActions {
return nil
}
func (a Action) IsObjectAction() bool {
if a[len(a)-1] == '*' {
pattern := strings.TrimSuffix(string(a), "*")
for act := range supportedObjectActionList {
if strings.HasPrefix(string(act), pattern) {
return getBoolPtr(true)
return true
}
}
return getBoolPtr(false)
return false
}
_, found := supportedObjectActionList[a]
return &found
return found
}
func (a Action) WildCardMatch(act Action) bool {
@@ -195,7 +159,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
var err error
if err = json.Unmarshal(data, &ss); err == nil {
if len(ss) == 0 {
return policyErrInvalidAction
return fmt.Errorf("actions can't be empty")
}
*a = make(Actions)
for _, s := range ss {
@@ -208,7 +172,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
var s string
if err = json.Unmarshal(data, &s); err == nil {
if s == "" {
return policyErrInvalidAction
return fmt.Errorf("actions can't be empty")
}
*a = make(Actions)
err = a.Add(s)

View File

@@ -30,6 +30,5 @@ func (bpat BucketPolicyAccessType) Validate() error {
return nil
}
//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
return fmt.Errorf("Invalid effect: %v", bpat)
return fmt.Errorf("invalid effect: %v", bpat)
}

View File

@@ -16,6 +16,7 @@ package auth
import (
"encoding/json"
"fmt"
)
type Principals map[string]struct{}
@@ -27,50 +28,23 @@ func (p Principals) Add(key string) {
// Override UnmarshalJSON method to decode both []string and string properties
func (p *Principals) UnmarshalJSON(data []byte) error {
ss := []string{}
var s string
var k struct {
AWS string
}
var err error
if err = json.Unmarshal(data, &ss); err == nil {
if len(ss) == 0 {
return policyErrInvalidPrincipal
return fmt.Errorf("principals can't be empty")
}
*p = make(Principals)
for _, s := range ss {
p.Add(s)
}
return nil
} else if err = json.Unmarshal(data, &s); err == nil {
if s == "" {
return policyErrInvalidPrincipal
}
*p = make(Principals)
p.Add(s)
return nil
} else if err = json.Unmarshal(data, &k); err == nil {
if k.AWS == "" {
return policyErrInvalidPrincipal
}
*p = make(Principals)
p.Add(k.AWS)
return nil
} else {
var sk struct {
AWS []string
}
if err = json.Unmarshal(data, &sk); err == nil {
if len(sk.AWS) == 0 {
return policyErrInvalidPrincipal
var s string
if err = json.Unmarshal(data, &s); err == nil {
if s == "" {
return fmt.Errorf("principals can't be empty")
}
*p = make(Principals)
for _, s := range sk.AWS {
p.Add(s)
}
p.Add(s)
}
}
@@ -97,7 +71,7 @@ func (p Principals) Validate(iam IAMService) error {
if len(p) == 1 {
return nil
}
return policyErrInvalidPrincipal
return fmt.Errorf("principals should either contain * or user access keys")
}
accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
@@ -105,7 +79,7 @@ func (p Principals) Validate(iam IAMService) error {
return err
}
if len(accs) > 0 {
return policyErrInvalidPrincipal
return fmt.Errorf("user accounts don't exist: %v", accs)
}
return nil

View File

@@ -16,6 +16,7 @@ package auth
import (
"encoding/json"
"fmt"
"strings"
)
@@ -29,7 +30,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
var err error
if err = json.Unmarshal(data, &ss); err == nil {
if len(ss) == 0 {
return policyErrInvalidResource
return fmt.Errorf("resources can't be empty")
}
*r = make(Resources)
for _, s := range ss {
@@ -42,7 +43,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
var s string
if err = json.Unmarshal(data, &s); err == nil {
if s == "" {
return policyErrInvalidResource
return fmt.Errorf("resources can't be empty")
}
*r = make(Resources)
err = r.Add(s)
@@ -59,7 +60,12 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
func (r Resources) Add(rc string) error {
ok, pattern := isValidResource(rc)
if !ok {
return policyErrInvalidResource
return fmt.Errorf("invalid resource: %v", rc)
}
_, found := r[pattern]
if found {
return fmt.Errorf("duplicate resource: %v", rc)
}
r[pattern] = struct{}{}
@@ -93,7 +99,7 @@ func (r Resources) ContainsBucketPattern() bool {
func (r Resources) Validate(bucket string) error {
for resource := range r {
if !strings.HasPrefix(resource, bucket) {
return policyErrInvalidResource
return fmt.Errorf("incorrect bucket name in %v", resource)
}
}
@@ -102,45 +108,21 @@ func (r Resources) Validate(bucket string) error {
func (r Resources) FindMatch(resource string) bool {
for res := range r {
if r.Match(res, resource) {
return true
if strings.HasSuffix(res, "*") {
pattern := strings.TrimSuffix(res, "*")
if strings.HasPrefix(resource, pattern) {
return true
}
} else {
if res == resource {
return true
}
}
}
return false
}
// Match checks if the input string matches the given pattern with wildcards (`*`, `?`).
// - `?` matches exactly one occurrence of any character.
// - `*` matches arbitrary many (including zero) occurrences of any character.
func (r Resources) Match(pattern, input string) bool {
pIdx, sIdx := 0, 0
starIdx, matchIdx := -1, 0
for sIdx < len(input) {
if pIdx < len(pattern) && (pattern[pIdx] == '?' || pattern[pIdx] == input[sIdx]) {
sIdx++
pIdx++
} else if pIdx < len(pattern) && pattern[pIdx] == '*' {
starIdx = pIdx
matchIdx = sIdx
pIdx++
} else if starIdx != -1 {
pIdx = starIdx + 1
matchIdx++
sIdx = matchIdx
} else {
return false
}
}
for pIdx < len(pattern) && pattern[pIdx] == '*' {
pIdx++
}
return pIdx == len(pattern)
}
// Checks the resource to have arn prefix and not starting with /
func isValidResource(rc string) (isValid bool, pattern string) {
if !strings.HasPrefix(rc, ResourceArnPrefix) {

View File

@@ -1,182 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"encoding/json"
"testing"
)
func TestUnmarshalJSON(t *testing.T) {
var r Resources
cases := []struct {
input string
expected int
wantErr bool
}{
{`"arn:aws:s3:::my-bucket/*"`, 1, false},
{`["arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket"]`, 2, false},
{`""`, 0, true},
{`[]`, 0, true},
{`["invalid-bucket"]`, 0, true},
}
for _, tc := range cases {
r = Resources{}
err := json.Unmarshal([]byte(tc.input), &r)
if (err != nil) != tc.wantErr {
t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
}
if len(r) != tc.expected {
t.Errorf("Expected %d resources, got %d", tc.expected, len(r))
}
}
}
func TestAdd(t *testing.T) {
r := Resources{}
cases := []struct {
input string
wantErr bool
}{
{"arn:aws:s3:::valid-bucket/*", false},
{"arn:aws:s3:::valid-bucket/object", false},
{"invalid-bucket/*", true},
{"/invalid-start", true},
}
for _, tc := range cases {
err := r.Add(tc.input)
if (err != nil) != tc.wantErr {
t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
}
}
}
func TestContainsObjectPattern(t *testing.T) {
cases := []struct {
resources []string
expected bool
}{
{[]string{"arn:aws:s3:::my-bucket/my-object"}, true},
{[]string{"arn:aws:s3:::my-bucket/*"}, true},
{[]string{"arn:aws:s3:::my-bucket"}, false},
}
for _, tc := range cases {
r := Resources{}
for _, res := range tc.resources {
r.Add(res)
}
if r.ContainsObjectPattern() != tc.expected {
t.Errorf("Expected object pattern to be %v for %v", tc.expected, tc.resources)
}
}
}
func TestContainsBucketPattern(t *testing.T) {
cases := []struct {
resources []string
expected bool
}{
{[]string{"arn:aws:s3:::my-bucket"}, true},
{[]string{"arn:aws:s3:::my-bucket/*"}, false},
{[]string{"arn:aws:s3:::my-bucket/object"}, false},
}
for _, tc := range cases {
r := Resources{}
for _, res := range tc.resources {
r.Add(res)
}
if r.ContainsBucketPattern() != tc.expected {
t.Errorf("Expected bucket pattern to be %v for %v", tc.expected, tc.resources)
}
}
}
func TestValidate(t *testing.T) {
cases := []struct {
resources []string
bucket string
expected bool
}{
{[]string{"arn:aws:s3:::valid-bucket/*"}, "valid-bucket", true},
{[]string{"arn:aws:s3:::wrong-bucket/*"}, "valid-bucket", false},
{[]string{"arn:aws:s3:::valid-bucket/*", "arn:aws:s3:::valid-bucket/object/*"}, "valid-bucket", true},
}
for _, tc := range cases {
r := Resources{}
for _, res := range tc.resources {
r.Add(res)
}
if (r.Validate(tc.bucket) == nil) != tc.expected {
t.Errorf("Expected validation to be %v for bucket %s", tc.expected, tc.bucket)
}
}
}
func TestFindMatch(t *testing.T) {
cases := []struct {
resources []string
input string
expected bool
}{
{[]string{"arn:aws:s3:::my-bucket/*"}, "my-bucket/my-object", true},
{[]string{"arn:aws:s3:::my-bucket/object"}, "other-bucket/my-object", false},
{[]string{"arn:aws:s3:::my-bucket/object"}, "my-bucket/object", true},
{[]string{"arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket/*"}, "other-bucket/something", true},
}
for _, tc := range cases {
r := Resources{}
for _, res := range tc.resources {
r.Add(res)
}
if r.FindMatch(tc.input) != tc.expected {
t.Errorf("Expected FindMatch to be %v for input %s", tc.expected, tc.input)
}
}
}
func TestMatch(t *testing.T) {
r := Resources{}
cases := []struct {
pattern string
input string
expected bool
}{
{"my-bucket/*", "my-bucket/object", true},
{"my-bucket/?bject", "my-bucket/object", true},
{"my-bucket/*", "other-bucket/object", false},
{"*", "any-bucket/object", true},
{"my-bucket/*", "my-bucket/subdir/object", true},
{"my-bucket/*", "other-bucket", false},
{"my-bucket/*/*", "my-bucket/hello", false},
{"my-bucket/*/*", "my-bucket/hello/world", true},
{"foo/???/bar", "foo/qux/bar", true},
{"foo/???/bar", "foo/quxx/bar", false},
{"foo/???/bar/*/?", "foo/qux/bar/hello/g", true},
{"foo/???/bar/*/?", "foo/qux/bar/hello/smth", false},
}
for _, tc := range cases {
if r.Match(tc.pattern, tc.input) != tc.expected {
t.Errorf("Match(%s, %s) failed, expected %v", tc.pattern, tc.input, tc.expected)
}
}
}

View File

@@ -28,49 +28,14 @@ const (
RoleUserPlus Role = "userplus"
)
func (r Role) IsValid() bool {
switch r {
case RoleAdmin:
return true
case RoleUser:
return true
case RoleUserPlus:
return true
default:
return false
}
}
// Account is a gateway IAM account
type Account struct {
Access string `json:"access"`
Secret string `json:"secret"`
Role Role `json:"role"`
UserID int `json:"userID"`
GroupID int `json:"groupID"`
}
type ListUserAccountsResult struct {
Accounts []Account
}
// Mutable props, which could be changed when updating an IAM account
type MutableProps struct {
Secret *string `json:"secret"`
UserID *int `json:"userID"`
GroupID *int `json:"groupID"`
}
func updateAcc(acc *Account, props MutableProps) {
if props.Secret != nil {
acc.Secret = *props.Secret
}
if props.GroupID != nil {
acc.GroupID = *props.GroupID
}
if props.UserID != nil {
acc.UserID = *props.UserID
}
Access string `json:"access"`
Secret string `json:"secret"`
Role Role `json:"role"`
UserID int `json:"userID"`
GroupID int `json:"groupID"`
ProjectID int `json:"projectID"`
}
// IAMService is the interface for all IAM service implementations
@@ -79,57 +44,33 @@ func updateAcc(acc *Account, props MutableProps) {
type IAMService interface {
CreateAccount(account Account) error
GetUserAccount(access string) (Account, error)
UpdateUserAccount(access string, props MutableProps) error
DeleteUserAccount(access string) error
ListUserAccounts() ([]Account, error)
Shutdown() error
}
var (
// ErrUserExists is returned when the user already exists
ErrUserExists = errors.New("user already exists")
// ErrNoSuchUser is returned when the user does not exist
ErrNoSuchUser = errors.New("user not found")
)
var ErrNoSuchUser = errors.New("user not found")
type Opts struct {
RootAccount Account
Dir string
LDAPServerURL string
LDAPBindDN string
LDAPPassword string
LDAPQueryBase string
LDAPObjClasses string
LDAPAccessAtr string
LDAPSecretAtr string
LDAPRoleAtr string
LDAPUserIdAtr string
LDAPGroupIdAtr string
VaultEndpointURL string
VaultSecretStoragePath string
VaultMountPath string
VaultRootToken string
VaultRoleId string
VaultRoleSecret string
VaultServerCert string
VaultClientCert string
VaultClientCertKey string
S3Access string
S3Secret string
S3Region string
S3Bucket string
S3Endpoint string
S3DisableSSlVerfiy bool
S3Debug bool
CacheDisable bool
CacheTTL int
CachePrune int
IpaHost string
IpaVaultName string
IpaUser string
IpaPassword string
IpaInsecure bool
IpaDebug bool
Dir string
LDAPServerURL string
LDAPBindDN string
LDAPPassword string
LDAPQueryBase string
LDAPObjClasses string
LDAPAccessAtr string
LDAPSecretAtr string
LDAPRoleAtr string
S3Access string
S3Secret string
S3Region string
S3Bucket string
S3Endpoint string
S3DisableSSlVerfiy bool
S3Debug bool
CacheDisable bool
CacheTTL int
CachePrune int
}
func New(o *Opts) (IAMService, error) {
@@ -138,30 +79,22 @@ func New(o *Opts) (IAMService, error) {
switch {
case o.Dir != "":
svc, err = NewInternal(o.RootAccount, o.Dir)
svc, err = NewInternal(o.Dir)
fmt.Printf("initializing internal IAM with %q\n", o.Dir)
case o.LDAPServerURL != "":
svc, err = NewLDAPService(o.RootAccount, o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
o.LDAPGroupIdAtr, o.LDAPObjClasses)
svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
o.LDAPObjClasses)
fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
case o.S3Endpoint != "":
svc, err = NewS3(o.RootAccount, o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
fmt.Printf("initializing S3 IAM with '%v/%v'\n",
o.S3Endpoint, o.S3Bucket)
case o.VaultEndpointURL != "":
svc, err = NewVaultIAMService(o.RootAccount, o.VaultEndpointURL, o.VaultSecretStoragePath,
o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
case o.IpaHost != "":
svc, err = NewIpaIAMService(o.RootAccount, o.IpaHost, o.IpaVaultName, o.IpaUser, o.IpaPassword, o.IpaInsecure, o.IpaDebug)
fmt.Printf("initializing IPA IAM with %q\n", o.IpaHost)
default:
// if no iam options selected, default to the single user mode
fmt.Println("No IAM service configured, enabling single account mode")
return NewIAMServiceSingle(o.RootAccount), nil
return IAMServiceSingle{}, nil
}
if err != nil {

View File

@@ -66,21 +66,6 @@ func (i *icache) get(k string) (Account, bool) {
return v.value, true
}
func (i *icache) update(k string, props MutableProps) {
i.Lock()
defer i.Unlock()
item, found := i.items[k]
if found {
updateAcc(&item.value, props)
// refresh the expiration date
item.exp = time.Now().Add(i.expire)
i.items[k] = item
}
}
func (i *icache) Delete(k string) {
i.Lock()
delete(i.items, k)
@@ -181,16 +166,6 @@ func (c *IAMCache) DeleteUserAccount(access string) error {
return nil
}
func (c *IAMCache) UpdateUserAccount(access string, props MutableProps) error {
err := c.service.UpdateUserAccount(access, props)
if err != nil {
return err
}
c.iamcache.update(access, props)
return nil
}
// ListUserAccounts is a passthrough to the underlying service and
// does not make use of the cache
func (c *IAMCache) ListUserAccounts() ([]Account, error) {

View File

@@ -22,7 +22,6 @@ import (
"os"
"path/filepath"
"sort"
"sync"
"time"
)
@@ -33,15 +32,7 @@ const (
// IAMServiceInternal manages the internal IAM service
type IAMServiceInternal struct {
// This mutex will help with racing updates to the IAM data
// from multiple requests to this gateway instance, but
// will not help with racing updates to multiple load balanced
// gateway instances. This is a limitation of the internal
// IAM service. All account updates should be sent to a single
// gateway instance if possible.
sync.RWMutex
dir string
rootAcc Account
dir string
}
// UpdateAcctFunc accepts the current data and returns the new data to be stored
@@ -55,10 +46,9 @@ type iAMConfig struct {
var _ IAMService = &IAMServiceInternal{}
// NewInternal creates a new instance for the Internal IAM service
func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
func NewInternal(dir string) (*IAMServiceInternal, error) {
i := &IAMServiceInternal{
dir: dir,
rootAcc: rootAcc,
dir: dir,
}
err := i.initIAM()
@@ -72,13 +62,6 @@ func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
// CreateAccount creates a new IAM account. Returns an error if the account
// already exists.
func (s *IAMServiceInternal) CreateAccount(account Account) error {
if account.Access == s.rootAcc.Access {
return ErrUserExists
}
s.Lock()
defer s.Unlock()
return s.storeIAM(func(data []byte) ([]byte, error) {
conf, err := parseIAM(data)
if err != nil {
@@ -87,7 +70,7 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
_, ok := conf.AccessAccounts[account.Access]
if ok {
return nil, ErrUserExists
return nil, fmt.Errorf("account already exists")
}
conf.AccessAccounts[account.Access] = account
@@ -103,13 +86,6 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
// GetUserAccount retrieves account info for the requested user. Returns
// ErrNoSuchUser if the account does not exist.
func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
if access == s.rootAcc.Access {
return s.rootAcc, nil
}
s.RLock()
defer s.RUnlock()
conf, err := s.getIAM()
if err != nil {
return Account{}, fmt.Errorf("get iam data: %w", err)
@@ -123,41 +99,9 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
return acct, nil
}
// UpdateUserAccount updates the specified user account fields. Returns
// ErrNoSuchUser if the account does not exist.
func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error {
s.Lock()
defer s.Unlock()
return s.storeIAM(func(data []byte) ([]byte, error) {
conf, err := parseIAM(data)
if err != nil {
return nil, fmt.Errorf("get iam data: %w", err)
}
acc, found := conf.AccessAccounts[access]
if !found {
return nil, ErrNoSuchUser
}
updateAcc(&acc, props)
conf.AccessAccounts[access] = acc
b, err := json.Marshal(conf)
if err != nil {
return nil, fmt.Errorf("failed to serialize iam: %w", err)
}
return b, nil
})
}
// DeleteUserAccount deletes the specified user account. Does not check if
// account exists.
func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
s.Lock()
defer s.Unlock()
return s.storeIAM(func(data []byte) ([]byte, error) {
conf, err := parseIAM(data)
if err != nil {
@@ -177,9 +121,6 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
// ListUserAccounts lists all the user accounts stored.
func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
s.RLock()
defer s.RUnlock()
conf, err := s.getIAM()
if err != nil {
return []Account{}, fmt.Errorf("get iam data: %w", err)
@@ -194,11 +135,12 @@ func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
var accs []Account
for _, k := range keys {
accs = append(accs, Account{
Access: k,
Secret: conf.AccessAccounts[k].Secret,
Role: conf.AccessAccounts[k].Role,
UserID: conf.AccessAccounts[k].UserID,
GroupID: conf.AccessAccounts[k].GroupID,
Access: k,
Secret: conf.AccessAccounts[k].Secret,
Role: conf.AccessAccounts[k].Role,
UserID: conf.AccessAccounts[k].UserID,
GroupID: conf.AccessAccounts[k].GroupID,
ProjectID: conf.AccessAccounts[k].ProjectID,
})
}
@@ -247,10 +189,6 @@ func parseIAM(b []byte) (iAMConfig, error) {
return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
}
if conf.AccessAccounts == nil {
conf.AccessAccounts = make(map[string]Account)
}
return conf, nil
}

View File

@@ -1,446 +0,0 @@
// Copyright 2025 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/http/cookiejar"
"net/url"
"strconv"
"strings"
)
const IpaVersion = "2.254"
type IpaIAMService struct {
client http.Client
id int
version string
host string
vaultName string
username string
password string
kraTransportKey *rsa.PublicKey
debug bool
rootAcc Account
}
var _ IAMService = &IpaIAMService{}
func NewIpaIAMService(rootAcc Account, host, vaultName, username, password string, isInsecure, debug bool) (*IpaIAMService, error) {
ipa := IpaIAMService{
id: 0,
version: IpaVersion,
host: host,
vaultName: vaultName,
username: username,
password: password,
debug: debug,
rootAcc: rootAcc,
}
jar, err := cookiejar.New(nil)
if err != nil {
// this should never happen
return nil, fmt.Errorf("cookie jar creation: %w", err)
}
mTLSConfig := &tls.Config{InsecureSkipVerify: isInsecure}
tr := &http.Transport{
TLSClientConfig: mTLSConfig,
}
ipa.client = http.Client{Jar: jar, Transport: tr}
err = ipa.login()
if err != nil {
return nil, fmt.Errorf("ipa login failed: %w", err)
}
req, err := ipa.newRequest("vaultconfig_show/1", []string{}, map[string]any{"all": true})
if err != nil {
return nil, fmt.Errorf("ipa vaultconfig_show: %w", err)
}
vaultConfig := struct {
Kra_Server_Server []string
Transport_Cert Base64EncodedWrapped
Wrapping_default_algorithm string
Wrapping_supported_algorithms []string
}{}
err = ipa.rpc(req, &vaultConfig)
if err != nil {
return nil, fmt.Errorf("ipa vault config: %w", err)
}
cert, err := x509.ParseCertificate(vaultConfig.Transport_Cert)
if err != nil {
return nil, fmt.Errorf("ipa cannot parse vault certificate: %w", err)
}
ipa.kraTransportKey = cert.PublicKey.(*rsa.PublicKey)
isSupported := false
for _, algo := range vaultConfig.Wrapping_supported_algorithms {
if algo == "aes-128-cbc" {
isSupported = true
break
}
}
if !isSupported {
return nil,
fmt.Errorf("IPA vault does not support aes-128-cbc. Only %v supported",
vaultConfig.Wrapping_supported_algorithms)
}
return &ipa, nil
}
func (ipa *IpaIAMService) CreateAccount(account Account) error {
return fmt.Errorf("not implemented")
}
func (ipa *IpaIAMService) GetUserAccount(access string) (Account, error) {
if access == ipa.rootAcc.Access {
return ipa.rootAcc, nil
}
req, err := ipa.newRequest("user_show/1", []string{access}, map[string]any{})
if err != nil {
return Account{}, fmt.Errorf("ipa user_show: %w", err)
}
userResult := struct {
Gidnumber []string
Uidnumber []string
}{}
err = ipa.rpc(req, &userResult)
if err != nil {
return Account{}, err
}
uid, err := strconv.Atoi(userResult.Uidnumber[0])
if err != nil {
return Account{}, fmt.Errorf("ipa uid invalid: %w", err)
}
gid, err := strconv.Atoi(userResult.Gidnumber[0])
if err != nil {
return Account{}, fmt.Errorf("ipa gid invalid: %w", err)
}
account := Account{
Access: access,
Role: RoleUser,
UserID: uid,
GroupID: gid,
}
session_key := make([]byte, 16)
_, err = rand.Read(session_key)
if err != nil {
return account, fmt.Errorf("ipa cannot generate session key: %w", err)
}
encryptedKey, err := rsa.EncryptPKCS1v15(rand.Reader, ipa.kraTransportKey, session_key)
if err != nil {
return account, fmt.Errorf("ipa vault secret retrieval: %w", err)
}
req, err = ipa.newRequest("vault_retrieve_internal/1", []string{ipa.vaultName},
map[string]any{"username": access,
"session_key": Base64EncodedWrapped(encryptedKey),
"wrapping_algo": "aes-128-cbc"})
if err != nil {
return Account{}, fmt.Errorf("ipa vault_retrieve_internal: %w", err)
}
data := struct {
Vault_data Base64EncodedWrapped
Nonce Base64EncodedWrapped
}{}
err = ipa.rpc(req, &data)
if err != nil {
return account, err
}
aes, err := aes.NewCipher(session_key)
if err != nil {
return account, fmt.Errorf("ipa cannot create AES cipher: %w", err)
}
cbc := cipher.NewCBCDecrypter(aes, data.Nonce)
cbc.CryptBlocks(data.Vault_data, data.Vault_data)
secretUnpaddedJson, err := pkcs7Unpad(data.Vault_data, 16)
if err != nil {
return account, fmt.Errorf("ipa cannot unpad decrypted result: %w", err)
}
secret := struct {
Data Base64Encoded
}{}
json.Unmarshal(secretUnpaddedJson, &secret)
account.Secret = string(secret.Data)
return account, nil
}
func (ipa *IpaIAMService) UpdateUserAccount(access string, props MutableProps) error {
return fmt.Errorf("not implemented")
}
func (ipa *IpaIAMService) DeleteUserAccount(access string) error {
return fmt.Errorf("not implemented")
}
func (ipa *IpaIAMService) ListUserAccounts() ([]Account, error) {
return []Account{}, fmt.Errorf("not implemented")
}
func (ipa *IpaIAMService) Shutdown() error {
return nil
}
// Implementation
func (ipa *IpaIAMService) login() error {
form := url.Values{}
form.Set("user", ipa.username)
form.Set("password", ipa.password)
req, err := http.NewRequest(
"POST",
fmt.Sprintf("%s/ipa/session/login_password", ipa.host),
strings.NewReader(form.Encode()))
if err != nil {
return err
}
req.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := ipa.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == 401 {
return errors.New("cannot login to FreeIPA: invalid credentials")
}
if resp.StatusCode != 200 {
return fmt.Errorf("cannot login to FreeIPA: status code %d", resp.StatusCode)
}
return nil
}
type rpcRequest = string
type rpcResponse struct {
Result json.RawMessage
Principal string
Id int
Version string
}
func (p rpcResponse) String() string {
return string(p.Result)
}
var errRpc = errors.New("IPA RPC error")
func (ipa *IpaIAMService) rpc(req rpcRequest, value any) error {
err := ipa.login()
if err != nil {
return err
}
res, err := ipa.rpcInternal(req)
if err != nil {
return err
}
return json.Unmarshal(res.Result, value)
}
func (ipa *IpaIAMService) rpcInternal(req rpcRequest) (rpcResponse, error) {
httpReq, err := http.NewRequest("POST",
fmt.Sprintf("%s/ipa/session/json", ipa.host),
strings.NewReader(req))
if err != nil {
return rpcResponse{}, err
}
ipa.log(fmt.Sprintf("%v", req))
httpReq.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
httpReq.Header.Set("Content-Type", "application/json")
httpResp, err := ipa.client.Do(httpReq)
if err != nil {
return rpcResponse{}, err
}
bytes, err := io.ReadAll(httpResp.Body)
ipa.log(string(bytes))
if err != nil {
return rpcResponse{}, err
}
result := struct {
Result struct {
Json json.RawMessage `json:"result"`
Value string `json:"value"`
Summary any `json:"summary"`
} `json:"result"`
Error json.RawMessage `json:"error"`
Id int `json:"id"`
Principal string `json:"principal"`
Version string `json:"version"`
}{}
err = json.Unmarshal(bytes, &result)
if err != nil {
return rpcResponse{}, err
}
if string(result.Error) != "null" {
return rpcResponse{}, fmt.Errorf("%s: %w", string(result.Error), errRpc)
}
return rpcResponse{
Result: result.Result.Json,
Principal: result.Principal,
Id: result.Id,
Version: result.Version,
}, nil
}
func (ipa *IpaIAMService) newRequest(method string, args []string, dict map[string]any) (rpcRequest, error) {
id := ipa.id
ipa.id++
dict["version"] = ipa.version
jmethod, errMethod := json.Marshal(method)
jargs, errArgs := json.Marshal(args)
jdict, errDict := json.Marshal(dict)
err := errors.Join(errMethod, errArgs, errDict)
if err != nil {
return "", fmt.Errorf("ipa request invalid: %w", err)
}
request := map[string]interface{}{
"id": id,
"method": json.RawMessage(jmethod),
"params": []json.RawMessage{json.RawMessage(jargs), json.RawMessage(jdict)},
}
requestJSON, err := json.Marshal(request)
if err != nil {
return "", fmt.Errorf("failed to marshal request: %w", err)
}
return string(requestJSON), nil
}
// pkcs7Unpad validates and unpads data from the given bytes slice.
// The returned value will be 1 to n bytes smaller depending on the
// amount of padding, where n is the block size.
func pkcs7Unpad(b []byte, blocksize int) ([]byte, error) {
if blocksize <= 0 {
return nil, errors.New("invalid blocksize")
}
if len(b) == 0 {
return nil, errors.New("invalid PKCS7 data (empty or not padded)")
}
if len(b)%blocksize != 0 {
return nil, errors.New("invalid padding on input")
}
c := b[len(b)-1]
n := int(c)
if n == 0 || n > len(b) {
return nil, errors.New("invalid padding on input")
}
for i := 0; i < n; i++ {
if b[len(b)-n+i] != c {
return nil, errors.New("invalid padding on input")
}
}
return b[:len(b)-n], nil
}
/*
e.g.
"value" {
"__base64__": "aGVsbG93b3JsZAo="
}
*/
type Base64EncodedWrapped []byte
func (b *Base64EncodedWrapped) UnmarshalJSON(data []byte) error {
intermediate := struct {
Base64 string `json:"__base64__"`
}{}
err := json.Unmarshal(data, &intermediate)
if err != nil {
return err
}
*b, err = base64.StdEncoding.DecodeString(intermediate.Base64)
return err
}
func (b *Base64EncodedWrapped) MarshalJSON() ([]byte, error) {
intermediate := struct {
Base64 string `json:"__base64__"`
}{Base64: base64.StdEncoding.EncodeToString(*b)}
return json.Marshal(intermediate)
}
/*
e.g.
"value": "aGVsbG93b3JsZAo="
*/
type Base64Encoded []byte
func (b *Base64Encoded) UnmarshalJSON(data []byte) error {
var intermediate string
err := json.Unmarshal(data, &intermediate)
if err != nil {
return err
}
*b, err = base64.StdEncoding.DecodeString(intermediate)
return err
}
func (ipa *IpaIAMService) log(msg string) {
if ipa.debug {
log.Println(msg)
}
}

View File

@@ -1,22 +1,7 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"fmt"
"strconv"
"strings"
"github.com/go-ldap/ldap/v3"
@@ -29,19 +14,15 @@ type LdapIAMService struct {
accessAtr string
secretAtr string
roleAtr string
groupIdAtr string
userIdAtr string
rootAcc Account
}
var _ IAMService = &LdapIAMService{}
func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
return nil, fmt.Errorf("required parameters list not fully provided")
}
conn, err := ldap.DialURL(url)
conn, err := ldap.Dial("tcp", url)
if err != nil {
return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
}
@@ -57,23 +38,15 @@ func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAt
accessAtr: accAtr,
secretAtr: secAtr,
roleAtr: roleAtr,
userIdAtr: userIdAtr,
groupIdAtr: groupIdAtr,
rootAcc: rootAcc,
}, nil
}
func (ld *LdapIAMService) CreateAccount(account Account) error {
if ld.rootAcc.Access == account.Access {
return ErrUserExists
}
userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
userEntry.Attribute("objectClass", ld.objClasses)
userEntry.Attribute(ld.accessAtr, []string{account.Access})
userEntry.Attribute(ld.secretAtr, []string{account.Secret})
userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
userEntry.Attribute(ld.groupIdAtr, []string{fmt.Sprint(account.GroupID)})
userEntry.Attribute(ld.userIdAtr, []string{fmt.Sprint(account.UserID)})
err := ld.conn.Add(userEntry)
if err != nil {
@@ -84,9 +57,6 @@ func (ld *LdapIAMService) CreateAccount(account Account) error {
}
func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
if access == ld.rootAcc.Access {
return ld.rootAcc, nil
}
searchRequest := ldap.NewSearchRequest(
ld.queryBase,
ldap.ScopeWholeSubtree,
@@ -95,7 +65,7 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
0,
false,
fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.userIdAtr, ld.groupIdAtr},
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
nil,
)
@@ -104,50 +74,14 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
return Account{}, err
}
if len(result.Entries) == 0 {
return Account{}, ErrNoSuchUser
}
entry := result.Entries[0]
groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
if err != nil {
return Account{}, fmt.Errorf("invalid entry value for group-id %q: %w",
entry.GetAttributeValue(ld.groupIdAtr), err)
}
userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
if err != nil {
return Account{}, fmt.Errorf("invalid entry value for user-id %q: %w",
entry.GetAttributeValue(ld.userIdAtr), err)
}
return Account{
Access: entry.GetAttributeValue(ld.accessAtr),
Secret: entry.GetAttributeValue(ld.secretAtr),
Role: Role(entry.GetAttributeValue(ld.roleAtr)),
GroupID: groupId,
UserID: userId,
Access: entry.GetAttributeValue(ld.accessAtr),
Secret: entry.GetAttributeValue(ld.secretAtr),
Role: Role(entry.GetAttributeValue(ld.roleAtr)),
}, nil
}
func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error {
req := ldap.NewModifyRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
if props.Secret != nil {
req.Replace(ld.secretAtr, []string{*props.Secret})
}
if props.GroupID != nil {
req.Replace(ld.groupIdAtr, []string{fmt.Sprint(*props.GroupID)})
}
if props.UserID != nil {
req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
}
err := ld.conn.Modify(req)
//TODO: Handle non existing user case
if err != nil {
return err
}
return nil
}
func (ld *LdapIAMService) DeleteUserAccount(access string) error {
delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
@@ -172,7 +106,7 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
0,
false,
fmt.Sprintf("(&%v)", searchFilter),
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.groupIdAtr, ld.userIdAtr},
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
nil,
)
@@ -183,22 +117,10 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
result := []Account{}
for _, el := range resp.Entries {
groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
if err != nil {
return nil, fmt.Errorf("invalid entry value for group-id %q: %w",
el.GetAttributeValue(ld.groupIdAtr), err)
}
userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
if err != nil {
return nil, fmt.Errorf("invalid entry value for user-id %q: %w",
el.GetAttributeValue(ld.userIdAtr), err)
}
result = append(result, Account{
Access: el.GetAttributeValue(ld.accessAtr),
Secret: el.GetAttributeValue(ld.secretAtr),
Role: Role(el.GetAttributeValue(ld.roleAtr)),
GroupID: groupId,
UserID: userId,
Access: el.GetAttributeValue(ld.accessAtr),
Secret: el.GetAttributeValue(ld.secretAtr),
Role: Role(el.GetAttributeValue(ld.roleAtr)),
})
}

View File

@@ -24,7 +24,6 @@ import (
"io"
"net/http"
"sort"
"sync"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
@@ -42,14 +41,6 @@ import (
// coming from iAMConfig and iamFile in iam_internal.
type IAMServiceS3 struct {
// This mutex will help with racing updates to the IAM data
// from multiple requests to this gateway instance, but
// will not help with racing updates to multiple load balanced
// gateway instances. This is a limitation of the internal
// IAM service. All account updates should be sent to a single
// gateway instance if possible.
sync.RWMutex
access string
secret string
region string
@@ -57,13 +48,12 @@ type IAMServiceS3 struct {
endpoint string
sslSkipVerify bool
debug bool
rootAcc Account
client *s3.Client
}
var _ IAMService = &IAMServiceS3{}
func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
if access == "" {
return nil, fmt.Errorf("must provide s3 IAM service access key")
}
@@ -88,7 +78,6 @@ func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, ssl
endpoint: endpoint,
sslSkipVerify: sslSkipVerify,
debug: debug,
rootAcc: rootAcc,
}
cfg, err := i.getConfig()
@@ -96,25 +85,11 @@ func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, ssl
return nil, fmt.Errorf("init s3 IAM: %v", err)
}
if endpoint != "" {
i.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = &endpoint
})
return i, nil
}
i.client = s3.NewFromConfig(cfg)
return i, nil
}
func (s *IAMServiceS3) CreateAccount(account Account) error {
if s.rootAcc.Access == account.Access {
return ErrUserExists
}
s.Lock()
defer s.Unlock()
conf, err := s.getAccounts()
if err != nil {
return err
@@ -122,7 +97,7 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
_, ok := conf.AccessAccounts[account.Access]
if ok {
return ErrUserExists
return fmt.Errorf("account already exists")
}
conf.AccessAccounts[account.Access] = account
@@ -130,13 +105,6 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
}
func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
if access == s.rootAcc.Access {
return s.rootAcc, nil
}
s.RLock()
defer s.RUnlock()
conf, err := s.getAccounts()
if err != nil {
return Account{}, err
@@ -150,30 +118,7 @@ func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
return acct, nil
}
func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error {
s.Lock()
defer s.Unlock()
conf, err := s.getAccounts()
if err != nil {
return err
}
acc, ok := conf.AccessAccounts[access]
if !ok {
return ErrNoSuchUser
}
updateAcc(&acc, props)
conf.AccessAccounts[access] = acc
return s.storeAccts(conf)
}
func (s *IAMServiceS3) DeleteUserAccount(access string) error {
s.Lock()
defer s.Unlock()
conf, err := s.getAccounts()
if err != nil {
return err
@@ -189,9 +134,6 @@ func (s *IAMServiceS3) DeleteUserAccount(access string) error {
}
func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
s.RLock()
defer s.RUnlock()
conf, err := s.getAccounts()
if err != nil {
return nil, err
@@ -206,17 +148,28 @@ func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
var accs []Account
for _, k := range keys {
accs = append(accs, Account{
Access: k,
Secret: conf.AccessAccounts[k].Secret,
Role: conf.AccessAccounts[k].Role,
UserID: conf.AccessAccounts[k].UserID,
GroupID: conf.AccessAccounts[k].GroupID,
Access: k,
Secret: conf.AccessAccounts[k].Secret,
Role: conf.AccessAccounts[k].Role,
UserID: conf.AccessAccounts[k].UserID,
GroupID: conf.AccessAccounts[k].GroupID,
ProjectID: conf.AccessAccounts[k].ProjectID,
})
}
return accs, nil
}
// ResolveEndpoint is used for on prem or non-aws endpoints
func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
URL: s.endpoint,
SigningRegion: s.region,
HostnameImmutable: true,
}, nil
}
func (s *IAMServiceS3) Shutdown() error {
return nil
}
@@ -235,6 +188,11 @@ func (s *IAMServiceS3) getConfig() (aws.Config, error) {
config.WithHTTPClient(client),
}
if s.endpoint != "" {
opts = append(opts,
config.WithEndpointResolverWithOptions(s))
}
if s.debug {
opts = append(opts,
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
@@ -252,15 +210,15 @@ func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
})
if err != nil {
// if the error is object not exists,
// init empty accounts struct and return that
// init empty accounts stuct and return that
var nsk *types.NoSuchKey
if errors.As(err, &nsk) {
return iAMConfig{AccessAccounts: map[string]Account{}}, nil
return iAMConfig{}, nil
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
if apiErr.ErrorCode() == "NotFound" {
return iAMConfig{AccessAccounts: map[string]Account{}}, nil
return iAMConfig{}, nil
}
}

View File

@@ -15,49 +15,34 @@
package auth
import (
"github.com/versity/versitygw/s3err"
"errors"
)
// IAMServiceSingle manages the single tenant (root-only) IAM service
type IAMServiceSingle struct {
root Account
}
type IAMServiceSingle struct{}
var _ IAMService = &IAMServiceSingle{}
func NewIAMServiceSingle(r Account) IAMService {
return &IAMServiceSingle{
root: r,
}
}
var ErrNotSupported = errors.New("method is not supported")
// CreateAccount not valid in single tenant mode
func (IAMServiceSingle) CreateAccount(account Account) error {
return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
return ErrNotSupported
}
// GetUserAccount returns root account, if the root access key
// is provided and "ErrAdminUserNotFound" otherwise
func (s IAMServiceSingle) GetUserAccount(access string) (Account, error) {
if access == s.root.Access {
return s.root, nil
}
return Account{}, s3err.GetAPIError(s3err.ErrAdminUserNotFound)
}
// UpdateUserAccount no accounts in single tenant mode
func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
// GetUserAccount no accounts in single tenant mode
func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
return Account{}, ErrNotSupported
}
// DeleteUserAccount no accounts in single tenant mode
func (IAMServiceSingle) DeleteUserAccount(access string) error {
return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
return ErrNotSupported
}
// ListUserAccounts no accounts in single tenant mode
func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
return []Account{}, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
return []Account{}, nil
}
// Shutdown graceful termination of service

View File

@@ -1,256 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
vault "github.com/hashicorp/vault-client-go"
"github.com/hashicorp/vault-client-go/schema"
)
type VaultIAMService struct {
client *vault.Client
reqOpts []vault.RequestOption
secretStoragePath string
rootAcc Account
}
var _ IAMService = &VaultIAMService{}
func NewVaultIAMService(rootAcc Account, endpoint, secretStoragePath, mountPath, rootToken, roleID, roleSecret, serverCert, clientCert, clientCertKey string) (IAMService, error) {
opts := []vault.ClientOption{
vault.WithAddress(endpoint),
// set request timeout to 10 secs
vault.WithRequestTimeout(10 * time.Second),
}
if serverCert != "" {
tls := vault.TLSConfiguration{}
tls.ServerCertificate.FromBytes = []byte(serverCert)
if clientCert != "" {
if clientCertKey == "" {
return nil, fmt.Errorf("client certificate and client certificate key should both be specified")
}
tls.ClientCertificate.FromBytes = []byte(clientCert)
tls.ClientCertificateKey.FromBytes = []byte(clientCertKey)
}
opts = append(opts, vault.WithTLS(tls))
}
client, err := vault.New(opts...)
if err != nil {
return nil, fmt.Errorf("init vault client: %w", err)
}
reqOpts := []vault.RequestOption{}
// if mount path is not specified, it defaults to "approle"
if mountPath != "" {
reqOpts = append(reqOpts, vault.WithMountPath(mountPath))
}
// Authentication
switch {
case rootToken != "":
err := client.SetToken(rootToken)
if err != nil {
return nil, fmt.Errorf("root token authentication failure: %w", err)
}
case roleID != "":
if roleSecret == "" {
return nil, fmt.Errorf("role id and role secret must both be specified")
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
resp, err := client.Auth.AppRoleLogin(ctx, schema.AppRoleLoginRequest{
RoleId: roleID,
SecretId: roleSecret,
}, reqOpts...)
cancel()
if err != nil {
return nil, fmt.Errorf("approle authentication failure: %w", err)
}
if err := client.SetToken(resp.Auth.ClientToken); err != nil {
return nil, fmt.Errorf("approle authentication set token failure: %w", err)
}
default:
return nil, fmt.Errorf("vault authentication requires either roleid/rolesecret or root token")
}
return &VaultIAMService{
client: client,
reqOpts: reqOpts,
secretStoragePath: secretStoragePath,
rootAcc: rootAcc,
}, nil
}
func (vt *VaultIAMService) CreateAccount(account Account) error {
if vt.rootAcc.Access == account.Access {
return ErrUserExists
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
_, err := vt.client.Secrets.KvV2Write(ctx, vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
Data: map[string]any{
account.Access: account,
},
Options: map[string]interface{}{
"cas": 0,
},
}, vt.reqOpts...)
cancel()
if err != nil {
if strings.Contains(err.Error(), "check-and-set") {
return ErrUserExists
}
return err
}
return nil
}
func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
if vt.rootAcc.Access == access {
return vt.rootAcc, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
resp, err := vt.client.Secrets.KvV2Read(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
cancel()
if err != nil {
return Account{}, err
}
acc, err := parseVaultUserAccount(resp.Data.Data, access)
if err != nil {
return Account{}, err
}
return acc, nil
}
func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error {
//TODO: We need something like a transaction here ?
acc, err := vt.GetUserAccount(access)
if err != nil {
return err
}
updateAcc(&acc, props)
err = vt.DeleteUserAccount(access)
if err != nil {
return err
}
err = vt.CreateAccount(acc)
if err != nil {
return err
}
return nil
}
func (vt *VaultIAMService) DeleteUserAccount(access string) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
_, err := vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
cancel()
if err != nil {
return err
}
return nil
}
func (vt *VaultIAMService) ListUserAccounts() ([]Account, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
resp, err := vt.client.Secrets.KvV2List(ctx, vt.secretStoragePath, vt.reqOpts...)
cancel()
if err != nil {
if vault.IsErrorStatus(err, 404) {
return []Account{}, nil
}
return nil, err
}
accs := []Account{}
for _, acss := range resp.Data.Keys {
acc, err := vt.GetUserAccount(acss)
if err != nil {
return nil, err
}
accs = append(accs, acc)
}
return accs, nil
}
// the client doesn't have explicit shutdown, as it uses http.Client
func (vt *VaultIAMService) Shutdown() error {
return nil
}
var errInvalidUser error = errors.New("invalid user account entry in secrets engine")
func parseVaultUserAccount(data map[string]interface{}, access string) (acc Account, err error) {
usrAcc, ok := data[access].(map[string]interface{})
if !ok {
return acc, errInvalidUser
}
acss, ok := usrAcc["access"].(string)
if !ok {
return acc, errInvalidUser
}
secret, ok := usrAcc["secret"].(string)
if !ok {
return acc, errInvalidUser
}
role, ok := usrAcc["role"].(string)
if !ok {
return acc, errInvalidUser
}
userIdJson, ok := usrAcc["userID"].(json.Number)
if !ok {
return acc, errInvalidUser
}
userId, err := userIdJson.Int64()
if err != nil {
return acc, errInvalidUser
}
groupIdJson, ok := usrAcc["groupID"].(json.Number)
if !ok {
return acc, errInvalidUser
}
groupId, err := groupIdJson.Int64()
if err != nil {
return acc, errInvalidUser
}
return Account{
Access: acss,
Secret: secret,
Role: Role(role),
UserID: int(userId),
GroupID: int(groupId),
}, nil
}

View File

@@ -1,269 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"context"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
type BucketLockConfig struct {
Enabled bool
DefaultRetention *types.DefaultRetention
CreatedAt *time.Time
}
func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
var lockConfig types.ObjectLockConfiguration
if err := xml.Unmarshal(input, &lockConfig); err != nil {
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
if lockConfig.ObjectLockEnabled != "" && lockConfig.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
config := BucketLockConfig{
Enabled: lockConfig.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
}
if lockConfig.Rule != nil && lockConfig.Rule.DefaultRetention != nil {
retention := lockConfig.Rule.DefaultRetention
if retention.Mode != types.ObjectLockRetentionModeCompliance && retention.Mode != types.ObjectLockRetentionModeGovernance {
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
if retention.Years != nil && retention.Days != nil {
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
if retention.Days != nil && *retention.Days <= 0 {
return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
}
if retention.Years != nil && *retention.Years <= 0 {
return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
}
config.DefaultRetention = retention
now := time.Now()
config.CreatedAt = &now
}
return json.Marshal(config)
}
func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfiguration, error) {
var config BucketLockConfig
if err := json.Unmarshal(input, &config); err != nil {
return nil, fmt.Errorf("parse object lock config: %w", err)
}
result := &types.ObjectLockConfiguration{
Rule: &types.ObjectLockRule{
DefaultRetention: config.DefaultRetention,
},
}
if config.Enabled {
result.ObjectLockEnabled = types.ObjectLockEnabledEnabled
}
return result, nil
}
func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
var retention s3response.PutObjectRetentionInput
if err := xml.Unmarshal(input, &retention); err != nil {
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
if retention.RetainUntilDate.Before(time.Now()) {
return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
}
switch retention.Mode {
case types.ObjectLockRetentionModeCompliance:
case types.ObjectLockRetentionModeGovernance:
default:
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
return json.Marshal(retention)
}
func ParseObjectLockRetentionOutput(input []byte) (*types.ObjectLockRetention, error) {
var retention types.ObjectLockRetention
if err := json.Unmarshal(input, &retention); err != nil {
return nil, fmt.Errorf("parse object lock retention: %w", err)
}
return &retention, nil
}
func ParseObjectLegalHoldOutput(status *bool) *s3response.GetObjectLegalHoldResult {
if status == nil {
return nil
}
if *status {
return &s3response.GetObjectLegalHoldResult{
Status: types.ObjectLockLegalHoldStatusOn,
}
}
return &s3response.GetObjectLegalHoldResult{
Status: types.ObjectLockLegalHoldStatusOff,
}
}
func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass bool, be backend.Backend) error {
data, err := be.GetObjectLockConfiguration(ctx, bucket)
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
return nil
}
return err
}
var bucketLockConfig BucketLockConfig
if err := json.Unmarshal(data, &bucketLockConfig); err != nil {
return fmt.Errorf("parse object lock config: %w", err)
}
if !bucketLockConfig.Enabled {
return nil
}
checkDefaultRetention := false
if bucketLockConfig.DefaultRetention != nil && bucketLockConfig.CreatedAt != nil {
expirationDate := *bucketLockConfig.CreatedAt
if bucketLockConfig.DefaultRetention.Days != nil {
expirationDate = expirationDate.AddDate(0, 0, int(*bucketLockConfig.DefaultRetention.Days))
}
if bucketLockConfig.DefaultRetention.Years != nil {
expirationDate = expirationDate.AddDate(int(*bucketLockConfig.DefaultRetention.Years), 0, 0)
}
if expirationDate.After(time.Now()) {
checkDefaultRetention = true
}
}
for _, obj := range objects {
var key, versionId string
if obj.Key != nil {
key = *obj.Key
}
if obj.VersionId != nil {
versionId = *obj.VersionId
}
checkRetention := true
retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId)
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
continue
}
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
checkRetention = false
}
if err != nil && checkRetention {
return err
}
if checkRetention {
retention, err := ParseObjectLockRetentionOutput(retentionData)
if err != nil {
return err
}
if retention.Mode != "" && retention.RetainUntilDate != nil {
if retention.RetainUntilDate.After(time.Now()) {
switch retention.Mode {
case types.ObjectLockRetentionModeGovernance:
if !bypass {
return s3err.GetAPIError(s3err.ErrObjectLocked)
} else {
policy, err := be.GetBucketPolicy(ctx, bucket)
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
if err != nil {
return err
}
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
if err != nil {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
}
case types.ObjectLockRetentionModeCompliance:
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
}
}
}
checkLegalHold := true
status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId)
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
continue
}
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
checkLegalHold = false
} else {
return err
}
}
if checkLegalHold && *status {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
if checkDefaultRetention {
switch bucketLockConfig.DefaultRetention.Mode {
case types.ObjectLockRetentionModeGovernance:
if !bypass {
return s3err.GetAPIError(s3err.ErrObjectLocked)
} else {
policy, err := be.GetBucketPolicy(ctx, bucket)
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
if err != nil {
return err
}
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
if err != nil {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
}
case types.ObjectLockRetentionModeCompliance:
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
}
}
return nil
}

View File

@@ -87,13 +87,13 @@ func TestStandaloneSign(t *testing.T) {
actual := req.Header.Get("Authorization")
if e, a := c.ExpSig, actual; e != a {
t.Errorf("expected %v, but received %v", e, a)
t.Errorf("expected %v, but recieved %v", e, a)
}
if e, a := c.OrigURI, req.URL.Path; e != a {
t.Errorf("expected %v, but received %v", e, a)
t.Errorf("expected %v, but recieved %v", e, a)
}
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
t.Errorf("expected %v, but received %v", e, a)
t.Errorf("expected %v, but recieved %v", e, a)
}
}
}
@@ -127,13 +127,13 @@ func TestStandaloneSign_RawPath(t *testing.T) {
actual := req.Header.Get("Authorization")
if e, a := c.ExpSig, actual; e != a {
t.Errorf("expected %v, but received %v", e, a)
t.Errorf("expected %v, but recieved %v", e, a)
}
if e, a := c.OrigURI, req.URL.Path; e != a {
t.Errorf("expected %v, but received %v", e, a)
t.Errorf("expected %v, but recieved %v", e, a)
}
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
t.Errorf("expected %v, but received %v", e, a)
t.Errorf("expected %v, but recieved %v", e, a)
}
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -18,9 +18,9 @@ import (
"bufio"
"context"
"fmt"
"io"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
"github.com/versity/versitygw/s3select"
@@ -32,46 +32,40 @@ type Backend interface {
Shutdown()
// bucket operations
ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
PutBucketAcl(_ context.Context, bucket string, data []byte) error
DeleteBucket(_ context.Context, bucket string) error
PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
DeleteBucket(context.Context, *s3.DeleteBucketInput) error
PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
DeleteBucketPolicy(_ context.Context, bucket string) error
PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
DeleteBucketOwnershipControls(_ context.Context, bucket string) error
PutBucketCors(context.Context, []byte) error
GetBucketCors(_ context.Context, bucket string) ([]byte, error)
DeleteBucketCors(_ context.Context, bucket string) error
// multipart operations
CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error)
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
// standard object operations
PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error)
PutObject(context.Context, *s3.PutObjectInput) (string, error)
HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error)
ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
DeleteObject(context.Context, *s3.DeleteObjectInput) error
DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
// special case object operations
RestoreObject(context.Context, *s3.RestoreObjectInput) error
@@ -87,23 +81,11 @@ type Backend interface {
PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
DeleteObjectTagging(_ context.Context, bucket, object string) error
// object lock operations
PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error
GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error)
PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error
GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error)
PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error
GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error)
// non AWS actions
ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error
ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
}
// InterfaceVersion tracks changes to the Backend interface for plugins.
// Increment this when the Backend interface changes.
const InterfaceVersion = 1
type BackendUnsupported struct{}
var _ Backend = &BackendUnsupported{}
@@ -115,7 +97,7 @@ func (BackendUnsupported) Shutdown() {}
func (BackendUnsupported) String() string {
return "Unsupported"
}
func (BackendUnsupported) ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
@@ -130,14 +112,14 @@ func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, [
func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucket(_ context.Context, bucket string) error {
func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -148,27 +130,9 @@ func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]b
func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
return types.ObjectOwnershipBucketOwnerEnforced, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketCors(context.Context, []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketCors(_ context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucketCors(_ context.Context, bucket string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -182,40 +146,40 @@ func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipar
func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CopyObject(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
@@ -241,8 +205,8 @@ func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.Sel
}
}
func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
@@ -265,26 +229,7 @@ func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error {
func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {

View File

@@ -18,8 +18,7 @@ import (
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"os"
"io/fs"
"strconv"
"strings"
"time"
@@ -29,13 +28,9 @@ import (
"github.com/versity/versitygw/s3response"
)
const (
// this is the media type for directories in AWS and Nextcloud
DirContentType = "application/x-directory"
DefaultContentType = "binary/octet-stream"
// this is the minimum allowed size for mp parts
MinPartSize = 5 * 1024 * 1024
var (
// RFC3339TimeFormat RFC3339 time format
RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
)
func IsValidBucketName(name string) bool { return true }
@@ -52,196 +47,65 @@ func (d ByObjectName) Len() int { return len(d) }
func (d ByObjectName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
func GetPtrFromString(str string) *string {
if str == "" {
return nil
}
return &str
}
func GetStringFromPtr(str *string) string {
if str == nil {
return ""
}
return *str
func GetStringPtr(s string) *string {
return &s
}
func GetTimePtr(t time.Time) *time.Time {
return &t
}
func TrimEtag(etag *string) *string {
if etag == nil {
return nil
}
return GetPtrFromString(strings.Trim(*etag, "\""))
}
var (
errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
errInvalidCopySourceRange = s3err.GetAPIError(s3err.ErrInvalidCopySourceRange)
errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
)
// ParseGetObjectRange parses input range header and returns startoffset, length, isValid
// and error. If no endoffset specified, then length is set to the object size
// for invalid inputs, it returns no error, but isValid=false
// `InvalidRange` error is returnd, only if startoffset is greater than the object size
func ParseGetObjectRange(size int64, acceptRange string) (int64, int64, bool, error) {
// ParseRange parses input range header and returns startoffset, length, and
// error. If no endoffset specified, then length is set to -1.
func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
if acceptRange == "" {
return 0, size, false, nil
return 0, fi.Size(), nil
}
rangeKv := strings.Split(acceptRange, "=")
if len(rangeKv) != 2 {
return 0, size, false, nil
}
if rangeKv[0] != "bytes" {
return 0, size, false, nil
if len(rangeKv) < 2 {
return 0, 0, errInvalidRange
}
bRange := strings.Split(rangeKv[1], "-")
if len(bRange) != 2 {
return 0, size, false, nil
if len(bRange) < 1 || len(bRange) > 2 {
return 0, 0, errInvalidRange
}
startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
if err != nil {
return 0, size, false, nil
return 0, 0, errInvalidRange
}
if startOffset >= size {
return 0, 0, false, errInvalidRange
endOffset := int64(-1)
if len(bRange) == 1 || bRange[1] == "" {
return startOffset, endOffset, nil
}
if bRange[1] == "" {
return startOffset, size - startOffset, true, nil
}
endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
if err != nil {
return 0, size, false, nil
return 0, 0, errInvalidRange
}
if endOffset < startOffset {
return 0, size, false, nil
}
if endOffset >= size {
return startOffset, size - startOffset, true, nil
}
return startOffset, endOffset - startOffset + 1, true, nil
}
// ParseCopySourceRange parses input range header and returns startoffset, length
// and error. If no endoffset specified, then length is set to the object size
func ParseCopySourceRange(size int64, acceptRange string) (int64, int64, error) {
if acceptRange == "" {
return 0, size, nil
}
rangeKv := strings.Split(acceptRange, "=")
if len(rangeKv) != 2 {
return 0, 0, errInvalidCopySourceRange
}
if rangeKv[0] != "bytes" {
return 0, 0, errInvalidCopySourceRange
}
bRange := strings.Split(rangeKv[1], "-")
if len(bRange) != 2 {
return 0, 0, errInvalidCopySourceRange
}
startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
if err != nil {
return 0, 0, errInvalidCopySourceRange
}
if startOffset >= size {
return 0, 0, s3err.CreateExceedingRangeErr(size)
}
if bRange[1] == "" {
return startOffset, size - startOffset + 1, nil
}
endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
if err != nil {
return 0, 0, errInvalidCopySourceRange
}
if endOffset < startOffset {
return 0, 0, errInvalidCopySourceRange
}
if endOffset >= size {
return 0, 0, s3err.CreateExceedingRangeErr(size)
return 0, 0, errInvalidRange
}
return startOffset, endOffset - startOffset + 1, nil
}
// ParseCopySource parses x-amz-copy-source header and returns source bucket,
// source object, versionId, error respectively
func ParseCopySource(copySourceHeader string) (string, string, string, error) {
if copySourceHeader[0] == '/' {
copySourceHeader = copySourceHeader[1:]
}
var copySource, versionId string
i := strings.LastIndex(copySourceHeader, "?versionId=")
if i == -1 {
copySource = copySourceHeader
} else {
copySource = copySourceHeader[:i]
versionId = copySourceHeader[i+11:]
}
srcBucket, srcObject, ok := strings.Cut(copySource, "/")
if !ok {
return "", "", "", s3err.GetAPIError(s3err.ErrInvalidCopySource)
}
return srcBucket, srcObject, versionId, nil
}
// ParseObjectTags parses the url encoded input string into
// map[string]string key-value tag set
func ParseObjectTags(t string) (map[string]string, error) {
if t == "" {
return nil, nil
}
tagging := make(map[string]string)
tagParts := strings.Split(t, "&")
for _, prt := range tagParts {
p := strings.Split(prt, "=")
if len(p) != 2 {
return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
}
if len(p[0]) > 128 || len(p[1]) > 256 {
return nil, s3err.GetAPIError(s3err.ErrInvalidTag)
}
tagging[p[0]] = p[1]
}
return tagging, nil
}
func GetMultipartMD5(parts []types.CompletedPart) string {
var partsEtagBytes []byte
for _, part := range parts {
partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
}
return fmt.Sprintf("\"%s-%d\"", md5String(partsEtagBytes), len(parts))
s3MD5 := fmt.Sprintf("%s-%d", md5String(partsEtagBytes), len(parts))
return s3MD5
}
func getEtagBytes(etag string) []byte {
@@ -256,16 +120,3 @@ func md5String(data []byte) string {
sum := md5.Sum(data)
return hex.EncodeToString(sum[:])
}
type FileSectionReadCloser struct {
R io.Reader
F *os.File
}
func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
return f.R.Read(p)
}
func (f *FileSectionReadCloser) Close() error {
return f.F.Close()
}

View File

@@ -1,42 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package meta
import "os"
// MetadataStorer defines the interface for managing metadata.
// When object == "", the operation is on the bucket.
type MetadataStorer interface {
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
// Returns the value of the attribute, or an error if the attribute does not exist.
RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error)
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
// If attribute already exists, new attribute should replace existing.
// Returns an error if the operation fails.
StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
// Returns an error if the operation fails.
DeleteAttribute(bucket, object, attribute string) error
// ListAttributes lists all attributes for an object or a bucket.
// Returns list of attribute names, or an error if the operation fails.
ListAttributes(bucket, object string) ([]string, error)
// DeleteAttributes removes all attributes for an object or a bucket.
// Returns an error if the operation fails.
DeleteAttributes(bucket, object string) error
}

View File

@@ -1,54 +0,0 @@
// Copyright 2025 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package meta
import (
"os"
)
// NoMeta is a metadata storer that does not store metadata.
// This can be useful for read only mounts where attempting to store metadata
// would fail.
type NoMeta struct{}
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
// always returns ErrNoSuchKey
func (NoMeta) RetrieveAttribute(_ *os.File, _, _, _ string) ([]byte, error) {
return nil, ErrNoSuchKey
}
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
// always returns nil without storing the attribute
func (NoMeta) StoreAttribute(_ *os.File, _, _, _ string, _ []byte) error {
return nil
}
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
// always returns nil without deleting the attribute
func (NoMeta) DeleteAttribute(_, _, _ string) error {
return nil
}
// ListAttributes lists all attributes for an object or a bucket.
// always returns an empty list of attributes
func (NoMeta) ListAttributes(_, _ string) ([]string, error) {
return []string{}, nil
}
// DeleteAttributes removes all attributes for an object or a bucket.
// always returns nil without deleting any attributes
func (NoMeta) DeleteAttributes(bucket, object string) error {
return nil
}

View File

@@ -1,139 +0,0 @@
// Copyright 2025 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package meta
import (
"errors"
"fmt"
"os"
"path/filepath"
)
// SideCar is a metadata storer that uses sidecar files to store metadata.
type SideCar struct {
dir string
}
const (
sidecarmeta = "meta"
)
// NewSideCar creates a new SideCar metadata storer.
func NewSideCar(dir string) (SideCar, error) {
fi, err := os.Lstat(dir)
if err != nil {
return SideCar{}, fmt.Errorf("failed to stat directory: %v", err)
}
if !fi.IsDir() {
return SideCar{}, fmt.Errorf("not a directory")
}
return SideCar{dir: dir}, nil
}
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) {
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
}
attr := filepath.Join(metadir, attribute)
value, err := os.ReadFile(attr)
if errors.Is(err, os.ErrNotExist) {
return nil, ErrNoSuchKey
}
if err != nil {
return nil, fmt.Errorf("failed to read attribute: %v", err)
}
return value, nil
}
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error {
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
}
err := os.MkdirAll(metadir, 0777)
if err != nil {
return fmt.Errorf("failed to create metadata directory: %v", err)
}
attr := filepath.Join(metadir, attribute)
err = os.WriteFile(attr, value, 0666)
if err != nil {
return fmt.Errorf("failed to write attribute: %v", err)
}
return nil
}
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
}
attr := filepath.Join(metadir, attribute)
err := os.Remove(attr)
if errors.Is(err, os.ErrNotExist) {
return ErrNoSuchKey
}
if err != nil {
return fmt.Errorf("failed to remove attribute: %v", err)
}
return nil
}
// ListAttributes lists all attributes for an object or a bucket.
func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
}
ents, err := os.ReadDir(metadir)
if errors.Is(err, os.ErrNotExist) {
return []string{}, nil
}
if err != nil {
return nil, fmt.Errorf("failed to list attributes: %v", err)
}
var attrs []string
for _, ent := range ents {
attrs = append(attrs, ent.Name())
}
return attrs, nil
}
// DeleteAttributes removes all attributes for an object or a bucket.
func (s SideCar) DeleteAttributes(bucket, object string) error {
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
}
err := os.RemoveAll(metadir)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("failed to remove attributes: %v", err)
}
return nil
}

View File

@@ -1,126 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package meta
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/pkg/xattr"
"github.com/versity/versitygw/s3err"
)
const (
xattrPrefix = "user."
)
var (
// ErrNoSuchKey is returned when the key does not exist.
ErrNoSuchKey = errors.New("no such key")
)
type XattrMeta struct{}
// RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
if f != nil {
b, err := xattr.FGet(f, xattrPrefix+attribute)
if errors.Is(err, xattr.ENOATTR) {
return nil, ErrNoSuchKey
}
return b, err
}
b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
if errors.Is(err, xattr.ENOATTR) {
return nil, ErrNoSuchKey
}
return b, err
}
// StoreAttribute stores the value of a specific attribute for an object in a bucket.
func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
if f != nil {
err := xattr.FSet(f, xattrPrefix+attribute, value)
if errors.Is(err, syscall.EROFS) {
return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return err
}
err := xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
if errors.Is(err, syscall.EROFS) {
return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return err
}
// DeleteAttribute removes the value of a specific attribute for an object in a bucket.
func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
err := xattr.Remove(filepath.Join(bucket, object), xattrPrefix+attribute)
if errors.Is(err, xattr.ENOATTR) {
return ErrNoSuchKey
}
if errors.Is(err, syscall.EROFS) {
return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return err
}
// DeleteAttributes is not implemented for xattr since xattrs
// are automatically removed when the file is deleted.
func (x XattrMeta) DeleteAttributes(bucket, object string) error {
return nil
}
// ListAttributes lists all attributes for an object in a bucket.
func (x XattrMeta) ListAttributes(bucket, object string) ([]string, error) {
attrs, err := xattr.List(filepath.Join(bucket, object))
if err != nil {
return nil, err
}
attributes := make([]string, 0, len(attrs))
for _, attr := range attrs {
if !isUserAttr(attr) {
continue
}
attributes = append(attributes, strings.TrimPrefix(attr, xattrPrefix))
}
return attributes, nil
}
func isUserAttr(attr string) bool {
return strings.HasPrefix(attr, xattrPrefix)
}
// Test is a helper function to test if xattrs are supported.
func (x XattrMeta) Test(path string) error {
// check for platform support
if !xattr.XATTR_SUPPORTED {
return fmt.Errorf("xattrs are not supported on this platform")
}
// check if the filesystem supports xattrs
_, err := xattr.Get(path, "user.test")
if errors.Is(err, syscall.ENOTSUP) {
return fmt.Errorf("xattrs are not supported on this filesystem")
}
return nil
}

View File

@@ -1,7 +1,6 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Copyright 2024 Versity Software
// MkdirAll borrowed from stdlib to add ability to set ownership
// as directories are created
@@ -15,6 +14,11 @@ import (
"github.com/versity/versitygw/s3err"
)
var (
// TODO: make this configurable
defaultDirPerm fs.FileMode = 0755
)
// MkdirAll is similar to os.MkdirAll but it will return
// ErrObjectParentIsFile when appropriate
// MkdirAll creates a directory named path,
@@ -25,9 +29,9 @@ import (
// Any newly created directory is set to provided uid/gid ownership.
// If path is already a directory, MkdirAll does nothing
// and returns nil.
// Any directory created will be set to provided uid/gid ownership
// Any directoy created will be set to provided uid/gid ownership
// if doChown is true.
func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) error {
func MkdirAll(path string, uid, gid int, doChown bool) error {
// Fast path: if we can tell whether path is a directory or file, stop with success or error.
dir, err := os.Stat(path)
if err == nil {
@@ -50,14 +54,14 @@ func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) erro
if j > 1 {
// Create parent.
err = MkdirAll(path[:j-1], uid, gid, doChown, dirPerm)
err = MkdirAll(path[:j-1], uid, gid, doChown)
if err != nil {
return err
}
}
// Parent now exists; invoke Mkdir and use its result.
err = os.Mkdir(path, dirPerm)
err = os.Mkdir(path, defaultDirPerm)
if err != nil {
// Handle arguments like "foo/." by
// double-checking that directory doesn't exist.

View File

@@ -1,516 +0,0 @@
// Copyright 2025 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package vgwplugin
import (
"bufio"
"context"
"fmt"
"plugin"
"reflect"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
// The plugin backend is used to dynamically load a Go plugin at runtime.
// It loads the plugin and calls the InitPlugin function to initialize it.
// A config string option is passed to init the plugin, it is expected that the
// plugin will handle its own configuration and initialization from this.
// If the plugin cannot be loaded or initialized, it returns an error.
// The InitPlugin function should be defined in the plugin and should have
// the signature func(configfile string) (version int, err error).
// The plugin should also implement the backend.Backend interface functions.
// However, the plugin does not need to implement all functions of the
// backend.Backend interface. It can implement only the functions it needs.
// Any non-implemented functions will return an error indicating that
// the function is not implemented.
// The plugin file should be compiled with the same Go version as the
// application using it. The plugin file should be built with the
// -buildmode=plugin flag.
// Example: go build -buildmode=plugin -o myplugin.so myplugin.go
// See the following for caveats and details:
// https://pkg.go.dev/plugin#hdr-Warnings
// PluginBackend implements the backend.Backend interface using Go plugins.
type PluginBackend struct {
p *plugin.Plugin
}
// NewPluginBackend creates a new PluginBackend. The path parameter should
// point to the compiled plugin file (e.g., .so file).
func NewPluginBackend(path, config string) (*PluginBackend, error) {
p, err := plugin.Open(path)
if err != nil {
return nil, fmt.Errorf("failed to open plugin: %w", err)
}
initSymbol, err := p.Lookup("InitPlugin")
if err != nil {
return nil, fmt.Errorf("failed to lookup InitPlugin symbol: %w", err)
}
initFunc, ok := initSymbol.(func(string) (int, error))
if !ok {
return nil, fmt.Errorf("InitPlugin symbol is not a func() (int, error)")
}
version, err := initFunc(config)
if err != nil {
return nil, fmt.Errorf("InitPlugin failed: %w", err)
}
if version != backend.InterfaceVersion {
return nil, fmt.Errorf("plugin interface version mismatch: gateway %v, plugin %v",
backend.InterfaceVersion, version)
}
return &PluginBackend{p: p}, nil
}
func (p *PluginBackend) callPluginFunc(name string, args []any) ([]reflect.Value, error) {
symbol, err := p.p.Lookup(name)
if err != nil {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
symbolValue := reflect.ValueOf(symbol)
if symbolValue.Kind() != reflect.Func {
return nil, fmt.Errorf("symbol %s is not a function", name)
}
numIn := symbolValue.Type().NumIn()
if len(args) != numIn {
return nil, fmt.Errorf("incorrect number of arguments for function %s, expected %d, got %d", name, numIn, len(args))
}
in := make([]reflect.Value, len(args))
for i := range args {
in[i] = reflect.ValueOf(args[i])
}
return symbolValue.Call(in), nil
}
func (p *PluginBackend) String() string { return "Plugin Gateway" }
func (p *PluginBackend) Shutdown() {}
func (p *PluginBackend) ListBuckets(ctx context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
results, err := p.callPluginFunc("ListBuckets", []any{ctx, input})
if err != nil {
return s3response.ListAllMyBucketsResult{}, err
}
return results[0].Interface().(s3response.ListAllMyBucketsResult), convertError(results[1])
}
func (p *PluginBackend) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
results, err := p.callPluginFunc("HeadBucket", []any{ctx, input})
if err != nil {
return nil, err
}
return results[0].Interface().(*s3.HeadBucketOutput), convertError(results[1])
}
func (p *PluginBackend) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
results, err := p.callPluginFunc("GetBucketAcl", []any{ctx, input})
if err != nil {
return nil, err
}
return results[0].Interface().([]byte), convertError(results[1])
}
func (p *PluginBackend) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, defaultACL []byte) error {
_, err := p.callPluginFunc("CreateBucket", []any{ctx, input, defaultACL})
return err
}
func (p *PluginBackend) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
_, err := p.callPluginFunc("PutBucketAcl", []any{ctx, bucket, data})
return err
}
func (p *PluginBackend) DeleteBucket(ctx context.Context, bucket string) error {
_, err := p.callPluginFunc("DeleteBucket", []any{ctx, bucket})
return err
}
func (p *PluginBackend) PutBucketVersioning(ctx context.Context, bucket string, status types.BucketVersioningStatus) error {
_, err := p.callPluginFunc("PutBucketVersioning", []any{ctx, bucket, status})
return err
}
func (p *PluginBackend) GetBucketVersioning(ctx context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
results, err := p.callPluginFunc("GetBucketVersioning", []any{ctx, bucket})
if err != nil {
return s3response.GetBucketVersioningOutput{}, err
}
return results[0].Interface().(s3response.GetBucketVersioningOutput), convertError(results[1])
}
func (p *PluginBackend) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
_, err := p.callPluginFunc("PutBucketPolicy", []any{ctx, bucket, policy})
return err
}
func (p *PluginBackend) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
results, err := p.callPluginFunc("GetBucketPolicy", []any{ctx, bucket})
if err != nil {
return nil, err
}
return results[0].Interface().([]byte), convertError(results[1])
}
func (p *PluginBackend) DeleteBucketPolicy(ctx context.Context, bucket string) error {
_, err := p.callPluginFunc("DeleteBucketPolicy", []any{ctx, bucket})
return err
}
func (p *PluginBackend) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
_, err := p.callPluginFunc("PutBucketOwnershipControls", []any{ctx, bucket, ownership})
return err
}
func (p *PluginBackend) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
results, err := p.callPluginFunc("GetBucketOwnershipControls", []any{ctx, bucket})
if err != nil {
return "", err
}
return results[0].Interface().(types.ObjectOwnership), convertError(results[1])
}
func (p *PluginBackend) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
_, err := p.callPluginFunc("DeleteBucketOwnershipControls", []any{ctx, bucket})
return err
}
func (p *PluginBackend) PutBucketCors(ctx context.Context, data []byte) error {
_, err := p.callPluginFunc("PutBucketCors", []any{ctx, data})
return err
}
func (p *PluginBackend) GetBucketCors(ctx context.Context, bucket string) ([]byte, error) {
results, err := p.callPluginFunc("GetBucketCors", []any{ctx, bucket})
if err != nil {
return nil, err
}
return results[0].Interface().([]byte), convertError(results[1])
}
func (p *PluginBackend) DeleteBucketCors(ctx context.Context, bucket string) error {
_, err := p.callPluginFunc("DeleteBucketCors", []any{ctx, bucket})
return err
}
func (p *PluginBackend) CreateMultipartUpload(ctx context.Context, input s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
results, err := p.callPluginFunc("CreateMultipartUpload", []any{ctx, input})
if err != nil {
return s3response.InitiateMultipartUploadResult{}, err
}
return results[0].Interface().(s3response.InitiateMultipartUploadResult), convertError(results[1])
}
func (p *PluginBackend) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
results, err := p.callPluginFunc("CompleteMultipartUpload", []any{ctx, input})
if err != nil {
return nil, err
}
return results[0].Interface().(*s3.CompleteMultipartUploadOutput), convertError(results[1])
}
func (p *PluginBackend) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
_, err := p.callPluginFunc("AbortMultipartUpload", []any{ctx, input})
return err
}
func (p *PluginBackend) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
results, err := p.callPluginFunc("ListMultipartUploads", []any{ctx, input})
if err != nil {
return s3response.ListMultipartUploadsResult{}, err
}
return results[0].Interface().(s3response.ListMultipartUploadsResult), convertError(results[1])
}
func (p *PluginBackend) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
results, err := p.callPluginFunc("ListParts", []any{ctx, input})
if err != nil {
return s3response.ListPartsResult{}, err
}
return results[0].Interface().(s3response.ListPartsResult), convertError(results[1])
}
func (p *PluginBackend) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
results, err := p.callPluginFunc("UploadPart", []any{ctx, input})
if err != nil {
return nil, err
}
return results[0].Interface().(*s3.UploadPartOutput), convertError(results[1])
}
func (p *PluginBackend) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
results, err := p.callPluginFunc("UploadPartCopy", []any{ctx, input})
if err != nil {
return s3response.CopyPartResult{}, err
}
return results[0].Interface().(s3response.CopyPartResult), convertError(results[1])
}
func (p *PluginBackend) PutObject(ctx context.Context, input s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
results, err := p.callPluginFunc("PutObject", []any{ctx, input})
if err != nil {
return s3response.PutObjectOutput{}, err
}
return results[0].Interface().(s3response.PutObjectOutput), convertError(results[1])
}
func (p *PluginBackend) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
results, err := p.callPluginFunc("HeadObject", []any{ctx, input})
if err != nil {
return nil, err
}
return results[0].Interface().(*s3.HeadObjectOutput), convertError(results[1])
}
func (p *PluginBackend) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
results, err := p.callPluginFunc("GetObject", []any{ctx, input})
if err != nil {
return nil, err
}
return results[0].Interface().(*s3.GetObjectOutput), convertError(results[1])
}
func (p *PluginBackend) GetObjectAcl(ctx context.Context, input *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
results, err := p.callPluginFunc("GetObjectAcl", []any{ctx, input})
if err != nil {
return nil, err
}
return results[0].Interface().(*s3.GetObjectAclOutput), convertError(results[1])
}
func (p *PluginBackend) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
results, err := p.callPluginFunc("GetObjectAttributes", []any{ctx, input})
if err != nil {
return s3response.GetObjectAttributesResponse{}, err
}
return results[0].Interface().(s3response.GetObjectAttributesResponse), convertError(results[1])
}
func (p *PluginBackend) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
results, err := p.callPluginFunc("CopyObject", []any{ctx, input})
if err != nil {
return nil, err
}
return results[0].Interface().(*s3.CopyObjectOutput), convertError(results[1])
}
func (p *PluginBackend) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
results, err := p.callPluginFunc("ListObjects", []any{ctx, input})
if err != nil {
return s3response.ListObjectsResult{}, err
}
return results[0].Interface().(s3response.ListObjectsResult), convertError(results[1])
}
func (p *PluginBackend) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
results, err := p.callPluginFunc("ListObjectsV2", []any{ctx, input})
if err != nil {
return s3response.ListObjectsV2Result{}, err
}
return results[0].Interface().(s3response.ListObjectsV2Result), convertError(results[1])
}
func (p *PluginBackend) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
results, err := p.callPluginFunc("DeleteObject", []any{ctx, input})
if err != nil {
return nil, err
}
return results[0].Interface().(*s3.DeleteObjectOutput), convertError(results[1])
}
func (p *PluginBackend) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
results, err := p.callPluginFunc("DeleteObjects", []any{ctx, input})
if err != nil {
return s3response.DeleteResult{}, err
}
return results[0].Interface().(s3response.DeleteResult), convertError(results[1])
}
func (p *PluginBackend) PutObjectAcl(ctx context.Context, input *s3.PutObjectAclInput) error {
_, err := p.callPluginFunc("PutObjectAcl", []any{ctx, input})
return err
}
func (p *PluginBackend) ListObjectVersions(ctx context.Context, input *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
results, err := p.callPluginFunc("ListObjectVersions", []any{ctx, input})
if err != nil {
return s3response.ListVersionsResult{}, err
}
return results[0].Interface().(s3response.ListVersionsResult), convertError(results[1])
}
func (p *PluginBackend) RestoreObject(ctx context.Context, input *s3.RestoreObjectInput) error {
_, err := p.callPluginFunc("RestoreObject", []any{ctx, input})
return err
}
func (p *PluginBackend) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
results, err := p.callPluginFunc("SelectObjectContent", []any{ctx, input})
if err != nil {
return func(w *bufio.Writer) {}
}
return results[0].Interface().(func(w *bufio.Writer))
}
func (p *PluginBackend) GetBucketTagging(ctx context.Context, bucket string) (map[string]string, error) {
results, err := p.callPluginFunc("GetBucketTagging", []any{ctx, bucket})
if err != nil {
return nil, err
}
return results[0].Interface().(map[string]string), convertError(results[1])
}
func (p *PluginBackend) PutBucketTagging(ctx context.Context, bucket string, tags map[string]string) error {
_, err := p.callPluginFunc("PutBucketTagging", []any{ctx, bucket, tags})
return err
}
func (p *PluginBackend) DeleteBucketTagging(ctx context.Context, bucket string) error {
_, err := p.callPluginFunc("DeleteBucketTagging", []any{ctx, bucket})
return err
}
func (p *PluginBackend) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
results, err := p.callPluginFunc("GetObjectTagging", []any{ctx, bucket, object})
if err != nil {
return nil, err
}
return results[0].Interface().(map[string]string), convertError(results[1])
}
func (p *PluginBackend) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
_, err := p.callPluginFunc("PutObjectTagging", []any{ctx, bucket, object, tags})
return err
}
func (p *PluginBackend) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
_, err := p.callPluginFunc("DeleteObjectTagging", []any{ctx, bucket, object})
return err
}
func (p *PluginBackend) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
_, err := p.callPluginFunc("PutObjectLockConfiguration", []any{ctx, bucket, config})
return err
}
func (p *PluginBackend) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) {
results, err := p.callPluginFunc("GetObjectLockConfiguration", []any{ctx, bucket})
if err != nil {
return nil, err
}
return results[0].Interface().([]byte), convertError(results[1])
}
func (p *PluginBackend) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
_, err := p.callPluginFunc("PutObjectRetention", []any{ctx, bucket, object, versionId, bypass, retention})
return err
}
func (p *PluginBackend) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) {
results, err := p.callPluginFunc("GetObjectRetention", []any{ctx, bucket, object, versionId})
if err != nil {
return nil, err
}
return results[0].Interface().([]byte), convertError(results[1])
}
func (p *PluginBackend) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
_, err := p.callPluginFunc("PutObjectLegalHold", []any{ctx, bucket, object, versionId, status})
return err
}
func (p *PluginBackend) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) {
results, err := p.callPluginFunc("GetObjectLegalHold", []any{ctx, bucket, object, versionId})
if err != nil {
return nil, err
}
val := results[0].Interface()
if val == nil {
return nil, convertError(results[1])
}
return val.(*bool), convertError(results[1])
}
func (p *PluginBackend) ChangeBucketOwner(ctx context.Context, bucket string, acl []byte) error {
_, err := p.callPluginFunc("ChangeBucketOwner", []any{ctx, bucket, acl})
return err
}
func (p *PluginBackend) ListBucketsAndOwners(ctx context.Context) ([]s3response.Bucket, error) {
results, err := p.callPluginFunc("ListBucketsAndOwners", []any{ctx})
if err != nil {
return nil, err
}
return results[0].Interface().([]s3response.Bucket), convertError(results[1])
}
func convertError(result reflect.Value) error {
if result.IsNil() {
return nil
}
err, ok := result.Interface().(error)
if !ok {
return fmt.Errorf("expected error, got %T", result.Interface())
}
return err
}
var _ backend.Backend = &PluginBackend{}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,24 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build !freebsd && !openbsd && !netbsd
// +build !freebsd,!openbsd,!netbsd
package posix
import "syscall"
var (
errNoData = syscall.ENODATA
)

View File

@@ -29,7 +29,6 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
"golang.org/x/sys/unix"
)
@@ -44,7 +43,6 @@ type tmpfile struct {
needsChown bool
uid int
gid int
newDirPerm fs.FileMode
}
var (
@@ -52,7 +50,7 @@ var (
defaultFilePerm uint32 = 0644
)
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool) (*tmpfile, error) {
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
uid, gid, doChown := p.getChownIDs(acct)
// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
@@ -63,12 +61,8 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
// this is not supported.
fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
if err != nil {
if errors.Is(err, syscall.EROFS) {
return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
// O_TMPFILE not supported, try fallback
err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
err = backend.MkdirAll(dir, uid, gid, doChown)
if err != nil {
return nil, fmt.Errorf("make temp dir: %w", err)
}
@@ -87,7 +81,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
gid: gid,
}
// falloc is best effort, its fine if this fails
if size > 0 && dofalloc {
if size > 0 {
tmp.falloc()
}
@@ -114,11 +108,10 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
needsChown: doChown,
uid: uid,
gid: gid,
newDirPerm: p.newDirPerm,
}
// falloc is best effort, its fine if this fails
if size > 0 && dofalloc {
if size > 0 {
tmp.falloc()
}
@@ -141,9 +134,6 @@ func (tmp *tmpfile) falloc() error {
}
func (tmp *tmpfile) link() error {
// make sure this is cleaned up in all error cases
defer tmp.f.Close()
// We use Linkat/Rename as the atomic operation for object puts. The
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
// with any other simultaneous uploads. The final operation is to move the
@@ -158,7 +148,7 @@ func (tmp *tmpfile) link() error {
dir := filepath.Dir(objPath)
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
if err != nil {
return fmt.Errorf("make parent dir: %w", err)
}
@@ -180,21 +170,11 @@ func (tmp *tmpfile) link() error {
}
defer dirf.Close()
for {
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
if errors.Is(err, syscall.EEXIST) {
err := os.Remove(objPath)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove stale path: %w", err)
}
continue
}
if err != nil {
return fmt.Errorf("link tmpfile (fd %q as %q): %w",
filepath.Base(tmp.f.Name()), objPath, err)
}
break
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
if err != nil {
return fmt.Errorf("link tmpfile (%q in %q): %w",
filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
}
err = tmp.f.Close()
@@ -241,7 +221,3 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
func (tmp *tmpfile) cleanup() {
tmp.f.Close()
}
func (tmp *tmpfile) File() *os.File {
return tmp.f
}

View File

@@ -0,0 +1,24 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build freebsd || openbsd || netbsd
// +build freebsd openbsd netbsd
package posix
import "syscall"
var (
errNoData = syscall.ENOATTR
)

View File

@@ -24,11 +24,9 @@ import (
"io/fs"
"os"
"path/filepath"
"syscall"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
)
type tmpfile struct {
@@ -38,24 +36,18 @@ type tmpfile struct {
size int64
}
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool) (*tmpfile, error) {
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
uid, gid, doChown := p.getChownIDs(acct)
// Create a temp file for upload while in progress (see link comments below).
var err error
err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
err = backend.MkdirAll(dir, uid, gid, doChown)
if err != nil {
if errors.Is(err, syscall.EROFS) {
return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return nil, fmt.Errorf("make temp dir: %w", err)
}
f, err := os.CreateTemp(dir,
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
if err != nil {
if errors.Is(err, syscall.EROFS) {
return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return nil, fmt.Errorf("create temp file: %w", err)
}
@@ -120,7 +112,3 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
func (tmp *tmpfile) cleanup() {
tmp.f.Close()
}
func (tmp *tmpfile) File() *os.File {
return tmp.f
}

View File

@@ -33,12 +33,6 @@ func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
return nil, err
}
if s.endpoint != "" {
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = &s.endpoint
}), nil
}
return s3.NewFromConfig(cfg), nil
}
@@ -56,6 +50,11 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
config.WithHTTPClient(client),
}
if s.endpoint != "" {
opts = append(opts,
config.WithEndpointResolverWithOptions(s))
}
if s.disableChecksum {
opts = append(opts,
config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
@@ -68,3 +67,13 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
return config.LoadDefaultConfig(ctx, opts...)
}
// ResolveEndpoint is used for on prem or non-aws endpoints
func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
URL: s.endpoint,
SigningRegion: s.awsRegion,
HostnameImmutable: true,
}, nil
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -23,24 +23,21 @@ import (
"os"
"path/filepath"
"strconv"
"syscall"
"golang.org/x/sys/unix"
"github.com/versity/scoutfs-go"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/backend/posix"
)
func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
metastore := meta.XattrMeta{}
p, err := posix.New(rootdir, metastore, posix.PosixOpts{
ChownUID: opts.ChownUID,
ChownGID: opts.ChownGID,
BucketLinks: opts.BucketLinks,
NewDirPerm: opts.NewDirPerm,
p, err := posix.New(rootdir, posix.PosixOpts{
ChownUID: opts.ChownUID,
ChownGID: opts.ChownGID,
ReadOnly: opts.ReadOnly,
})
if err != nil {
return nil, err
@@ -52,15 +49,12 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
}
return &ScoutFS{
Posix: p,
rootfd: f,
rootdir: rootdir,
meta: metastore,
chownuid: opts.ChownUID,
chowngid: opts.ChownGID,
glaciermode: opts.GlacierMode,
newDirPerm: opts.NewDirPerm,
disableNoArchive: opts.DisableNoArchive,
Posix: p,
rootfd: f,
rootdir: rootdir,
chownuid: opts.ChownUID,
chowngid: opts.ChownGID,
readonly: opts.ReadOnly,
}, nil
}
@@ -74,10 +68,10 @@ type tmpfile struct {
needsChown bool
uid int
gid int
newDirPerm fs.FileMode
}
var (
// TODO: make this configurable
defaultFilePerm uint32 = 0644
)
@@ -105,7 +99,11 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
needsChown: doChown,
uid: uid,
gid: gid,
newDirPerm: s.newDirPerm,
}
// falloc is best effort, its fine if this fails
if size > 0 {
tmp.falloc()
}
if doChown {
@@ -118,6 +116,14 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
return tmp, nil
}
func (tmp *tmpfile) falloc() error {
err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
if err != nil {
return fmt.Errorf("fallocate: %v", err)
}
return nil
}
func (tmp *tmpfile) link() error {
// We use Linkat/Rename as the atomic operation for object puts. The
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
@@ -133,7 +139,7 @@ func (tmp *tmpfile) link() error {
dir := filepath.Dir(objPath)
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
if err != nil {
return fmt.Errorf("make parent dir: %w", err)
}
@@ -178,10 +184,6 @@ func (tmp *tmpfile) cleanup() {
tmp.f.Close()
}
func (tmp *tmpfile) File() *os.File {
return tmp.f
}
func moveData(from *os.File, to *os.File) error {
return scoutfs.MoveData(from, to)
}

View File

@@ -28,7 +28,9 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
return nil, fmt.Errorf("scoutfs only available on linux")
}
type tmpfile struct{}
type tmpfile struct {
f *os.File
}
var (
errNotSupported = errors.New("not supported")
@@ -40,6 +42,7 @@ func (s *ScoutFS) openTmpFile(_, _, _ string, _ int64, _ auth.Account) (*tmpfile
_ = s.chowngid
_ = s.euid
_ = s.egid
_ = s.readonly
return nil, errNotSupported
}
@@ -54,10 +57,6 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
func (tmp *tmpfile) cleanup() {
}
func (tmp *tmpfile) File() *os.File {
return nil
}
func moveData(_, _ *os.File) error {
return errNotSupported
}

View File

@@ -0,0 +1,24 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build !freebsd && !openbsd && !netbsd
// +build !freebsd,!openbsd,!netbsd
package scoutfs
import "syscall"
var (
errNoData = syscall.ENODATA
)

View File

@@ -0,0 +1,24 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build freebsd || openbsd || netbsd
// +build freebsd openbsd netbsd
package scoutfs
import "syscall"
var (
errNoData = syscall.ENOATTR
)

View File

@@ -15,66 +15,46 @@
package backend
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"sort"
"strings"
"syscall"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3response"
)
type WalkResults struct {
CommonPrefixes []types.CommonPrefix
Objects []s3response.Object
Objects []types.Object
Truncated bool
NextMarker string
}
type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
var ErrSkipObj = errors.New("skip this object")
// Walk walks the supplied fs.FS and returns results compatible with list
// objects responses
func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
cpmap := make(map[string]struct{})
var objects []s3response.Object
// if max is 0, it should return empty non-truncated result
if max == 0 {
return WalkResults{
Truncated: false,
}, nil
}
var objects []types.Object
var pastMarker bool
if marker == "" {
pastMarker = true
}
var pastMax bool
pastMax := max == 0
var newMarker string
var truncated bool
root := "."
if strings.Contains(prefix, "/") {
idx := strings.LastIndex(prefix, "/")
if idx > 0 {
root = prefix[:idx]
}
}
err := fs.WalkDir(fileSystem, root, func(path string, d fs.DirEntry, err error) error {
err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if ctx.Err() != nil {
return ctx.Err()
}
// Ignore the root directory
if path == "." {
return nil
@@ -83,9 +63,14 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
return fs.SkipDir
}
// After this point, return skipflag instead of nil
// so we can skip a directory without an early return
var skipflag error
if pastMax {
if len(objects) != 0 {
newMarker = *objects[len(objects)-1].Key
truncated = true
}
return fs.SkipAll
}
if d.IsDir() {
// If prefix is defined and the directory does not match prefix,
// do not descend into the directory because nothing will
@@ -95,66 +80,44 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
// building to match. So only skip if path isn't a prefix of prefix
// and prefix isn't a prefix of path.
if prefix != "" &&
!strings.HasPrefix(path+"/", prefix) &&
!strings.HasPrefix(prefix, path+"/") {
!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
return fs.SkipDir
}
// Don't recurse into subdirectories which contain the delimiter
// after reaching the prefix
if delimiter != "" &&
strings.HasPrefix(path+"/", prefix) &&
strings.Contains(strings.TrimPrefix(path+"/", prefix), delimiter) {
skipflag = fs.SkipDir
} else {
if delimiter == "" {
dirobj, err := getObj(path+"/", d)
if err == ErrSkipObj {
return skipflag
}
if err != nil {
return fmt.Errorf("directory to object %q: %w", path, err)
}
if pastMax {
truncated = true
return fs.SkipAll
}
objects = append(objects, dirobj)
if (len(objects) + len(cpmap)) == int(max) {
newMarker = path
pastMax = true
}
return skipflag
}
// TODO: can we do better here rather than a second readdir
// per directory?
ents, err := fs.ReadDir(fileSystem, path)
if err != nil {
return fmt.Errorf("readdir %q: %w", path, err)
}
if len(ents) != 0 {
return skipflag
}
// TODO: can we do better here rather than a second readdir
// per directory?
ents, err := fs.ReadDir(fileSystem, path)
if err != nil {
return fmt.Errorf("readdir %q: %w", path, err)
}
path += "/"
if len(ents) == 0 {
dirobj, err := getObj(path, d)
if err == ErrSkipObj {
return nil
}
if err != nil {
return fmt.Errorf("directory to object %q: %w", path, err)
}
objects = append(objects, dirobj)
}
return nil
}
if !pastMarker {
if path == marker {
pastMarker = true
return skipflag
return nil
}
if path < marker {
return skipflag
return nil
}
}
// If object doesn't have prefix, don't include in results.
if prefix != "" && !strings.HasPrefix(path, prefix) {
return skipflag
return nil
}
if delimiter == "" {
@@ -162,24 +125,18 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
// prefix are included in results
obj, err := getObj(path, d)
if err == ErrSkipObj {
return skipflag
return nil
}
if err != nil {
return fmt.Errorf("file to object %q: %w", path, err)
}
if pastMax {
truncated = true
return fs.SkipAll
}
objects = append(objects, obj)
if (len(objects) + len(cpmap)) == int(max) {
newMarker = path
if max > 0 && (len(objects)+len(cpmap)) == int(max) {
pastMax = true
}
return skipflag
return nil
}
// Since delimiter is specified, we only want results that
@@ -208,55 +165,29 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
if !found {
obj, err := getObj(path, d)
if err == ErrSkipObj {
return skipflag
return nil
}
if err != nil {
return fmt.Errorf("file to object %q: %w", path, err)
}
if pastMax {
truncated = true
return fs.SkipAll
}
objects = append(objects, obj)
if (len(objects) + len(cpmap)) == int(max) {
newMarker = path
pastMax = true
}
return skipflag
return nil
}
// Common prefixes are a set, so should not have duplicates.
// These are abstractly a "directory", so need to include the
// delimiter at the end when we add to the map.
cprefNoDelim := prefix + before
cpref := prefix + before + delimiter
if cpref == marker {
pastMarker = true
return skipflag
}
if marker != "" && strings.HasPrefix(marker, cprefNoDelim) {
// skip common prefixes that are before the marker
return skipflag
}
if pastMax {
truncated = true
return fs.SkipAll
}
cpmap[cpref] = struct{}{}
// delimiter at the end.
cpmap[prefix+before+delimiter] = struct{}{}
if (len(objects) + len(cpmap)) == int(max) {
newMarker = cpref
pastMax = true
}
return skipflag
return nil
})
if err != nil {
// suppress file not found caused by user's prefix
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
return WalkResults{}, nil
}
return WalkResults{}, err
}
@@ -273,10 +204,6 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
})
}
if !truncated {
newMarker = ""
}
return WalkResults{
CommonPrefixes: commonPrefixes,
Objects: objects,
@@ -293,213 +220,3 @@ func contains(a string, strs []string) bool {
}
return false
}
type WalkVersioningResults struct {
CommonPrefixes []types.CommonPrefix
ObjectVersions []types.ObjectVersion
DelMarkers []types.DeleteMarkerEntry
Truncated bool
NextMarker string
NextVersionIdMarker string
}
type ObjVersionFuncResult struct {
ObjectVersions []types.ObjectVersion
DelMarkers []types.DeleteMarkerEntry
NextVersionIdMarker string
Truncated bool
}
type GetVersionsFunc func(path, versionIdMarker string, pastVersionIdMarker *bool, availableObjCount int, d fs.DirEntry) (*ObjVersionFuncResult, error)
// WalkVersions walks the supplied fs.FS and returns results compatible with
// ListObjectVersions action response
func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyMarker, versionIdMarker string, max int, getObj GetVersionsFunc, skipdirs []string) (WalkVersioningResults, error) {
cpmap := make(map[string]struct{})
var objects []types.ObjectVersion
var delMarkers []types.DeleteMarkerEntry
var pastMarker bool
if keyMarker == "" {
pastMarker = true
}
var nextMarker string
var nextVersionIdMarker string
var truncated bool
pastVersionIdMarker := versionIdMarker == ""
err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if ctx.Err() != nil {
return ctx.Err()
}
// Ignore the root directory
if path == "." {
return nil
}
if contains(d.Name(), skipdirs) {
return fs.SkipDir
}
if !pastMarker {
if path == keyMarker {
pastMarker = true
}
if path < keyMarker {
return nil
}
}
if d.IsDir() {
// If prefix is defined and the directory does not match prefix,
// do not descend into the directory because nothing will
// match this prefix. Make sure to append the / at the end of
// directories since this is implied as a directory path name.
// If path is a prefix of prefix, then path could still be
// building to match. So only skip if path isn't a prefix of prefix
// and prefix isn't a prefix of path.
if prefix != "" &&
!strings.HasPrefix(path+"/", prefix) &&
!strings.HasPrefix(prefix, path+"/") {
return fs.SkipDir
}
// Don't recurse into subdirectories when listing with delimiter.
if delimiter == "/" &&
prefix != path+"/" &&
strings.HasPrefix(path+"/", prefix) {
cpmap[path+"/"] = struct{}{}
return fs.SkipDir
}
res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
if err == ErrSkipObj {
return nil
}
if err != nil {
return fmt.Errorf("directory to object %q: %w", path, err)
}
objects = append(objects, res.ObjectVersions...)
delMarkers = append(delMarkers, res.DelMarkers...)
if res.Truncated {
truncated = true
nextMarker = path
nextVersionIdMarker = res.NextVersionIdMarker
return fs.SkipAll
}
return nil
}
// If object doesn't have prefix, don't include in results.
if prefix != "" && !strings.HasPrefix(path, prefix) {
return nil
}
if delimiter == "" {
// If no delimiter specified, then all files with matching
// prefix are included in results
res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
if err == ErrSkipObj {
return nil
}
if err != nil {
return fmt.Errorf("file to object %q: %w", path, err)
}
objects = append(objects, res.ObjectVersions...)
delMarkers = append(delMarkers, res.DelMarkers...)
if res.Truncated {
truncated = true
nextMarker = path
nextVersionIdMarker = res.NextVersionIdMarker
return fs.SkipAll
}
return nil
}
// Since delimiter is specified, we only want results that
// do not contain the delimiter beyond the prefix. If the
// delimiter exists past the prefix, then the substring
// between the prefix and delimiter is part of common prefixes.
//
// For example:
// prefix = A/
// delimiter = /
// and objects:
// A/file
// A/B/file
// B/C
// would return:
// objects: A/file
// common prefix: A/B/
//
// Note: No objects are included past the common prefix since
// these are all rolled up into the common prefix.
// Note: The delimiter can be anything, so we have to operate on
// the full path without any assumptions on posix directory hierarchy
// here. Usually the delimiter will be "/", but thats not required.
suffix := strings.TrimPrefix(path, prefix)
before, _, found := strings.Cut(suffix, delimiter)
if !found {
res, err := getObj(path, versionIdMarker, &pastVersionIdMarker, max-len(objects)-len(delMarkers)-len(cpmap), d)
if err == ErrSkipObj {
return nil
}
if err != nil {
return fmt.Errorf("file to object %q: %w", path, err)
}
objects = append(objects, res.ObjectVersions...)
delMarkers = append(delMarkers, res.DelMarkers...)
if res.Truncated {
truncated = true
nextMarker = path
nextVersionIdMarker = res.NextVersionIdMarker
return fs.SkipAll
}
return nil
}
// Common prefixes are a set, so should not have duplicates.
// These are abstractly a "directory", so need to include the
// delimiter at the end.
cpmap[prefix+before+delimiter] = struct{}{}
if (len(objects) + len(cpmap)) == int(max) {
nextMarker = path
truncated = true
return fs.SkipAll
}
return nil
})
if err != nil {
return WalkVersioningResults{}, err
}
var commonPrefixStrings []string
for k := range cpmap {
commonPrefixStrings = append(commonPrefixStrings, k)
}
sort.Strings(commonPrefixStrings)
commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
for _, cp := range commonPrefixStrings {
pfx := cp
commonPrefixes = append(commonPrefixes, types.CommonPrefix{
Prefix: &pfx,
})
}
return WalkVersioningResults{
CommonPrefixes: commonPrefixes,
ObjectVersions: objects,
DelMarkers: delMarkers,
Truncated: truncated,
NextMarker: nextMarker,
NextVersionIdMarker: nextVersionIdMarker,
}, nil
}

View File

@@ -15,50 +15,36 @@
package backend_test
import (
"context"
"crypto/md5"
"encoding/hex"
"fmt"
"io/fs"
"sync"
"testing"
"testing/fstest"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3response"
)
type walkTest struct {
fsys fs.FS
getobj backend.GetObjFunc
cases []testcase
fsys fs.FS
expected backend.WalkResults
getobj backend.GetObjFunc
}
type testcase struct {
name string
prefix string
delimiter string
marker string
maxObjs int32
expected backend.WalkResults
}
func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
func getObj(path string, d fs.DirEntry) (types.Object, error) {
if d.IsDir() {
etag := getMD5(path)
fi, err := d.Info()
if err != nil {
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
mtime := fi.ModTime()
return s3response.Object{
return types.Object{
ETag: &etag,
Key: &path,
LastModified: &mtime,
LastModified: backend.GetTimePtr(fi.ModTime()),
}, nil
}
@@ -66,16 +52,15 @@ func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
fi, err := d.Info()
if err != nil {
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
size := fi.Size()
mtime := fi.ModTime()
return s3response.Object{
return types.Object{
ETag: &etag,
Key: &path,
LastModified: &mtime,
LastModified: backend.GetTimePtr(fi.ModTime()),
Size: &size,
}, nil
}
@@ -97,154 +82,50 @@ func TestWalk(t *testing.T) {
"photos/2006/February/sample3.jpg": {},
"photos/2006/February/sample4.jpg": {},
},
getobj: getObj,
cases: []testcase{
{
name: "aws example",
delimiter: "/",
maxObjs: 1000,
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("photos/"),
}},
Objects: []s3response.Object{{
Key: backend.GetPtrFromString("sample.jpg"),
}},
},
},
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetStringPtr("photos/"),
}},
Objects: []types.Object{{
Key: backend.GetStringPtr("sample.jpg"),
}},
},
getobj: getObj,
},
{
// test case single dir/single file
fsys: fstest.MapFS{
"test/file": {},
},
getobj: getObj,
cases: []testcase{
{
name: "single dir single file",
delimiter: "/",
maxObjs: 1000,
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("test/"),
}},
Objects: []s3response.Object{},
},
},
},
},
{
// non-standard delimiter
fsys: fstest.MapFS{
"photo|s/200|6/Januar|y/sampl|e1.jpg": {},
"photo|s/200|6/Januar|y/sampl|e2.jpg": {},
"photo|s/200|6/Januar|y/sampl|e3.jpg": {},
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetStringPtr("test/"),
}},
Objects: []types.Object{},
},
getobj: getObj,
cases: []testcase{
{
name: "different delimiter 1",
delimiter: "|",
maxObjs: 1000,
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("photo|"),
}},
},
},
{
name: "different delimiter 2",
delimiter: "|",
maxObjs: 1000,
prefix: "photo|",
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("photo|s/200|"),
}},
},
},
{
name: "different delimiter 3",
delimiter: "|",
maxObjs: 1000,
prefix: "photo|s/200|",
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
}},
},
},
{
name: "different delimiter 4",
delimiter: "|",
maxObjs: 1000,
prefix: "photo|s/200|",
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
}},
},
},
{
name: "different delimiter 5",
delimiter: "|",
maxObjs: 1000,
prefix: "photo|s/200|6/Januar|",
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|"),
}},
},
},
{
name: "different delimiter 6",
delimiter: "|",
maxObjs: 1000,
prefix: "photo|s/200|6/Januar|y/sampl|",
expected: backend.WalkResults{
Objects: []s3response.Object{
{
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e1.jpg"),
},
{
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e2.jpg"),
},
{
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e3.jpg"),
},
},
},
},
},
},
}
for _, tt := range tests {
for _, tc := range tt.cases {
res, err := backend.Walk(context.Background(),
tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs,
tt.getobj, []string{})
if err != nil {
t.Errorf("tc.name: walk: %v", err)
}
compareResults(tc.name, res, tc.expected, t)
res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
if err != nil {
t.Fatalf("walk: %v", err)
}
compareResults(res, tt.expected, t)
}
}
func compareResults(name string, got, wanted backend.WalkResults, t *testing.T) {
func compareResults(got, wanted backend.WalkResults, t *testing.T) {
if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
t.Errorf("%v: unexpected common prefix, got %v wanted %v",
name,
t.Errorf("unexpected common prefix, got %v wanted %v",
printCommonPrefixes(got.CommonPrefixes),
printCommonPrefixes(wanted.CommonPrefixes))
}
if !compareObjects(got.Objects, wanted.Objects) {
t.Errorf("%v: unexpected object, got %v wanted %v",
name,
t.Errorf("unexpected object, got %v wanted %v",
printObjects(got.Objects),
printObjects(wanted.Objects))
}
@@ -287,7 +168,7 @@ func printCommonPrefixes(list []types.CommonPrefix) string {
return res + "]"
}
func compareObjects(a, b []s3response.Object) bool {
func compareObjects(a, b []types.Object) bool {
if len(a) == 0 && len(b) == 0 {
return true
}
@@ -303,7 +184,7 @@ func compareObjects(a, b []s3response.Object) bool {
return false
}
func containsObject(c s3response.Object, list []s3response.Object) bool {
func containsObject(c types.Object, list []types.Object) bool {
for _, cp := range list {
if *c.Key == *cp.Key {
return true
@@ -312,67 +193,14 @@ func containsObject(c s3response.Object, list []s3response.Object) bool {
return false
}
func printObjects(list []s3response.Object) string {
func printObjects(list []types.Object) string {
res := "["
for _, cp := range list {
var key string
if cp.Key == nil {
key = "<nil>"
} else {
key = *cp.Key
}
if res == "[" {
res = res + key
res = res + *cp.Key
} else {
res = res + ", " + key
res = res + ", " + *cp.Key
}
}
return res + "]"
}
type slowFS struct {
fstest.MapFS
}
const (
readDirPause = 100 * time.Millisecond
// walkTimeOut should be less than the tree traversal time
// which is the readdirPause time * the number of directories
walkTimeOut = 500 * time.Millisecond
)
func (s *slowFS) ReadDir(name string) ([]fs.DirEntry, error) {
time.Sleep(readDirPause)
return s.MapFS.ReadDir(name)
}
func TestWalkStop(t *testing.T) {
s := &slowFS{MapFS: fstest.MapFS{
"/a/b/c/d/e/f/g/h/i/g/k/l/m/n": &fstest.MapFile{},
}}
ctx, cancel := context.WithTimeout(context.Background(), walkTimeOut)
defer cancel()
var err error
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
_, err = backend.Walk(ctx, s, "", "/", "", 1000,
func(path string, d fs.DirEntry) (s3response.Object, error) {
return s3response.Object{}, nil
}, []string{})
}()
select {
case <-time.After(1 * time.Second):
t.Fatalf("walk is not terminated in time")
case <-ctx.Done():
}
wg.Wait()
if err != ctx.Err() {
t.Fatalf("unexpected error: %v", err)
}
}

View File

@@ -17,9 +17,8 @@ package main
import (
"bytes"
"crypto/sha256"
"crypto/tls"
"encoding/hex"
"encoding/xml"
"encoding/json"
"fmt"
"io"
"net/http"
@@ -29,7 +28,6 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/smithy-go"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3response"
@@ -38,9 +36,7 @@ import (
var (
adminAccess string
adminSecret string
adminRegion string
adminEndpoint string
allowInsecure bool
)
func adminCommand() *cli.Command {
@@ -82,33 +78,10 @@ func adminCommand() *cli.Command {
Usage: "groupID for the new user",
Aliases: []string{"gi"},
},
},
},
{
Name: "update-user",
Usage: "Updates a user account",
Action: updateUser,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "access",
Usage: "user access key id to be updated",
Required: true,
Aliases: []string{"a"},
},
&cli.StringFlag{
Name: "secret",
Usage: "secret access key for the new user",
Aliases: []string{"s"},
},
&cli.IntFlag{
Name: "user-id",
Usage: "userID for the new user",
Aliases: []string{"ui"},
},
&cli.IntFlag{
Name: "group-id",
Usage: "groupID for the new user",
Aliases: []string{"gi"},
Name: "project-id",
Usage: "projectID for the new user",
Aliases: []string{"pi"},
},
},
},
@@ -173,14 +146,6 @@ func adminCommand() *cli.Command {
Required: true,
Destination: &adminSecret,
},
&cli.StringFlag{
Name: "region",
Usage: "admin s3 region string",
EnvVars: []string{"ADMIN_REGION"},
Value: "us-east-1",
Destination: &adminRegion,
Aliases: []string{"r"},
},
&cli.StringFlag{
Name: "endpoint-url",
Usage: "admin apis endpoint url",
@@ -189,80 +154,65 @@ func adminCommand() *cli.Command {
Required: true,
Destination: &adminEndpoint,
},
&cli.BoolFlag{
Name: "allow-insecure",
Usage: "disable tls certificate verification for the admin endpoint",
EnvVars: []string{"ADMIN_ALLOW_INSECURE"},
Aliases: []string{"ai"},
Destination: &allowInsecure,
},
},
}
}
func initHTTPClient() *http.Client {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: allowInsecure},
}
return &http.Client{Transport: tr}
}
func createUser(ctx *cli.Context) error {
access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
userID, groupID := ctx.Int("user-id"), ctx.Int("group-id")
userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
if access == "" || secret == "" {
return fmt.Errorf("invalid input parameters for the new user access/secret keys")
return fmt.Errorf("invalid input parameters for the new user")
}
if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
return fmt.Errorf("invalid input parameter for role: %v", role)
}
acc := auth.Account{
Access: access,
Secret: secret,
Role: auth.Role(role),
UserID: userID,
GroupID: groupID,
Access: access,
Secret: secret,
Role: auth.Role(role),
UserID: userID,
GroupID: groupID,
ProjectID: projectID,
}
accxml, err := xml.Marshal(acc)
accJson, err := json.Marshal(acc)
if err != nil {
return fmt.Errorf("failed to parse user data: %w", err)
}
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accxml))
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256(accxml)
hashedPayload := sha256.Sum256(accJson)
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return parseApiError(body)
}
fmt.Printf("%s\n", body)
return nil
}
@@ -270,7 +220,7 @@ func createUser(ctx *cli.Context) error {
func deleteUser(ctx *cli.Context) error {
access := ctx.String("access")
if access == "" {
return fmt.Errorf("invalid input parameter for the user access key")
return fmt.Errorf("invalid input parameter for the new user")
}
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
@@ -285,82 +235,25 @@ func deleteUser(ctx *cli.Context) error {
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode >= 400 {
return parseApiError(body)
}
return nil
}
func updateUser(ctx *cli.Context) error {
access, secret, userId, groupId := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id")
props := auth.MutableProps{}
if ctx.IsSet("secret") {
props.Secret = &secret
}
if ctx.IsSet("user-id") {
props.UserID = &userId
}
if ctx.IsSet("group-id") {
props.GroupID = &groupId
}
propsxml, err := xml.Marshal(props)
if err != nil {
return fmt.Errorf("failed to parse user attributes: %w", err)
}
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsxml))
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256(propsxml)
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode >= 400 {
return parseApiError(body)
}
fmt.Printf("%s\n", body)
return nil
}
@@ -378,34 +271,34 @@ func listUsers(ctx *cli.Context) error {
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return parseApiError(body)
return fmt.Errorf("%s", body)
}
var accs auth.ListUserAccountsResult
if err := xml.Unmarshal(body, &accs); err != nil {
var accs []auth.Account
if err := json.Unmarshal(body, &accs); err != nil {
return err
}
printAcctTable(accs.Accounts)
printAcctTable(accs)
return nil
}
@@ -422,10 +315,10 @@ const (
func printAcctTable(accs []auth.Account) {
w := new(tabwriter.Writer)
w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID")
fmt.Fprintln(w, "-------\t----\t------\t-------")
fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
for _, acc := range accs {
fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID)
fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
}
fmt.Fprintln(w)
w.Flush()
@@ -445,27 +338,25 @@ func changeBucketOwner(ctx *cli.Context) error {
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return parseApiError(body)
}
fmt.Println(string(body))
return nil
}
@@ -495,45 +386,34 @@ func listBuckets(ctx *cli.Context) error {
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return parseApiError(body)
return fmt.Errorf("%s", body)
}
var result s3response.ListBucketsResult
if err := xml.Unmarshal(body, &result); err != nil {
var buckets []s3response.Bucket
if err := json.Unmarshal(body, &buckets); err != nil {
return err
}
printBuckets(result.Buckets)
printBuckets(buckets)
return nil
}
func parseApiError(body []byte) error {
var apiErr smithy.GenericAPIError
err := xml.Unmarshal(body, &apiErr)
if err != nil {
apiErr.Code = "InternalServerError"
apiErr.Message = err.Error()
}
return &apiErr
}

View File

@@ -7,9 +7,7 @@ import (
"path/filepath"
"sync"
"testing"
"time"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/backend/posix"
"github.com/versity/versitygw/tests/integration"
)
@@ -58,9 +56,7 @@ func initPosix(ctx context.Context) {
log.Fatalf("make temp directory: %v", err)
}
be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{
NewDirPerm: 0755,
})
be, err := posix.New(tempdir, posix.PosixOpts{})
if err != nil {
log.Fatalf("init posix: %v", err)
}
@@ -78,9 +74,6 @@ func initPosix(ctx context.Context) {
}
wg.Done()
}()
// wait for server to start
time.Sleep(1 * time.Second)
}
func TestIntegration(t *testing.T) {

View File

@@ -19,17 +19,14 @@ import (
"crypto/tls"
"fmt"
"log"
"net"
"net/http"
_ "net/http/pprof"
"os"
"strings"
"github.com/gofiber/fiber/v2"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3event"
@@ -37,46 +34,33 @@ import (
)
var (
port, admPort string
rootUserAccess string
rootUserSecret string
region string
admCertFile, admKeyFile string
certFile, keyFile string
kafkaURL, kafkaTopic, kafkaKey string
natsURL, natsTopic string
eventWebhookURL string
eventConfigFilePath string
logWebhookURL, accessLog string
adminLogFile string
healthPath string
debug bool
pprof string
quiet bool
readonly bool
iamDir string
ldapURL, ldapBindDN, ldapPassword string
ldapQueryBase, ldapObjClasses string
ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
ldapUserIdAtr, ldapGroupIdAtr string
vaultEndpointURL, vaultSecretStoragePath string
vaultMountPath, vaultRootToken string
vaultRoleId, vaultRoleSecret string
vaultServerCert, vaultClientCert string
vaultClientCertKey string
s3IamAccess, s3IamSecret string
s3IamRegion, s3IamBucket string
s3IamEndpoint string
s3IamSslNoVerify, s3IamDebug bool
iamCacheDisable bool
iamCacheTTL int
iamCachePrune int
metricsService string
statsdServers string
dogstatsServers string
ipaHost, ipaVaultName string
ipaUser, ipaPassword string
ipaInsecure, ipaDebug bool
port, admPort string
rootUserAccess string
rootUserSecret string
region string
admCertFile, admKeyFile string
certFile, keyFile string
kafkaURL, kafkaTopic, kafkaKey string
natsURL, natsTopic string
eventWebhookURL string
eventConfigFilePath string
logWebhookURL string
accessLog string
healthPath string
debug bool
pprof string
quiet bool
iamDir string
ldapURL, ldapBindDN, ldapPassword string
ldapQueryBase, ldapObjClasses string
ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
s3IamAccess, s3IamSecret string
s3IamRegion, s3IamBucket string
s3IamEndpoint string
s3IamSslNoVerify, s3IamDebug bool
iamCacheDisable bool
iamCacheTTL int
iamCachePrune int
)
var (
@@ -98,7 +82,6 @@ func main() {
scoutfsCommand(),
s3Command(),
azureCommand(),
pluginCommand(),
adminCommand(),
testCommand(),
utilsCommand(),
@@ -118,13 +101,10 @@ func main() {
func initApp() *cli.App {
return &cli.App{
Usage: "Versity S3 Gateway",
Description: `The Versity S3 Gateway is an S3 protocol translator that allows an S3 client
to access the supported backend storage as if it was a native S3 service.
VersityGW is an open-source project licensed under the Apache 2.0 License. The
source code is hosted on GitHub at https://github.com/versity/versitygw, and
documentation can be found in the GitHub wiki.`,
Copyright: "Copyright (c) 2023-2024 Versity Software",
Name: "versitygw",
Usage: "Start S3 gateway service with specified backend storage.",
Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
to access the supported backend storage as if it was a native S3 service.`,
Action: func(ctx *cli.Context) error {
return ctx.App.Command("help").Run(ctx)
},
@@ -210,7 +190,6 @@ func initFlags() []cli.Flag {
&cli.BoolFlag{
Name: "debug",
Usage: "enable debug output",
Value: false,
EnvVars: []string{"VGW_DEBUG"},
Destination: &debug,
},
@@ -233,12 +212,6 @@ func initFlags() []cli.Flag {
EnvVars: []string{"LOGFILE", "VGW_ACCESS_LOG"},
Destination: &accessLog,
},
&cli.StringFlag{
Name: "admin-access-log",
Usage: "enable admin server access logging to specified file",
EnvVars: []string{"LOGFILE", "VGW_ADMIN_ACCESS_LOG"},
Destination: &adminLogFile,
},
&cli.StringFlag{
Name: "log-webhook-url",
Usage: "webhook url to send the audit logs",
@@ -348,72 +321,6 @@ func initFlags() []cli.Flag {
EnvVars: []string{"VGW_IAM_LDAP_ROLE_ATR"},
Destination: &ldapRoleAtr,
},
&cli.StringFlag{
Name: "iam-ldap-user-id-atr",
Usage: "ldap server user id attribute name",
EnvVars: []string{"VGW_IAM_LDAP_USER_ID_ATR"},
Destination: &ldapUserIdAtr,
},
&cli.StringFlag{
Name: "iam-ldap-group-id-atr",
Usage: "ldap server user group id attribute name",
EnvVars: []string{"VGW_IAM_LDAP_GROUP_ID_ATR"},
Destination: &ldapGroupIdAtr,
},
&cli.StringFlag{
Name: "iam-vault-endpoint-url",
Usage: "vault server url",
EnvVars: []string{"VGW_IAM_VAULT_ENDPOINT_URL"},
Destination: &vaultEndpointURL,
},
&cli.StringFlag{
Name: "iam-vault-secret-storage-path",
Usage: "vault server secret storage path",
EnvVars: []string{"VGW_IAM_VAULT_SECRET_STORAGE_PATH"},
Destination: &vaultSecretStoragePath,
},
&cli.StringFlag{
Name: "iam-vault-mount-path",
Usage: "vault server mount path",
EnvVars: []string{"VGW_IAM_VAULT_MOUNT_PATH"},
Destination: &vaultMountPath,
},
&cli.StringFlag{
Name: "iam-vault-root-token",
Usage: "vault server root token",
EnvVars: []string{"VGW_IAM_VAULT_ROOT_TOKEN"},
Destination: &vaultRootToken,
},
&cli.StringFlag{
Name: "iam-vault-role-id",
Usage: "vault server user role id",
EnvVars: []string{"VGW_IAM_VAULT_ROLE_ID"},
Destination: &vaultRoleId,
},
&cli.StringFlag{
Name: "iam-vault-role-secret",
Usage: "vault server user role secret",
EnvVars: []string{"VGW_IAM_VAULT_ROLE_SECRET"},
Destination: &vaultRoleSecret,
},
&cli.StringFlag{
Name: "iam-vault-server_cert",
Usage: "vault server TLS certificate",
EnvVars: []string{"VGW_IAM_VAULT_SERVER_CERT"},
Destination: &vaultServerCert,
},
&cli.StringFlag{
Name: "iam-vault-client_cert",
Usage: "vault client TLS certificate",
EnvVars: []string{"VGW_IAM_VAULT_CLIENT_CERT"},
Destination: &vaultClientCert,
},
&cli.StringFlag{
Name: "iam-vault-client_cert_key",
Usage: "vault client TLS certificate key",
EnvVars: []string{"VGW_IAM_VAULT_CLIENT_CERT_KEY"},
Destination: &vaultClientCertKey,
},
&cli.StringFlag{
Name: "s3-iam-access",
Usage: "s3 IAM access key",
@@ -484,69 +391,6 @@ func initFlags() []cli.Flag {
EnvVars: []string{"VGW_HEALTH"},
Destination: &healthPath,
},
&cli.BoolFlag{
Name: "readonly",
Usage: "allow only read operations across all the gateway",
EnvVars: []string{"VGW_READ_ONLY"},
Destination: &readonly,
},
&cli.StringFlag{
Name: "metrics-service-name",
Usage: "service name tag for metrics, hostname if blank",
EnvVars: []string{"VGW_METRICS_SERVICE_NAME"},
Aliases: []string{"msn"},
Destination: &metricsService,
},
&cli.StringFlag{
Name: "metrics-statsd-servers",
Usage: "StatsD server urls comma separated. e.g. 'statsd1.example.com:8125,statsd2.example.com:8125'",
EnvVars: []string{"VGW_METRICS_STATSD_SERVERS"},
Aliases: []string{"mss"},
Destination: &statsdServers,
},
&cli.StringFlag{
Name: "metrics-dogstatsd-servers",
Usage: "DogStatsD server urls comma separated. e.g. '127.0.0.1:8125,dogstats.example.com:8125'",
EnvVars: []string{"VGW_METRICS_DOGSTATS_SERVERS"},
Aliases: []string{"mds"},
Destination: &dogstatsServers,
},
&cli.StringFlag{
Name: "ipa-host",
Usage: "FreeIPA server url e.g. https://ipa.example.test",
EnvVars: []string{"VGW_IPA_HOST"},
Destination: &ipaHost,
},
&cli.StringFlag{
Name: "ipa-vault-name",
Usage: "A name of the user vault containing their secret",
EnvVars: []string{"VGW_IPA_VAULT_NAME"},
Destination: &ipaVaultName,
},
&cli.StringFlag{
Name: "ipa-user",
Usage: "Username used to connect to FreeIPA. Needs permissions to read user vault contents",
EnvVars: []string{"VGW_IPA_USER"},
Destination: &ipaUser,
},
&cli.StringFlag{
Name: "ipa-password",
Usage: "Password of the user used to connect to FreeIPA.",
EnvVars: []string{"VGW_IPA_PASSWORD"},
Destination: &ipaPassword,
},
&cli.BoolFlag{
Name: "ipa-insecure",
Usage: "Verify TLS certificate of FreeIPA server. Default is 'true'.",
EnvVars: []string{"VGW_IPA_INSECURE"},
Destination: &ipaInsecure,
},
&cli.BoolFlag{
Name: "ipa-debug",
Usage: "FreeIPA IAM debug output",
EnvVars: []string{"VGW_IPA_DEBUG"},
Destination: &ipaDebug,
},
}
}
@@ -564,12 +408,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
}
app := fiber.New(fiber.Config{
AppName: "versitygw",
ServerHeader: "VERSITYGW",
StreamRequestBody: true,
DisableKeepalive: true,
Network: fiber.NetworkTCP,
DisableStartupMessage: true,
AppName: "versitygw",
ServerHeader: "VERSITYGW",
StreamRequestBody: true,
DisableKeepalive: true,
})
var opts []s3api.Option
@@ -600,15 +442,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
if healthPath != "" {
opts = append(opts, s3api.WithHealth(healthPath))
}
if readonly {
opts = append(opts, s3api.WithReadOnly())
}
admApp := fiber.New(fiber.Config{
AppName: "versitygw",
ServerHeader: "VERSITYGW",
Network: fiber.NetworkTCP,
DisableStartupMessage: true,
AppName: "versitygw",
ServerHeader: "VERSITYGW",
})
var admOpts []s3api.AdminOpt
@@ -629,70 +466,38 @@ func runGateway(ctx context.Context, be backend.Backend) error {
}
iam, err := auth.New(&auth.Opts{
RootAccount: auth.Account{
Access: rootUserAccess,
Secret: rootUserSecret,
Role: auth.RoleAdmin,
},
Dir: iamDir,
LDAPServerURL: ldapURL,
LDAPBindDN: ldapBindDN,
LDAPPassword: ldapPassword,
LDAPQueryBase: ldapQueryBase,
LDAPObjClasses: ldapObjClasses,
LDAPAccessAtr: ldapAccessAtr,
LDAPSecretAtr: ldapSecAtr,
LDAPRoleAtr: ldapRoleAtr,
LDAPUserIdAtr: ldapUserIdAtr,
LDAPGroupIdAtr: ldapGroupIdAtr,
VaultEndpointURL: vaultEndpointURL,
VaultSecretStoragePath: vaultSecretStoragePath,
VaultMountPath: vaultMountPath,
VaultRootToken: vaultRootToken,
VaultRoleId: vaultRoleId,
VaultRoleSecret: vaultRoleSecret,
VaultServerCert: vaultServerCert,
VaultClientCert: vaultClientCert,
VaultClientCertKey: vaultClientCertKey,
S3Access: s3IamAccess,
S3Secret: s3IamSecret,
S3Region: s3IamRegion,
S3Bucket: s3IamBucket,
S3Endpoint: s3IamEndpoint,
S3DisableSSlVerfiy: s3IamSslNoVerify,
S3Debug: s3IamDebug,
CacheDisable: iamCacheDisable,
CacheTTL: iamCacheTTL,
CachePrune: iamCachePrune,
IpaHost: ipaHost,
IpaVaultName: ipaVaultName,
IpaUser: ipaUser,
IpaPassword: ipaPassword,
IpaInsecure: ipaInsecure,
IpaDebug: ipaDebug,
Dir: iamDir,
LDAPServerURL: ldapURL,
LDAPBindDN: ldapBindDN,
LDAPPassword: ldapPassword,
LDAPQueryBase: ldapQueryBase,
LDAPObjClasses: ldapObjClasses,
LDAPAccessAtr: ldapAccessAtr,
LDAPSecretAtr: ldapSecAtr,
LDAPRoleAtr: ldapRoleAtr,
S3Access: s3IamAccess,
S3Secret: s3IamSecret,
S3Region: s3IamRegion,
S3Bucket: s3IamBucket,
S3Endpoint: s3IamEndpoint,
S3DisableSSlVerfiy: s3IamSslNoVerify,
S3Debug: s3IamDebug,
CacheDisable: iamCacheDisable,
CacheTTL: iamCacheTTL,
CachePrune: iamCachePrune,
})
if err != nil {
return fmt.Errorf("setup iam: %w", err)
}
loggers, err := s3log.InitLogger(&s3log.LogConfig{
LogFile: accessLog,
WebhookURL: logWebhookURL,
AdminLogFile: adminLogFile,
logger, err := s3log.InitLogger(&s3log.LogConfig{
LogFile: accessLog,
WebhookURL: logWebhookURL,
})
if err != nil {
return fmt.Errorf("setup logger: %w", err)
}
metricsManager, err := metrics.NewManager(ctx, metrics.Config{
ServiceName: metricsService,
StatsdServers: statsdServers,
DogStatsdServers: dogstatsServers,
})
if err != nil {
return fmt.Errorf("init metrics manager: %w", err)
}
evSender, err := s3event.InitEventSender(&s3event.EventConfig{
KafkaURL: kafkaURL,
KafkaTopic: kafkaTopic,
@@ -709,16 +514,12 @@ func runGateway(ctx context.Context, be backend.Backend) error {
srv, err := s3api.New(app, be, middlewares.RootUserConfig{
Access: rootUserAccess,
Secret: rootUserSecret,
}, port, region, iam, loggers.S3Logger, loggers.AdminLogger, evSender, metricsManager, opts...)
}, port, region, iam, logger, evSender, opts...)
if err != nil {
return fmt.Errorf("init gateway: %v", err)
}
admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, loggers.AdminLogger, admOpts...)
if !quiet {
printBanner(port, admPort, certFile != "", admCertFile != "")
}
admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
c := make(chan error, 2)
go func() { c <- srv.Serve() }()
@@ -735,17 +536,10 @@ Loop:
case err = <-c:
break Loop
case <-sigHup:
if loggers.S3Logger != nil {
err = loggers.S3Logger.HangUp()
if logger != nil {
err = logger.HangUp()
if err != nil {
err = fmt.Errorf("HUP s3 logger: %w", err)
break Loop
}
}
if loggers.AdminLogger != nil {
err = loggers.AdminLogger.HangUp()
if err != nil {
err = fmt.Errorf("HUP admin logger: %w", err)
err = fmt.Errorf("HUP logger: %w", err)
break Loop
}
}
@@ -763,22 +557,13 @@ Loop:
fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
}
if loggers.S3Logger != nil {
err := loggers.S3Logger.Shutdown()
if logger != nil {
err := logger.Shutdown()
if err != nil {
if saveErr == nil {
saveErr = err
}
fmt.Fprintf(os.Stderr, "shutdown s3 logger: %v\n", err)
}
}
if loggers.AdminLogger != nil {
err := loggers.AdminLogger.Shutdown()
if err != nil {
if saveErr == nil {
saveErr = err
}
fmt.Fprintf(os.Stderr, "shutdown admin logger: %v\n", err)
fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
}
}
@@ -792,183 +577,5 @@ Loop:
}
}
if metricsManager != nil {
metricsManager.Close()
}
return saveErr
}
func printBanner(port, admPort string, ssl, admSsl bool) {
interfaces, err := getMatchingIPs(port)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to match local IP addresses: %v\n", err)
return
}
var admInterfaces []string
if admPort != "" {
admInterfaces, err = getMatchingIPs(admPort)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to match admin port local IP addresses: %v\n", err)
return
}
}
title := "VersityGW"
version := fmt.Sprintf("Version %v, Build %v", Version, Build)
urls := []string{}
hst, prt, err := net.SplitHostPort(port)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
return
}
for _, ip := range interfaces {
url := fmt.Sprintf("http://%s:%s", ip, prt)
if ssl {
url = fmt.Sprintf("https://%s:%s", ip, prt)
}
urls = append(urls, url)
}
if hst == "" {
hst = "0.0.0.0"
}
boundHost := fmt.Sprintf("(bound on host %s and port %s)", hst, prt)
lines := []string{
centerText(title),
centerText(version),
centerText(boundHost),
centerText(""),
}
if len(admInterfaces) > 0 {
lines = append(lines,
leftText("S3 service listening on:"),
)
} else {
lines = append(lines,
leftText("Admin/S3 service listening on:"),
)
}
for _, url := range urls {
lines = append(lines, leftText(" "+url))
}
if len(admInterfaces) > 0 {
lines = append(lines,
centerText(""),
leftText("Admin service listening on:"),
)
_, prt, err := net.SplitHostPort(admPort)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
return
}
for _, ip := range admInterfaces {
url := fmt.Sprintf("http://%s:%s", ip, prt)
if admSsl {
url = fmt.Sprintf("https://%s:%s", ip, prt)
}
lines = append(lines, leftText(" "+url))
}
}
// Print the top border
fmt.Println("┌" + strings.Repeat("─", columnWidth-2) + "┐")
// Print each line
for _, line := range lines {
fmt.Printf("│%-*s│\n", columnWidth-2, line)
}
// Print the bottom border
fmt.Println("└" + strings.Repeat("─", columnWidth-2) + "┘")
}
// getMatchingIPs returns all IP addresses for local system interfaces that
// match the input address specification.
func getMatchingIPs(spec string) ([]string, error) {
// Split the input spec into IP and port
host, _, err := net.SplitHostPort(spec)
if err != nil {
return nil, fmt.Errorf("parse address/port: %v", err)
}
// Handle cases where IP is omitted (e.g., ":1234")
if host == "" {
host = "0.0.0.0"
}
ipaddr, err := net.ResolveIPAddr("ip", host)
if err != nil {
return nil, err
}
parsedInputIP := ipaddr.IP
var result []string
// Get all network interfaces
interfaces, err := net.Interfaces()
if err != nil {
return nil, err
}
for _, iface := range interfaces {
// Get all addresses associated with the interface
addrs, err := iface.Addrs()
if err != nil {
return nil, err
}
for _, addr := range addrs {
// Parse the address to get the IP part
ipAddr, _, err := net.ParseCIDR(addr.String())
if err != nil {
return nil, err
}
if ipAddr.IsLinkLocalUnicast() {
continue
}
if ipAddr.IsInterfaceLocalMulticast() {
continue
}
if ipAddr.IsLinkLocalMulticast() {
continue
}
// Check if the IP matches the input specification
if parsedInputIP.Equal(net.IPv4(0, 0, 0, 0)) || parsedInputIP.Equal(ipAddr) {
result = append(result, ipAddr.String())
}
}
}
return result, nil
}
const columnWidth = 70
func centerText(text string) string {
padding := (columnWidth - 2 - len(text)) / 2
if padding < 0 {
padding = 0
}
return strings.Repeat(" ", padding) + text
}
func leftText(text string) string {
if len(text) > columnWidth-2 {
return text
}
return text + strings.Repeat(" ", columnWidth-2-len(text))
}

View File

@@ -1,64 +0,0 @@
// Copyright 2025 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package main
import (
"fmt"
"github.com/urfave/cli/v2"
vgwplugin "github.com/versity/versitygw/backend/plugin"
)
var (
pluginPath string
pluginConfig string
)
func pluginCommand() *cli.Command {
return &cli.Command{
Name: "plugin",
Usage: "plugin storage backend",
Description: `This tells the gateway to load the backend from a dynamic runtime plugin.`,
Action: runPlugin,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "file",
Usage: "path to plugin shared object file",
Value: "",
Required: true,
EnvVars: []string{"VGW_PLUGIN_FILE"},
Destination: &pluginPath,
Aliases: []string{"f"},
},
&cli.StringFlag{
Name: "config",
Usage: "configuration option for the plugin",
Value: "",
Required: true,
EnvVars: []string{"VGW_PLUGIN_CONFIG"},
Destination: &pluginConfig,
Aliases: []string{"c"},
},
},
}
}
func runPlugin(ctx *cli.Context) error {
be, err := vgwplugin.NewPluginBackend(pluginPath, pluginConfig)
if err != nil {
return fmt.Errorf("init plugin backend: %w", err)
}
return runGateway(ctx.Context, be)
}

View File

@@ -16,21 +16,14 @@ package main
import (
"fmt"
"io/fs"
"math"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/backend/posix"
)
var (
chownuid, chowngid bool
bucketlinks bool
versioningDir string
dirPerms uint
sidecar string
nometa bool
readonly bool
)
func posixCommand() *cli.Command {
@@ -62,36 +55,10 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
Destination: &chowngid,
},
&cli.BoolFlag{
Name: "bucketlinks",
Usage: "allow symlinked directories at bucket level to be treated as buckets",
EnvVars: []string{"VGW_BUCKET_LINKS"},
Destination: &bucketlinks,
},
&cli.StringFlag{
Name: "versioning-dir",
Usage: "the directory path to enable bucket versioning",
EnvVars: []string{"VGW_VERSIONING_DIR"},
Destination: &versioningDir,
},
&cli.UintFlag{
Name: "dir-perms",
Usage: "default directory permissions for new directories",
EnvVars: []string{"VGW_DIR_PERMS"},
Destination: &dirPerms,
DefaultText: "0755",
Value: 0755,
},
&cli.StringFlag{
Name: "sidecar",
Usage: "use provided sidecar directory to store metadata",
EnvVars: []string{"VGW_META_SIDECAR"},
Destination: &sidecar,
},
&cli.BoolFlag{
Name: "nometa",
Usage: "disable metadata storage",
EnvVars: []string{"VGW_META_NONE"},
Destination: &nometa,
Name: "readonly",
Usage: "allow only read operations to backend",
EnvVars: []string{"VGW_READ_ONLY"},
Destination: &readonly,
},
},
}
@@ -102,46 +69,13 @@ func runPosix(ctx *cli.Context) error {
return fmt.Errorf("no directory provided for operation")
}
gwroot := (ctx.Args().Get(0))
if dirPerms > math.MaxUint32 {
return fmt.Errorf("invalid directory permissions: %d", dirPerms)
}
if nometa && sidecar != "" {
return fmt.Errorf("cannot use both nometa and sidecar metadata")
}
opts := posix.PosixOpts{
ChownUID: chownuid,
ChownGID: chowngid,
BucketLinks: bucketlinks,
VersioningDir: versioningDir,
NewDirPerm: fs.FileMode(dirPerms),
}
var ms meta.MetadataStorer
switch {
case sidecar != "":
sc, err := meta.NewSideCar(sidecar)
if err != nil {
return fmt.Errorf("failed to init sidecar metadata: %w", err)
}
ms = sc
opts.SideCarDir = sidecar
case nometa:
ms = meta.NoMeta{}
default:
ms = meta.XattrMeta{}
err := meta.XattrMeta{}.Test(gwroot)
if err != nil {
return fmt.Errorf("xattr check failed: %w", err)
}
}
be, err := posix.New(gwroot, ms, opts)
be, err := posix.New(ctx.Args().Get(0), posix.PosixOpts{
ChownUID: chownuid,
ChownGID: chowngid,
ReadOnly: readonly,
})
if err != nil {
return fmt.Errorf("failed to init posix backend: %w", err)
return fmt.Errorf("init posix: %v", err)
}
return runGateway(ctx.Context, be)

View File

@@ -16,16 +16,17 @@ package main
import (
"fmt"
"io/fs"
"math"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/scoutfs"
)
var (
glacier bool
disableNoArchive bool
glacier bool
// defined in posix.go:
// chownuid, chowngid bool
// readonly bool
)
func scoutfsCommand() *cli.Command {
@@ -67,24 +68,10 @@ move interfaces as well as support for tiered filesystems.`,
Destination: &chowngid,
},
&cli.BoolFlag{
Name: "bucketlinks",
Usage: "allow symlinked directories at bucket level to be treated as buckets",
EnvVars: []string{"VGW_BUCKET_LINKS"},
Destination: &bucketlinks,
},
&cli.UintFlag{
Name: "dir-perms",
Usage: "default directory permissions for new directories",
EnvVars: []string{"VGW_DIR_PERMS"},
Destination: &dirPerms,
DefaultText: "0755",
Value: 0755,
},
&cli.BoolFlag{
Name: "disable-noarchive",
Usage: "disable setting noarchive for multipart part uploads",
EnvVars: []string{"VGW_DISABLE_NOARCHIVE"},
Destination: &disableNoArchive,
Name: "readonly",
Usage: "allow only read operations to backend",
EnvVars: []string{"VGW_READ_ONLY"},
Destination: &readonly,
},
},
}
@@ -95,17 +82,11 @@ func runScoutfs(ctx *cli.Context) error {
return fmt.Errorf("no directory provided for operation")
}
if dirPerms > math.MaxUint32 {
return fmt.Errorf("invalid directory permissions: %d", dirPerms)
}
var opts scoutfs.ScoutfsOpts
opts.GlacierMode = glacier
opts.ChownUID = chownuid
opts.ChownGID = chowngid
opts.BucketLinks = bucketlinks
opts.NewDirPerm = fs.FileMode(dirPerms)
opts.DisableNoArchive = disableNoArchive
opts.ReadOnly = readonly
be, err := scoutfs.New(ctx.Args().Get(0), opts)
if err != nil {

View File

@@ -22,23 +22,20 @@ import (
)
var (
awsID string
awsSecret string
endpoint string
prefix string
dstBucket string
partSize int64
objSize int64
concurrency int
files int
totalReqs int
upload bool
download bool
pathStyle bool
checksumDisable bool
versioningEnabled bool
azureTests bool
tlsStatus bool
awsID string
awsSecret string
endpoint string
prefix string
dstBucket string
partSize int64
objSize int64
concurrency int
files int
totalReqs int
upload bool
download bool
pathStyle bool
checksumDisable bool
)
func testCommand() *cli.Command {
@@ -80,12 +77,6 @@ func initTestFlags() []cli.Flag {
Aliases: []string{"d"},
Destination: &debug,
},
&cli.BoolFlag{
Name: "allow-insecure",
Usage: "skip tls verification",
Aliases: []string{"ai"},
Destination: &tlsStatus,
},
}
}
@@ -96,33 +87,11 @@ func initTestCommands() []*cli.Command {
Usage: "Tests the full flow of gateway.",
Description: `Runs all the available tests to test the full flow of the gateway.`,
Action: getAction(integration.TestFullFlow),
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "versioning-enabled",
Usage: "Test the bucket object versioning, if the versioning is enabled",
Destination: &versioningEnabled,
Aliases: []string{"vs"},
},
&cli.BoolFlag{
Name: "azure-test-mode",
Usage: "Skips tests that are not supported by Azure",
Destination: &azureTests,
Aliases: []string{"azure"},
},
},
},
{
Name: "posix",
Usage: "Tests posix specific features",
Action: getAction(integration.TestPosix),
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "versioning-enabled",
Usage: "Test posix when versioning is enabled",
Destination: &versioningEnabled,
Aliases: []string{"vs"},
},
},
},
{
Name: "iam",
@@ -218,7 +187,6 @@ func initTestCommands() []*cli.Command {
integration.WithEndpoint(endpoint),
integration.WithConcurrency(concurrency),
integration.WithPartSize(partSize),
integration.WithTLSStatus(tlsStatus),
}
if debug {
opts = append(opts, integration.WithDebug())
@@ -279,7 +247,6 @@ func initTestCommands() []*cli.Command {
integration.WithRegion(region),
integration.WithEndpoint(endpoint),
integration.WithConcurrency(concurrency),
integration.WithTLSStatus(tlsStatus),
}
if debug {
opts = append(opts, integration.WithDebug())
@@ -305,17 +272,10 @@ func getAction(tf testFunc) func(*cli.Context) error {
integration.WithSecret(awsSecret),
integration.WithRegion(region),
integration.WithEndpoint(endpoint),
integration.WithTLSStatus(tlsStatus),
}
if debug {
opts = append(opts, integration.WithDebug())
}
if versioningEnabled {
opts = append(opts, integration.WithVersioningEnabled())
}
if azureTests {
opts = append(opts, integration.WithAzureMode())
}
s := integration.NewS3Conf(opts...)
tf(s)
@@ -343,27 +303,15 @@ func extractIntTests() (commands []*cli.Command) {
integration.WithSecret(awsSecret),
integration.WithRegion(region),
integration.WithEndpoint(endpoint),
integration.WithTLSStatus(tlsStatus),
}
if debug {
opts = append(opts, integration.WithDebug())
}
if versioningEnabled {
opts = append(opts, integration.WithVersioningEnabled())
}
s := integration.NewS3Conf(opts...)
err := testFunc(s)
return err
},
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "versioning-enabled",
Usage: "Test the bucket object versioning, if the versioning is enabled",
Destination: &versioningEnabled,
Aliases: []string{"vs"},
},
},
})
}
return

View File

@@ -54,24 +54,22 @@ func generateEventFiltersConfig(ctx *cli.Context) error {
}
config := s3event.EventFilter{
s3event.EventObjectCreated: true,
s3event.EventObjectCreatedPut: true,
s3event.EventObjectCreatedPost: true,
s3event.EventObjectCreatedCopy: true,
s3event.EventCompleteMultipartUpload: true,
s3event.EventObjectRemoved: true,
s3event.EventObjectRemovedDelete: true,
s3event.EventObjectRemovedDeleteObjects: true,
s3event.EventObjectTagging: true,
s3event.EventObjectTaggingPut: true,
s3event.EventObjectTaggingDelete: true,
s3event.EventObjectAclPut: true,
s3event.EventObjectRestore: true,
s3event.EventObjectRestorePost: true,
s3event.EventObjectRestoreCompleted: true,
s3event.EventObjectCreated: true,
s3event.EventObjectCreatedPut: true,
s3event.EventObjectCreatedPost: true,
s3event.EventObjectCreatedCopy: true,
s3event.EventCompleteMultipartUpload: true,
s3event.EventObjectDeleted: true,
s3event.EventObjectTagging: true,
s3event.EventObjectTaggingPut: true,
s3event.EventObjectTaggingDelete: true,
s3event.EventObjectAclPut: true,
s3event.EventObjectRestore: true,
s3event.EventObjectRestorePost: true,
s3event.EventObjectRestoreCompleted: true,
}
configBytes, err := json.MarshalIndent(config, "", " ")
configBytes, err := json.Marshal(config)
if err != nil {
return fmt.Errorf("parse event config: %w", err)
}

View File

@@ -1,8 +1,9 @@
version: "3"
services:
posix:
build:
context: .
dockerfile: tests/Dockerfile.dev
dockerfile: ./Dockerfile.dev
args:
- IAM_DIR=${IAM_DIR}
- SETUP_DIR=${SETUP_DIR}
@@ -14,7 +15,7 @@ services:
proxy:
build:
context: .
dockerfile: tests/Dockerfile.dev
dockerfile: ./Dockerfile.dev
volumes:
- ./:/app
ports:
@@ -30,13 +31,13 @@ services:
hostname: azurite
command: "azurite --oauth basic --cert /tests/certs/azurite.pem --key /tests/certs/azurite-key.pem --blobHost 0.0.0.0"
volumes:
- ./tests/certs:/tests/certs
- ./certs:/certs
azuritegw:
build:
context: .
dockerfile: tests/Dockerfile.dev
dockerfile: ./Dockerfile.dev
volumes:
- ./:/app
ports:
- 7070:7070
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -buildvcs=false -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]

View File

@@ -1,19 +0,0 @@
# Versity Gateway Dashboard
This project is a dashboard that visualizes data in the six metrics emitted by the Versity Gateway, displayed in Grafana.
The Versity Gateway emits metrics in the statsd format. We used Telegraf as the bridge from statsd to influxdb.
This implementation uses the influxql query language.
## Usage
From the root of this repository, run `docker compose -f docker-compose-metrics.yml up` to start the stack.
To shut it down, run `docker compose -f docker-compose-metrics.yml down -v`.
The Grafana database is explicitly not destroyed when shutting down containers. The influxdb one, however, is.
The dashbaord is automatically provisioned at container bring up and is visible at http://localhost:3000 with username: `admin` and password: `admin`.
To use the gateway and generate metrics, `source metrics-exploration/aws_env_setup.sh` and start using your aws cli as usual.

View File

@@ -1,4 +0,0 @@
export AWS_SECRET_ACCESS_KEY=password
export AWS_ACCESS_KEY_ID=user
export AWS_ENDPOINT_URL=http://127.0.0.1:7070
export AWS_REGION=us-east-1

View File

@@ -1,64 +0,0 @@
services:
telegraf:
image: telegraf
container_name: telegraf
restart: always
volumes:
- ./metrics-exploration/telegraf.conf:/etc/telegraf/telegraf.conf:ro
depends_on:
- influxdb
links:
- influxdb
ports:
- '8125:8125/udp'
influxdb:
image: influxdb
container_name: influxdb
restart: always
environment:
- DOCKER_INFLUXDB_INIT_MODE=setup
- DOCKER_INFLUXDB_INIT_USERNAME=admin
- DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
- DOCKER_INFLUXDB_INIT_ORG=myorg
- DOCKER_INFLUXDB_INIT_BUCKET=metrics
- DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
ports:
- '8086:8086'
volumes:
- influxdb_data:/var/lib/influxdb
grafana:
image: grafana/grafana
container_name: grafana-server
restart: always
depends_on:
- influxdb
environment:
- GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_INSTALL_PLUGINS=
links:
- influxdb
ports:
- '3000:3000'
volumes:
- ./metrics-exploration/grafana_data:/var/lib/grafana
- ./metrics-exploration/provisioning:/etc/grafana/provisioning
versitygw:
image: versity/versitygw:latest
container_name: versitygw
ports:
- "7070:7070"
environment:
- ROOT_ACCESS_KEY=user
- ROOT_SECRET_KEY=password
- VGW_METRICS_STATSD_SERVERS=telegraf:8125
depends_on:
- telegraf
command: >
posix /tmp/vgw
volumes:
influxdb_data: {}

View File

@@ -1,64 +0,0 @@
services:
telegraf:
image: telegraf
container_name: telegraf
restart: always
volumes:
- ./telegraf.conf:/etc/telegraf/telegraf.conf:ro
depends_on:
- influxdb
links:
- influxdb
ports:
- '8125:8125/udp'
influxdb:
image: influxdb
container_name: influxdb
restart: always
environment:
- DOCKER_INFLUXDB_INIT_MODE=setup
- DOCKER_INFLUXDB_INIT_USERNAME=admin
- DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
- DOCKER_INFLUXDB_INIT_ORG=myorg
- DOCKER_INFLUXDB_INIT_BUCKET=metrics
- DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
ports:
- '8086:8086'
volumes:
- influxdb_data:/var/lib/influxdb
grafana:
image: grafana/grafana
container_name: grafana-server
restart: always
depends_on:
- influxdb
environment:
- GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_INSTALL_PLUGINS=
links:
- influxdb
ports:
- '3000:3000'
volumes:
- ./grafana_data:/var/lib/grafana
- ./provisioning:/etc/grafana/provisioning
versitygw:
image: versity/versitygw:latest
container_name: versitygw
ports:
- "7070:7070"
environment:
- ROOT_ACCESS_KEY=user
- ROOT_SECRET_KEY=password
- VGW_METRICS_STATSD_SERVERS=telegraf:8125
depends_on:
- telegraf
command: >
posix /tmp/vgw
volumes:
influxdb_data: {}

View File

@@ -1,25 +0,0 @@
apiVersion: 1
providers:
# <string> an unique provider name. Required
- name: 'influxql'
# <int> Org id. Default to 1
orgId: 1
# <string> name of the dashboard folder.
folder: 'influxql'
# <string> folder UID. will be automatically generated if not specified
folderUid: ''
# <string> provider type. Default to 'file'
type: file
# <bool> disable dashboard deletion
disableDeletion: false
# <int> how often Grafana will scan for changed dashboards
updateIntervalSeconds: 10
# <bool> allow updating provisioned dashboards from the UI
allowUiUpdates: true
options:
# <string, required> path to dashboard files on disk. Required when using the 'file' type
path: /etc/grafana/provisioning/dashboards/influxql
# <bool> use folder names from filesystem to create folders in Grafana
foldersFromFilesStructure: true

View File

@@ -1,13 +0,0 @@
apiVersion: 1
datasources:
- name: influxdb
type: influxdb
isDefault: true
access: proxy
url: http://influxdb:8086
jsonData:
dbName: 'metrics'
httpHeaderName1: 'Authorization'
secureJsonData:
httpHeaderValue1: 'Token my-super-secret-auth-token'

View File

@@ -1,34 +0,0 @@
[global_tags]
[agent]
debug = true
quiet = false
interval = "60s"
round_interval = true
metric_batch_size = 1000
metric_buffer_limit = 10000
collection_jitter = "0s"
flush_interval = "10s"
flush_jitter = "0s"
precision = ""
hostname = "versitygw"
omit_hostname = false
[[outputs.file]]
files = ["stdout"]
[[outputs.influxdb_v2]]
urls = ["http://influxdb:8086"]
timeout = "5s"
token = "my-super-secret-auth-token"
organization = "myorg"
bucket = "metrics"
[[inputs.statsd]]
protocol = "udp4"
service_address = ":8125"
percentiles = [90]
metric_separator = "_"
datadog_extensions = false
allowed_pending_messages = 10000
percentile_limit = 1000

View File

@@ -1,6 +0,0 @@
#!/usr/bin/env bash
. ./aws_env_setup.sh
aws s3 mb s3://test
aws s3 cp docker-compose.yml s3://test/test.yaml

View File

@@ -153,14 +153,6 @@ ROOT_SECRET_ACCESS_KEY=
# specified, all configured bucket events will be sent to the webhook.
#VGW_EVENT_WEBHOOK_URL=
# Bucket events can be filtered for any of the above event types. The
# VGW_EVENT_FILTER option specifies a config file that contains the event
# filter rules. The event filter rules are used to determine which events are
# sent to the configured event services. Run:
# versitygw utils gen-event-filter-config --path .
# to generate a default rules file "event_config.json" in the current directory.
#VGW_EVENT_FILTER=
#######################
# Debug / Diagnostics #
#######################
@@ -190,40 +182,7 @@ ROOT_SECRET_ACCESS_KEY=
# as a dedicated IAM service.
#VGW_IAM_DIR=
# The Vault options will enable the Vault IAM service with accounts stored in
# the HashiCorp Vault service. The Vault URL is the address and port of the
# Vault server with the format <IP/host>:<port>. A root taken can be used for
# testing, but it is recommended to use the role based authentication in
# production. The Vault server certificate, client certificate, and client
# certificate key are optional, and will default to not verifying the server
# certificate and not using client certificates. The Vault server certificate
# is used to verify the Vault server, and the client certificate and key are
# used to authenticate the gateway to the Vault server. See wiki documentation
# for an example of using Vault in dev mode with the gateway.
#VGW_IAM_VAULT_ENDPOINT_URL=
#VGW_IAM_VAULT_SECRET_STORAGE_PATH=
#VGW_IAM_VAULT_MOUNT_PATH=
#VGW_IAM_VAULT_ROOT_TOKEN=
#VGW_IAM_VAULT_ROLE_ID=
#VGW_IAM_VAULT_ROLE_SECRET=
#VGW_IAM_VAULT_SERVER_CERT=
#VGW_IAM_VAULT_CLIENT_CERT=
#VGW_IAM_VAULT_CLIENT_CERT_KEY=
# The VGW_S3 IAM service is similar to the internal IAM service, but instead
# stores the account information JSON encoded in an S3 object. This should use
# a bucket that is not accessible to general users when using s3 backend to
# prevent access to account credentials. This IAM service is added for
# convenience, but is not considered as secure or scalable as a dedicated IAM
# service.
#VGW_S3_IAM_ACCESS_KEY=
#VGW_S3_IAM_SECRET_KEY=
#VGW_S3_IAM_REGION=
#VGW_S3_IAM_ENDPOINT=
#VGW_S3_IAM_BUCKET=
#VGW_S3_IAM_NO_VERIFY=
# The LDAP options will enable the LDAP IAM service with accounts stored in an
# The ldap options will enable the LDAP IAM service with accounts stored in an
# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
# secret credentials and role respectively. The other options are used to
@@ -237,8 +196,19 @@ ROOT_SECRET_ACCESS_KEY=
#VGW_IAM_LDAP_ACCESS_ATR=
#VGW_IAM_LDAP_SECRET_ATR=
#VGW_IAM_LDAP_ROLE_ATR=
#VGW_IAM_LDAP_USER_ID_ATR=
#VGW_IAM_LDAP_GROUP_ID_ATR=
# The VGW_S3 IAM service is similar to the internal IAM service, but instead
# stores the account information JSON encoded in an S3 object. This should use
# a bucket that is not accessible to general users when using s3 backend to
# prevent access to account credentials. This IAM service is added for
# convenience, but is not considered as secure or scalable as a dedicated IAM
# service.
#VGW_S3_IAM_ACCESS_KEY=
#VGW_S3_IAM_SECRET_KEY=
#VGW_S3_IAM_REGION=
#VGW_S3_IAM_ENDPOINT=
#VGW_S3_IAM_BUCKET=
#VGW_S3_IAM_NO_VERIFY=
###############
# IAM caching #
@@ -258,29 +228,6 @@ ROOT_SECRET_ACCESS_KEY=
#VGW_IAM_CACHE_TTL=120
#VGW_IAM_CACHE_PRUNE=3600
###########
# Metrics #
###########
# The metrics service name is a tag that is added to all metrics to help
# identify the source of the metrics. This is especially useful when multiple
# gateways are running. The default is the hostname of the system.
#VGW_METRICS_SERVICE_NAME=$HOSTNAME
# The metrics service will send metrics to the configured statsd servers. The
# servers are specified as a comma separated list of host:port pairs. The
# default is to not send metrics to any statsd servers. The gateway uses
# InfluxDB flavor of statsd metrics tags for the StatsD metrics type.
#VGW_METRICS_STATSD_SERVERS=
# The metrics service will send metrics to the configured dogstatsd servers.
# The servers are specified as a comma separated list of host:port pairs. The
# default is to not send metrics to any dogstatsd servers. Generally
# DataDog recommends installing a local agent to collect metrics and forward
# them to the DataDog service. In this case the option value would be the
# local agent address: 127.0.0.1:8125.
#VGW_METRICS_DOGSTATS_SERVERS=
######################################
# VersityGW Backend Specific Options #
######################################
@@ -307,15 +254,11 @@ ROOT_SECRET_ACCESS_KEY=
#VGW_CHOWN_UID=false
#VGW_CHOWN_GID=false
# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
# to directories at the top level gateway directory as buckets.
#VGW_BUCKET_LINKS=false
# The default permissions mode when creating new directories is 0755. Use
# VGW_DIR_PERMS option to set a different mode for any new directory that the
# gateway creates. This applies to buckets created through the gateway as well
# as any parent directories automatically created with object uploads.
#VGW_DIR_PERMS=0755
# The VGW_READ_ONLY option will disable all write operations to the backend
# filesystem. This is useful for creating a read-only gateway for clients.
# This will prevent any PUT, POST, DELETE, and multipart upload operations
# for objects and buckets as well as preventing updating metadata for objects.
#VGW_READ_ONLY=false
###########
# scoutfs #
@@ -348,22 +291,11 @@ ROOT_SECRET_ACCESS_KEY=
#VGW_CHOWN_UID=false
#VGW_CHOWN_GID=false
# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
# to directories at the top level gateway directory as buckets.
#VGW_BUCKET_LINKS=false
# The default permissions mode when creating new directories is 0755. Use
# VGW_DIR_PERMS option to set a different mode for any new directory that the
# gateway creates. This applies to buckets created through the gateway as well
# as any parent directories automatically created with object uploads.
#VGW_DIR_PERMS=0755
# The default behavior of the gateway is to automatically set the noarchive
# flag on the multipart upload parts while the multipart upload is in progress.
# This is to prevent the parts from being archived since they are temporary
# and will be deleted after the multipart upload is completed or aborted. The
# VGW_DISABLE_NOARCHIVE option can be set to true to disable this behavior.
#VGW_DISABLE_NOARCHIVE=false
# The VGW_READ_ONLY option will disable all write operations to the backend
# filesystem. This is useful for creating a read-only gateway for clients.
# This will prevent any PUT, POST, DELETE, and multipart upload operations
# for objects and buckets as well as preventing updating metadata for objects.
#VGW_READ_ONLY=false
######
# s3 #

109
go.mod
View File

@@ -1,83 +1,70 @@
module github.com/versity/versitygw
go 1.23.0
toolchain go1.24.1
go 1.21
require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0
github.com/DataDog/datadog-go/v5 v5.6.0
github.com/aws/aws-sdk-go-v2 v1.36.3
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1
github.com/aws/smithy-go v1.22.3
github.com/go-ldap/ldap/v3 v3.4.10
github.com/gofiber/fiber/v2 v2.52.6
github.com/google/go-cmp v0.7.0
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
github.com/aws/aws-sdk-go-v2 v1.26.1
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
github.com/aws/smithy-go v1.20.2
github.com/go-ldap/ldap/v3 v3.4.6
github.com/gofiber/fiber/v2 v2.52.4
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
github.com/hashicorp/vault-client-go v0.4.3
github.com/nats-io/nats.go v1.41.0
github.com/oklog/ulid/v2 v2.1.0
github.com/pkg/xattr v0.4.10
github.com/nats-io/nats.go v1.34.1
github.com/pkg/xattr v0.4.9
github.com/segmentio/kafka-go v0.4.47
github.com/smira/go-statsd v1.3.4
github.com/urfave/cli/v2 v2.27.6
github.com/valyala/fasthttp v1.60.0
github.com/urfave/cli/v2 v2.27.1
github.com/valyala/fasthttp v1.52.0
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
golang.org/x/sync v0.13.0
golang.org/x/sys v0.32.0
golang.org/x/sys v0.19.0
)
require (
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/nats-io/nkeys v0.4.10 // indirect
github.com/nats-io/nkeys v0.4.7 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
golang.org/x/crypto v0.37.0 // indirect
golang.org/x/net v0.38.0 // indirect
golang.org/x/text v0.24.0 // indirect
golang.org/x/time v0.11.0 // indirect
golang.org/x/crypto v0.22.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/text v0.14.0 // indirect
)
require (
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
github.com/aws/aws-sdk-go-v2/config v1.29.13
github.com/aws/aws-sdk-go-v2/credentials v1.17.66
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
github.com/aws/aws-sdk-go-v2/config v1.27.11
github.com/aws/aws-sdk-go-v2/credentials v1.17.11
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
)

318
go.sum
View File

@@ -1,187 +1,131 @@
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 h1:F0gBpfdPLGsw+nsgk6aqqkZS1jiixa5WwFe3fk/T3Ys=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2/go.mod h1:SqINnQ9lVVdRlyC8cd1lCI0SdX4n2paeABd2K8ggfnE=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0 h1:Bg8m3nq/X1DeePkAbCfb6ml6F3F0IunEhE8TMh+lY48=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.0/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw=
github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
github.com/aws/aws-sdk-go-v2/config v1.29.13 h1:RgdPqWoE8nPpIekpVpDJsBckbqT4Liiaq9f35pbTh1Y=
github.com/aws/aws-sdk-go-v2/config v1.29.13/go.mod h1:NI28qs/IOUIRhsR7GQ/JdexoqRN9tDxkIrYZq0SOF44=
github.com/aws/aws-sdk-go-v2/credentials v1.17.66 h1:aKpEKaTy6n4CEJeYI1MNj97oSDLi4xro3UzQfwf5RWE=
github.com/aws/aws-sdk-go-v2/credentials v1.17.66/go.mod h1:xQ5SusDmHb/fy55wU0QqTy0yNfLqxzec59YcsRZB+rI=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71 h1:s43gLuY+zGmtpx+KybfFP4IckopmTfDOPdlf/L++N5I=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71/go.mod h1:KH6wWmY3O3c/jVAjHk0MGzVAFDxkOSt42Eoe4ZO4ge0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 h1:lguz0bmOoGzozP9XfRJR1QIayEYo+2vP/No3OfLF0pU=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1 h1:2Ku1xwAohSSXHR1tpAnyVDSQSxoDMA+/NZBytW+f4qg=
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc=
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8=
github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 h1:xz7WvTMfSStb9Y8NpCT82FXLNC3QasqBfuAFHY4Pk5g=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.18/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k=
github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-ldap/ldap/v3 v3.4.10 h1:ot/iwPOhfpNVgB1o+AVXljizWZ9JTp7YF5oeyONmcJU=
github.com/go-ldap/ldap/v3 v3.4.10/go.mod h1:JXh4Uxgi40P6E9rdsYqpUtbW46D9UTjJ9QSwGRznplY=
github.com/gofiber/fiber/v2 v2.52.6 h1:Rfp+ILPiYSvvVuIPvxrBns+HJp8qGLDnLJawAu27XVI=
github.com/gofiber/fiber/v2 v2.52.6/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY=
github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg=
github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=
github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8=
github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/nats-io/nats.go v1.41.0 h1:PzxEva7fflkd+n87OtQTXqCTyLfIIMFJBpyccHLE2Ko=
github.com/nats-io/nats.go v1.41.0/go.mod h1:wV73x0FSI/orHPSYoyMeJB+KajMDoWyXmFaRrrYaaTo=
github.com/nats-io/nkeys v0.4.10 h1:glmRrpCmYLHByYcePvnTBEAwawwapjCPMjy2huw20wc=
github.com/nats-io/nkeys v0.4.10/go.mod h1:OjRrnIKnWBFl+s4YK5ChQfvHP2fxqZexrKJoVVyWB3U=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c=
github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g=
github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.60.0 h1:kBRYS0lOhVJ6V+bYN8PqAHELKHtXqwq9zNMLKx1MBsw=
github.com/valyala/fasthttp v1.60.0/go.mod h1:iY4kDgV3Gc6EqhRZ8icqcmlG6bqhcDXfuHgTO4FXCvc=
github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -190,87 +134,49 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -278,25 +184,17 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -1,284 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package metrics
type Action struct {
Name string
Service string
}
var (
ActionMap map[string]Action
)
var (
ActionUndetected = "ActionUnDetected"
ActionAbortMultipartUpload = "s3_AbortMultipartUpload"
ActionCompleteMultipartUpload = "s3_CompleteMultipartUpload"
ActionCopyObject = "s3_CopyObject"
ActionCreateBucket = "s3_CreateBucket"
ActionCreateMultipartUpload = "s3_CreateMultipartUpload"
ActionDeleteBucket = "s3_DeleteBucket"
ActionDeleteBucketPolicy = "s3_DeleteBucketPolicy"
ActionDeleteBucketTagging = "s3_DeleteBucketTagging"
ActionDeleteObject = "s3_DeleteObject"
ActionDeleteObjectTagging = "s3_DeleteObjectTagging"
ActionDeleteObjects = "s3_DeleteObjects"
ActionGetBucketAcl = "s3_GetBucketAcl"
ActionGetBucketPolicy = "s3_GetBucketPolicy"
ActionGetBucketTagging = "s3_GetBucketTagging"
ActionGetBucketVersioning = "s3_GetBucketVersioning"
ActionGetObject = "s3_GetObject"
ActionGetObjectAcl = "s3_GetObjectAcl"
ActionGetObjectAttributes = "s3_GetObjectAttributes"
ActionGetObjectLegalHold = "s3_GetObjectLegalHold"
ActionGetObjectLockConfiguration = "s3_GetObjectLockConfiguration"
ActionGetObjectRetention = "s3_GetObjectRetention"
ActionGetObjectTagging = "s3_GetObjectTagging"
ActionHeadBucket = "s3_HeadBucket"
ActionHeadObject = "s3_HeadObject"
ActionListAllMyBuckets = "s3_ListAllMyBuckets"
ActionListMultipartUploads = "s3_ListMultipartUploads"
ActionListObjectVersions = "s3_ListObjectVersions"
ActionListObjects = "s3_ListObjects"
ActionListObjectsV2 = "s3_ListObjectsV2"
ActionListParts = "s3_ListParts"
ActionPutBucketAcl = "s3_PutBucketAcl"
ActionPutBucketPolicy = "s3_PutBucketPolicy"
ActionPutBucketTagging = "s3_PutBucketTagging"
ActionPutBucketVersioning = "s3_PutBucketVersioning"
ActionPutObject = "s3_PutObject"
ActionPutObjectAcl = "s3_PutObjectAcl"
ActionPutObjectLegalHold = "s3_PutObjectLegalHold"
ActionPutObjectLockConfiguration = "s3_PutObjectLockConfiguration"
ActionPutObjectRetention = "s3_PutObjectRetention"
ActionPutObjectTagging = "s3_PutObjectTagging"
ActionRestoreObject = "s3_RestoreObject"
ActionSelectObjectContent = "s3_SelectObjectContent"
ActionUploadPart = "s3_UploadPart"
ActionUploadPartCopy = "s3_UploadPartCopy"
ActionPutBucketOwnershipControls = "s3_PutBucketOwnershipControls"
ActionGetBucketOwnershipControls = "s3_GetBucketOwnershipControls"
ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
ActionPutBucketCors = "s3_PutBucketCors"
ActionGetBucketCors = "s3_GetBucketCors"
ActionDeleteBucketCors = "s3_DeleteBucketCors"
// Admin actions
ActionAdminCreateUser = "admin_CreateUser"
ActionAdminUpdateUser = "admin_UpdateUser"
ActionAdminDeleteUser = "admin_DeleteUser"
ActionAdminChangeBucketOwner = "admin_ChangeBucketOwner"
ActionAdminListUsers = "admin_ListUsers"
ActionAdminListBuckets = "admin_ListBuckets"
)
func init() {
ActionMap = make(map[string]Action)
ActionMap[ActionUndetected] = Action{
Name: "ActionUnDetected",
Service: "unknown",
}
ActionMap[ActionAbortMultipartUpload] = Action{
Name: "AbortMultipartUpload",
Service: "s3",
}
ActionMap[ActionCompleteMultipartUpload] = Action{
Name: "CompleteMultipartUpload",
Service: "s3",
}
ActionMap[ActionCopyObject] = Action{
Name: "CopyObject",
Service: "s3",
}
ActionMap[ActionCreateBucket] = Action{
Name: "CreateBucket",
Service: "s3",
}
ActionMap[ActionCreateMultipartUpload] = Action{
Name: "CreateMultipartUpload",
Service: "s3",
}
ActionMap[ActionDeleteBucket] = Action{
Name: "DeleteBucket",
Service: "s3",
}
ActionMap[ActionDeleteBucketPolicy] = Action{
Name: "DeleteBucketPolicy",
Service: "s3",
}
ActionMap[ActionDeleteBucketTagging] = Action{
Name: "DeleteBucketTagging",
Service: "s3",
}
ActionMap[ActionDeleteObject] = Action{
Name: "DeleteObject",
Service: "s3",
}
ActionMap[ActionDeleteObjectTagging] = Action{
Name: "DeleteObjectTagging",
Service: "s3",
}
ActionMap[ActionDeleteObjects] = Action{
Name: "DeleteObjects",
Service: "s3",
}
ActionMap[ActionGetBucketAcl] = Action{
Name: "GetBucketAcl",
Service: "s3",
}
ActionMap[ActionGetBucketPolicy] = Action{
Name: "GetBucketPolicy",
Service: "s3",
}
ActionMap[ActionGetBucketTagging] = Action{
Name: "GetBucketTagging",
Service: "s3",
}
ActionMap[ActionGetBucketVersioning] = Action{
Name: "GetBucketVersioning",
Service: "s3",
}
ActionMap[ActionGetObject] = Action{
Name: "GetObject",
Service: "s3",
}
ActionMap[ActionGetObjectAcl] = Action{
Name: "GetObjectAcl",
Service: "s3",
}
ActionMap[ActionGetObjectAttributes] = Action{
Name: "GetObjectAttributes",
Service: "s3",
}
ActionMap[ActionGetObjectLegalHold] = Action{
Name: "GetObjectLegalHold",
Service: "s3",
}
ActionMap[ActionGetObjectLockConfiguration] = Action{
Name: "GetObjectLockConfiguration",
Service: "s3",
}
ActionMap[ActionGetObjectRetention] = Action{
Name: "GetObjectRetention",
Service: "s3",
}
ActionMap[ActionGetObjectTagging] = Action{
Name: "GetObjectTagging",
Service: "s3",
}
ActionMap[ActionHeadBucket] = Action{
Name: "HeadBucket",
Service: "s3",
}
ActionMap[ActionHeadObject] = Action{
Name: "HeadObject",
Service: "s3",
}
ActionMap[ActionListAllMyBuckets] = Action{
Name: "ListAllMyBuckets",
Service: "s3",
}
ActionMap[ActionListMultipartUploads] = Action{
Name: "ListMultipartUploads",
Service: "s3",
}
ActionMap[ActionListObjectVersions] = Action{
Name: "ListObjectVersions",
Service: "s3",
}
ActionMap[ActionListObjects] = Action{
Name: "ListObjects",
Service: "s3",
}
ActionMap[ActionListObjectsV2] = Action{
Name: "ListObjectsV2",
Service: "s3",
}
ActionMap[ActionListParts] = Action{
Name: "ListParts",
Service: "s3",
}
ActionMap[ActionPutBucketAcl] = Action{
Name: "PutBucketAcl",
Service: "s3",
}
ActionMap[ActionPutBucketPolicy] = Action{
Name: "PutBucketPolicy",
Service: "s3",
}
ActionMap[ActionPutBucketTagging] = Action{
Name: "PutBucketTagging",
Service: "s3",
}
ActionMap[ActionPutBucketVersioning] = Action{
Name: "PutBucketVersioning",
Service: "s3",
}
ActionMap[ActionPutObject] = Action{
Name: "PutObject",
Service: "s3",
}
ActionMap[ActionPutObjectAcl] = Action{
Name: "PutObjectAcl",
Service: "s3",
}
ActionMap[ActionPutObjectLegalHold] = Action{
Name: "PutObjectLegalHold",
Service: "s3",
}
ActionMap[ActionPutObjectLockConfiguration] = Action{
Name: "PutObjectLockConfiguration",
Service: "s3",
}
ActionMap[ActionPutObjectRetention] = Action{
Name: "PutObjectRetention",
Service: "s3",
}
ActionMap[ActionPutObjectTagging] = Action{
Name: "PutObjectTagging",
Service: "s3",
}
ActionMap[ActionRestoreObject] = Action{
Name: "RestoreObject",
Service: "s3",
}
ActionMap[ActionSelectObjectContent] = Action{
Name: "SelectObjectContent",
Service: "s3",
}
ActionMap[ActionUploadPart] = Action{
Name: "UploadPart",
Service: "s3",
}
ActionMap[ActionUploadPartCopy] = Action{
Name: "UploadPartCopy",
Service: "s3",
}
ActionMap[ActionPutBucketCors] = Action{
Name: "PutBucketCors",
Service: "s3",
}
ActionMap[ActionGetBucketCors] = Action{
Name: "GetBucketCors",
Service: "s3",
}
ActionMap[ActionDeleteBucketCors] = Action{
Name: "DeleteBucketCors",
Service: "s3",
}
}

View File

@@ -1,65 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package metrics
import (
"fmt"
dogstats "github.com/DataDog/datadog-go/v5/statsd"
)
// vgwDogStatsd metrics type
type vgwDogStatsd struct {
c *dogstats.Client
}
var (
rateSampleAlways = 1.0
)
// newDogStatsd takes a server address and returns a statsd merics
func newDogStatsd(server string, service string) (*vgwDogStatsd, error) {
c, err := dogstats.New(server,
dogstats.WithMaxMessagesPerPayload(1000),
dogstats.WithNamespace("versitygw"),
dogstats.WithTags([]string{
"service:" + service,
}))
if err != nil {
return nil, err
}
return &vgwDogStatsd{c: c}, nil
}
// Close closes statsd connections
func (s *vgwDogStatsd) Close() {
s.c.Close()
}
func (t Tag) ddString() string {
if t.Value == "" {
return t.Key
}
return fmt.Sprintf("%v:%v", t.Key, t.Value)
}
// Add adds value to key
func (s *vgwDogStatsd) Add(key string, value int64, tags ...Tag) {
stags := make([]string, len(tags))
for i, t := range tags {
stags[i] = t.ddString()
}
s.c.Count(key, value, stags, rateSampleAlways)
}

View File

@@ -1,225 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package metrics
import (
"context"
"errors"
"fmt"
"net/http"
"os"
"strings"
"sync"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3err"
)
var (
// max size of data items to buffer before dropping
// new incoming data items
dataItemCount = 100000
)
// Tag is added metadata for metrics
type Tag struct {
// Key is tag name
Key string
// Value is tag data
Value string
}
// Manager is a manager of metrics plugins
type Manager struct {
wg sync.WaitGroup
ctx context.Context
config Config
publishers []publisher
addDataChan chan datapoint
}
type Config struct {
ServiceName string
StatsdServers string
DogStatsdServers string
}
// NewManager initializes metrics plugins and returns a new metrics manager
func NewManager(ctx context.Context, conf Config) (*Manager, error) {
if len(conf.StatsdServers) == 0 && len(conf.DogStatsdServers) == 0 {
return nil, nil
}
if conf.ServiceName == "" {
hostname, err := os.Hostname()
if err != nil {
return nil, fmt.Errorf("failed to get hostname: %w", err)
}
conf.ServiceName = hostname
}
addDataChan := make(chan datapoint, dataItemCount)
mgr := &Manager{
addDataChan: addDataChan,
ctx: ctx,
config: conf,
}
// setup statsd endpoints
if len(conf.StatsdServers) > 0 {
statsdServers := strings.Split(conf.StatsdServers, ",")
for _, server := range statsdServers {
statsd, err := newStatsd(server, conf.ServiceName)
if err != nil {
return nil, err
}
mgr.publishers = append(mgr.publishers, statsd)
}
}
// setup dogstatsd endpoints
if len(conf.DogStatsdServers) > 0 {
dogStatsdServers := strings.Split(conf.DogStatsdServers, ",")
for _, server := range dogStatsdServers {
dogStatsd, err := newDogStatsd(server, conf.ServiceName)
if err != nil {
return nil, err
}
mgr.publishers = append(mgr.publishers, dogStatsd)
}
}
mgr.wg.Add(1)
go mgr.addForwarder(addDataChan)
return mgr, nil
}
func (m *Manager) Send(ctx *fiber.Ctx, err error, action string, count int64, status int) {
// In case of Authentication failures, url parsing ...
if action == "" {
action = ActionUndetected
}
a := ActionMap[action]
reqTags := []Tag{
{Key: "method", Value: ctx.Method()},
{Key: "api", Value: a.Service},
{Key: "action", Value: a.Name},
}
reqStatus := status
if err != nil {
var apierr s3err.APIError
if errors.As(err, &apierr) {
reqStatus = apierr.HTTPStatusCode
} else {
reqStatus = http.StatusInternalServerError
}
}
if reqStatus == 0 {
reqStatus = http.StatusOK
}
reqTags = append(reqTags, Tag{
Key: "status",
Value: fmt.Sprintf("%v", reqStatus),
})
if err != nil {
m.increment("failed_count", reqTags...)
} else {
m.increment("success_count", reqTags...)
}
switch action {
case ActionPutObject:
m.add("bytes_written", count, reqTags...)
m.increment("object_created_count", reqTags...)
case ActionCompleteMultipartUpload:
m.increment("object_created_count", reqTags...)
case ActionUploadPart:
m.add("bytes_written", count, reqTags...)
case ActionGetObject:
m.add("bytes_read", count, reqTags...)
case ActionDeleteObject:
m.increment("object_removed_count", reqTags...)
case ActionDeleteObjects:
m.add("object_removed_count", count, reqTags...)
}
}
// increment increments the key by one
func (m *Manager) increment(key string, tags ...Tag) {
m.add(key, 1, tags...)
}
// add adds value to key
func (m *Manager) add(key string, value int64, tags ...Tag) {
if m.ctx.Err() != nil {
return
}
d := datapoint{
key: key,
value: value,
tags: tags,
}
select {
case m.addDataChan <- d:
default:
// channel full, drop the updates
}
}
// Close closes metrics channels, waits for data to complete, closes all plugins
func (m *Manager) Close() {
// drain the datapoint channels
close(m.addDataChan)
m.wg.Wait()
// close all publishers
for _, p := range m.publishers {
p.Close()
}
}
// publisher is the interface for interacting with the metrics plugins
type publisher interface {
Add(key string, value int64, tags ...Tag)
Close()
}
func (m *Manager) addForwarder(addChan <-chan datapoint) {
for data := range addChan {
for _, s := range m.publishers {
s.Add(data.key, data.value, data.tags...)
}
}
m.wg.Done()
}
type datapoint struct {
key string
value int64
tags []Tag
}

View File

@@ -1,51 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package metrics
import (
"github.com/smira/go-statsd"
)
// vgwStatsd metrics type
type vgwStatsd struct {
c *statsd.Client
}
// newStatsd takes a server address and returns a statsd merics
// Supply service name to be used as a tag to identify the spcific
// gateway instance, this may typically be the gateway hostname
func newStatsd(server string, service string) (*vgwStatsd, error) {
c := statsd.NewClient(
server,
statsd.MetricPrefix("versitygw."),
statsd.TagStyle(statsd.TagFormatInfluxDB),
statsd.DefaultTags(statsd.StringTag("service", service)),
)
return &vgwStatsd{c: c}, nil
}
// Close closes statsd connections
func (s *vgwStatsd) Close() {
s.c.Close()
}
// Add adds value to key
func (s *vgwStatsd) Add(key string, value int64, tags ...Tag) {
stags := make([]statsd.Tag, len(tags))
for i, t := range tags {
stags[i] = statsd.StringTag(t.Key, t.Value)
}
s.c.Incr(key, value, stags...)
}

View File

@@ -5,27 +5,15 @@ rm -rf /tmp/gw
mkdir /tmp/gw
rm -rf /tmp/covdata
mkdir /tmp/covdata
rm -rf /tmp/versioing.covdata
mkdir /tmp/versioning.covdata
rm -rf /tmp/versioningdir
mkdir /tmp/versioningdir
# setup tls certificate and key
ECHO "Generating TLS certificate and key in the cert.pem and key.pem files"
openssl genpkey -algorithm RSA -out key.pem -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key key.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
ECHO "Running the sdk test over http"
# run server in background not versioning-enabled
# port: 7070(default)
# run server in background
GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
GW_PID=$!
# wait a second for server to start up
sleep 1
# check if gateway process is still running
# check if server is still running
if ! kill -0 $GW_PID; then
echo "server no longer running"
exit 1
@@ -51,110 +39,8 @@ if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 iam; then
exit 1
fi
# kill off server
kill $GW_PID
ECHO "Running the sdk test over https"
# run server in background with TLS certificate
# port: 7071(default)
GOCOVERDIR=/tmp/https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7071 -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
GW_HTTPS_PID=$!
sleep 1
# check if https gateway process is still running
if ! kill -0 $GW_HTTPS_PID; then
echo "server no longer running"
exit 1
fi
# run tests
# full flow tests
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 full-flow; then
echo "full flow tests failed"
kill $GW_HTTPS_PID
exit 1
fi
# posix tests
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 posix; then
echo "posix tests failed"
kill $GW_HTTPS_PID
exit 1
fi
# iam tests
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7071 iam; then
echo "iam tests failed"
kill $GW_HTTPS_PID
exit 1
fi
kill $GW_HTTPS_PID
ECHO "Running the sdk test over http against the versioning-enabled gateway"
# run server in background versioning-enabled
# port: 7072
GOCOVERDIR=/tmp/versioning.covdata ./versitygw -p :7072 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
GW_VS_PID=$!
# wait a second for server to start up
sleep 1
# check if versioning-enabled gateway process is still running
if ! kill -0 $GW_VS_PID; then
echo "versioning-enabled server no longer running"
exit 1
fi
# run tests
# full flow tests
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 full-flow -vs; then
echo "versioning-enabled full-flow tests failed"
kill $GW_VS_PID
exit 1
fi
# posix tests
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7072 posix -vs; then
echo "versiongin-enabled posix tests failed"
kill $GW_VS_PID
exit 1
fi
# kill off server
kill $GW_VS_PID
ECHO "Running the sdk test over https against the versioning-enabled gateway"
# run server in background versioning-enabled
# port: 7073
GOCOVERDIR=/tmp/versioning.https.covdata ./versitygw --cert "$PWD/cert.pem" --key "$PWD/key.pem" -p :7073 -a user -s pass --iam-dir /tmp/gw posix --versioning-dir /tmp/versioningdir /tmp/gw &
GW_VS_HTTPS_PID=$!
# wait a second for server to start up
sleep 1
# check if versioning-enabled gateway process is still running
if ! kill -0 $GW_VS_HTTPS_PID; then
echo "versioning-enabled server no longer running"
exit 1
fi
# run tests
# full flow tests
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 full-flow -vs; then
echo "versioning-enabled full-flow tests failed"
kill $GW_VS_HTTPS_PID
exit 1
fi
# posix tests
if ! ./versitygw test --allow-insecure -a user -s pass -e https://127.0.0.1:7073 posix -vs; then
echo "versiongin-enabled posix tests failed"
kill $GW_VS_HTTPS_PID
exit 1
fi
# kill off server
kill $GW_VS_HTTPS_PID
exit 0
# if the above binary was built with -cover enabled (make testbin),

View File

@@ -19,13 +19,12 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3log"
)
type S3AdminRouter struct{}
func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) {
controller := controllers.NewAdminController(iam, be, logger)
func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
controller := controllers.NewAdminController(iam, be)
// CreateUser admin api
app.Patch("/create-user", controller.CreateUser)
@@ -33,9 +32,6 @@ func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMSe
// DeleteUsers admin api
app.Patch("/delete-user", controller.DeleteUser)
// UpdateUser admin api
app.Patch("/update-user", controller.UpdateUser)
// ListUsers admin api
app.Patch("/list-users", controller.ListUsers)

View File

@@ -22,7 +22,6 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3log"
)
type S3AdminServer struct {
@@ -33,7 +32,7 @@ type S3AdminServer struct {
cert *tls.Certificate
}
func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {
func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
server := &S3AdminServer{
app: app,
backend: be,
@@ -47,16 +46,14 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
// Logging middlewares
app.Use(logger.New())
app.Use(middlewares.DecodeURL(l, nil))
app.Use(middlewares.DecodeURL(nil))
// Authentication middlewares
app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false))
app.Use(middlewares.VerifyMD5Body(l))
app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
app.Use(middlewares.VerifyMD5Body(nil))
app.Use(middlewares.AclParser(be, nil))
// Admin role checker
app.Use(middlewares.IsAdmin(l))
server.router.Init(app, be, iam, l)
server.router.Init(app, be, iam)
return server
}

View File

@@ -16,188 +16,107 @@ package controllers
import (
"encoding/json"
"encoding/xml"
"fmt"
"net/http"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
"github.com/versity/versitygw/s3response"
)
type AdminController struct {
iam auth.IAMService
be backend.Backend
l s3log.AuditLogger
}
func NewAdminController(iam auth.IAMService, be backend.Backend, l s3log.AuditLogger) AdminController {
return AdminController{iam: iam, be: be, l: l}
func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
return AdminController{iam: iam, be: be}
}
func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return fmt.Errorf("access denied: only admin users have access to this resource")
}
var usr auth.Account
err := xml.Unmarshal(ctx.Body(), &usr)
err := json.Unmarshal(ctx.Body(), &usr)
if err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminCreateUser,
})
return fmt.Errorf("failed to parse request body: %w", err)
}
if !usr.Role.IsValid() {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminInvalidUserRole),
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminCreateUser,
})
if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
}
err = c.iam.CreateAccount(usr)
if err != nil {
if strings.Contains(err.Error(), "user already exists") {
err = s3err.GetAPIError(s3err.ErrAdminUserExists)
}
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminCreateUser,
})
return fmt.Errorf("failed to create a user: %w", err)
}
return SendResponse(ctx, nil,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminCreateUser,
Status: http.StatusCreated,
})
}
func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
access := ctx.Query("access")
if access == "" {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminMissingUserAcess),
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminUpdateUser,
})
}
var props auth.MutableProps
if err := xml.Unmarshal(ctx.Body(), &props); err != nil {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedXML),
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminUpdateUser,
})
}
err := c.iam.UpdateUserAccount(access, props)
if err != nil {
if strings.Contains(err.Error(), "user not found") {
err = s3err.GetAPIError(s3err.ErrAdminUserNotFound)
}
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminUpdateUser,
})
}
return SendResponse(ctx, nil,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminUpdateUser,
})
return ctx.SendString("The user has been created successfully")
}
func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
access := ctx.Query("access")
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return fmt.Errorf("access denied: only admin users have access to this resource")
}
err := c.iam.DeleteUserAccount(access)
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminDeleteUser,
})
if err != nil {
return err
}
return ctx.SendString("The user has been deleted successfully")
}
func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return fmt.Errorf("access denied: only admin users have access to this resource")
}
accs, err := c.iam.ListUserAccounts()
return SendXMLResponse(ctx,
auth.ListUserAccountsResult{
Accounts: accs,
}, err,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminListUsers,
})
if err != nil {
return err
}
return ctx.JSON(accs)
}
func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return fmt.Errorf("access denied: only admin users have access to this resource")
}
owner := ctx.Query("owner")
bucket := ctx.Query("bucket")
accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
if err != nil {
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminChangeBucketOwner,
})
return err
}
if len(accs) > 0 {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminUserNotFound),
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminChangeBucketOwner,
})
return fmt.Errorf("user specified as the new bucket owner does not exist")
}
acl := auth.ACL{
Owner: owner,
Grantees: []auth.Grantee{
{
Permission: auth.PermissionFullControl,
Access: owner,
Type: types.TypeCanonicalUser,
},
},
}
aclParsed, err := json.Marshal(acl)
err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
if err != nil {
return SendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err),
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminChangeBucketOwner,
})
return err
}
err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed)
return SendResponse(ctx, err,
&MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminChangeBucketOwner,
})
return ctx.Status(201).SendString("Bucket owner has been updated successfully")
}
func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return fmt.Errorf("access denied: only admin users have access to this resource")
}
buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
return SendXMLResponse(ctx,
s3response.ListBucketsResult{
Buckets: buckets,
}, err, &MetaOpts{
Logger: c.l,
Action: metrics.ActionAdminListBuckets,
})
if err != nil {
return err
}
return ctx.JSON(buckets)
}

View File

@@ -15,11 +15,12 @@
package controllers
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gofiber/fiber/v2"
@@ -42,26 +43,33 @@ func TestAdminController_CreateUser(t *testing.T) {
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
app.Patch("/create-user", adminController.CreateUser)
succUser := `
<Account>
<Access>access</Access>
<Secret>secret</Secret>
<Role>admin</Role>
<UserID>0</UserID>
<GroupID>0</GroupID>
</Account>
`
invuser := `
<Account>
<Access>access</Access>
<Secret>secret</Secret>
<Role>invalid_role</Role>
<UserID>0</UserID>
<GroupID>0</GroupID>
</Account>
`
appErr := fiber.New()
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
return ctx.Next()
})
usr := auth.Account{
Access: "access",
Secret: "secret",
Role: "invalid role",
}
user, _ := json.Marshal(&usr)
usr.Role = "admin"
succUsr, _ := json.Marshal(&usr)
appErr.Patch("/create-user", adminController.CreateUser)
tests := []struct {
name string
@@ -71,31 +79,31 @@ func TestAdminController_CreateUser(t *testing.T) {
statusCode int
}{
{
name: "Admin-create-user-malformed-body",
name: "Admin-create-user-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Admin-create-user-invalid-user-role",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
},
wantErr: false,
statusCode: 500,
},
{
name: "Admin-create-user-invalid-requester-role",
app: appErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Admin-create-user-invalid-requester-role",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(invuser)),
},
wantErr: false,
statusCode: 400,
},
{
name: "Admin-create-user-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/create-user", strings.NewReader(succUser)),
},
wantErr: false,
statusCode: 201,
statusCode: 500,
},
}
for _, tt := range tests {
@@ -111,100 +119,6 @@ func TestAdminController_CreateUser(t *testing.T) {
}
}
func TestAdminController_UpdateUser(t *testing.T) {
type args struct {
req *http.Request
}
adminController := AdminController{
iam: &IAMServiceMock{
UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
return nil
},
},
}
app := fiber.New()
app.Patch("/update-user", adminController.UpdateUser)
adminControllerErr := AdminController{
iam: &IAMServiceMock{
UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
return auth.ErrNoSuchUser
},
},
}
appNotFound := fiber.New()
appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
succUser := `
<Account>
<Secret>secret</Secret>
<UserID>0</UserID>
<GroupID>0</GroupID>
</Account>
`
tests := []struct {
name string
app *fiber.App
args args
wantErr bool
statusCode int
}{
{
name: "Admin-update-user-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Admin-update-user-missing-access",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/update-user", strings.NewReader(succUser)),
},
wantErr: false,
statusCode: 404,
},
{
name: "Admin-update-user-invalid-request-body",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Admin-update-user-not-found",
app: appNotFound,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", strings.NewReader(succUser)),
},
wantErr: false,
statusCode: 404,
},
}
for _, tt := range tests {
resp, err := tt.app.Test(tt.args.req)
if (err != nil) != tt.wantErr {
t.Errorf("AdminController.UpdateUser() error = %v, wantErr %v", err, tt.wantErr)
}
if resp.StatusCode != tt.statusCode {
t.Errorf("AdminController.UpdateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
}
}
}
func TestAdminController_DeleteUser(t *testing.T) {
type args struct {
req *http.Request
@@ -220,8 +134,22 @@ func TestAdminController_DeleteUser(t *testing.T) {
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
app.Patch("/delete-user", adminController.DeleteUser)
appErr := fiber.New()
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
return ctx.Next()
})
appErr.Patch("/delete-user", adminController.DeleteUser)
tests := []struct {
name string
app *fiber.App
@@ -238,6 +166,15 @@ func TestAdminController_DeleteUser(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Admin-delete-user-invalid-requester-role",
app: appErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
},
wantErr: false,
statusCode: 500,
},
}
for _, tt := range tests {
resp, err := tt.app.Test(tt.args.req)
@@ -274,9 +211,30 @@ func TestAdminController_ListUsers(t *testing.T) {
}
appErr := fiber.New()
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
appErr.Patch("/list-users", adminControllerErr.ListUsers)
appRoleErr := fiber.New()
appRoleErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
return ctx.Next()
})
appRoleErr.Patch("/list-users", adminController.ListUsers)
appSucc := fiber.New()
appSucc.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
appSucc.Patch("/list-users", adminController.ListUsers)
tests := []struct {
@@ -286,6 +244,15 @@ func TestAdminController_ListUsers(t *testing.T) {
wantErr bool
statusCode int
}{
{
name: "Admin-list-users-access-denied",
app: appRoleErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
},
wantErr: false,
statusCode: 500,
},
{
name: "Admin-list-users-iam-error",
app: appErr,
@@ -324,7 +291,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
}
adminController := AdminController{
be: &BackendMock{
ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error {
ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, newOwner string) error {
return nil
},
},
@@ -352,12 +319,39 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
}
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
appRoleErr := fiber.New()
appRoleErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
return ctx.Next()
})
appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
appIamErr := fiber.New()
appIamErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
appIamNoSuchUser := fiber.New()
appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
tests := []struct {
@@ -367,6 +361,15 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
wantErr bool
statusCode int
}{
{
name: "Change-bucket-owner-access-denied",
app: appRoleErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
},
wantErr: false,
statusCode: 500,
},
{
name: "Change-bucket-owner-check-account-server-error",
app: appIamErr,
@@ -383,7 +386,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
},
wantErr: false,
statusCode: 404,
statusCode: 500,
},
{
name: "Change-bucket-owner-success",
@@ -392,7 +395,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
},
wantErr: false,
statusCode: 200,
statusCode: 201,
},
}
for _, tt := range tests {
@@ -421,8 +424,23 @@ func TestAdminController_ListBuckets(t *testing.T) {
}
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
app.Patch("/list-buckets", adminController.ListBuckets)
appRoleErr := fiber.New()
appRoleErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
return ctx.Next()
})
appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
tests := []struct {
name string
app *fiber.App
@@ -430,6 +448,15 @@ func TestAdminController_ListBuckets(t *testing.T) {
wantErr bool
statusCode int
}{
{
name: "List-buckets-incorrect-role",
app: appRoleErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
},
wantErr: false,
statusCode: 500,
},
{
name: "List-buckets-success",
app: app,

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -19,6 +19,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"reflect"
@@ -76,7 +77,7 @@ func TestNew(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := New(tt.args.be, tt.args.iam, nil, nil, nil, false, false)
got := New(tt.args.be, tt.args.iam, nil, nil, false)
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("New() = %v, want %v", got, tt.want)
}
@@ -92,7 +93,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, nil
},
},
@@ -109,7 +110,7 @@ func TestS3ApiController_ListBuckets(t *testing.T) {
appErr := fiber.New()
s3ApiControllerErr := S3ApiController{
be: &BackendMock{
ListBucketsFunc: func(contextMoqParam context.Context, listBucketsInput s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
ListBucketsFunc: func(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
},
},
@@ -187,10 +188,10 @@ func TestS3ApiController_GetActions(t *testing.T) {
GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
return &s3.GetObjectAclOutput{}, nil
},
GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
return s3response.GetObjectAttributesResponse{}, nil
GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
return &s3.GetObjectAttributesOutput{}, nil
},
GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
GetObjectFunc: func(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
return &s3.GetObjectOutput{
Metadata: map[string]string{"hello": "world"},
ContentType: getPtr("application/xml"),
@@ -204,19 +205,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
GetObjectTaggingFunc: func(_ context.Context, bucket, object string) (map[string]string, error) {
return map[string]string{"hello": "world"}, nil
},
GetObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) {
result, err := json.Marshal(types.ObjectLockRetention{
Mode: types.ObjectLockRetentionModeCompliance,
})
if err != nil {
return nil, err
}
return result, nil
},
GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) {
result := true
return &result, nil
},
},
}
app.Use(func(ctx *fiber.Ctx) error {
@@ -232,9 +220,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
getObjAttrs := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
getObjAttrs.Header.Set("X-Amz-Object-Attributes", "hello")
invalidChecksumMode := httptest.NewRequest(http.MethodGet, "/my-bucket/key", nil)
invalidChecksumMode.Header.Set("x-amz-checksum-mode", "invalid_checksum_mode")
tests := []struct {
name string
app *fiber.App
@@ -251,24 +236,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-get-object-retention-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?retention", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-get-object-legal-hold-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?legal-hold", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-invalid-max-parts-string",
app: app,
@@ -332,15 +299,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-get-object-invalid-checksum-mode",
app: app,
args: args{
req: invalidChecksumMode,
},
wantErr: false,
statusCode: 400,
},
{
name: "Get-actions-get-object-success",
app: app,
@@ -371,11 +329,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
req *http.Request
}
objectLockResult, err := json.Marshal(auth.BucketLockConfig{})
if err != nil {
t.Errorf("failed to parse object lock result %v", err)
}
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
@@ -385,30 +338,24 @@ func TestS3ApiController_ListActions(t *testing.T) {
ListMultipartUploadsFunc: func(_ context.Context, output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
return s3response.ListMultipartUploadsResult{}, nil
},
ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
return s3response.ListObjectsV2Result{}, nil
ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
return &s3.ListObjectsV2Output{}, nil
},
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
return s3response.ListObjectsResult{}, nil
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return &s3.ListObjectsOutput{}, nil
},
GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
return map[string]string{}, nil
},
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
return s3response.GetBucketVersioningOutput{}, nil
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
return &s3.GetBucketVersioningOutput{}, nil
},
ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
return s3response.ListVersionsResult{}, nil
ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
return &s3.ListObjectVersionsOutput{}, nil
},
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return []byte{}, nil
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return objectLockResult, nil
},
GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
return types.ObjectOwnershipBucketOwnerEnforced, nil
},
},
}
@@ -422,14 +369,14 @@ func TestS3ApiController_ListActions(t *testing.T) {
app.Get("/:bucket", s3ApiController.ListActions)
// Error case
//Error case
s3ApiControllerError := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
},
GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -462,15 +409,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
wantErr: false,
statusCode: 404,
},
{
name: "Get-bucket-ownership-control-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket?ownershipControls", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-bucket-tagging-success",
app: app,
@@ -480,15 +418,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Get-object-lock-configuration-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket?object-lock", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-bucket-acl-success",
app: app,
@@ -585,7 +514,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
app := fiber.New()
// Mock valid acl
acl := auth.ACL{Owner: "valid access"}
acl := auth.ACL{Owner: "valid access", ACL: "public-read-write"}
acldata, err := json.Marshal(acl)
if err != nil {
t.Errorf("Failed to parse the params: %v", err.Error())
@@ -616,6 +545,14 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
</AccessControlPolicy>
`
succBody := `
<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Owner>
<ID>valid access</ID>
</Owner>
</AccessControlPolicy>
`
tagBody := `
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
@@ -634,7 +571,8 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
</VersioningConfiguration>
`
policyBody := `{
policyBody := `
{
"Statement": [
{
"Effect": "Allow",
@@ -646,34 +584,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
}
`
objectLockBody := `
<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<ObjectLockEnabled>Enabled</ObjectLockEnabled>
<Rule>
<DefaultRetention>
<Mode>GOVERNANCE</Mode>
<Years>2</Years>
</DefaultRetention>
</Rule>
</ObjectLockConfiguration>
`
ownershipBody := `
<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Rule>
<ObjectOwnership>BucketOwnerEnforced</ObjectOwnership>
</Rule>
</OwnershipControls>
`
invalidOwnershipBody := `
<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Rule>
<ObjectOwnership>invalid_value</ObjectOwnership>
</Rule>
</OwnershipControls>
`
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -688,21 +598,12 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
return nil
},
PutBucketVersioningFunc: func(contextMoqParam context.Context, bucket string, status types.BucketVersioningStatus) error {
PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
return nil
},
PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
return nil
},
PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error {
return nil
},
PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
return nil
},
GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
return types.ObjectOwnershipBucketOwnerPreferred, nil
},
},
}
// Mock ctx.Locals
@@ -726,18 +627,16 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
// PutBucketAcl incorrect bucket owner case
incorrectBucketOwner := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(invOwnerBody))
incorrectBucketOwner.Header.Set("X-Amz-Acl", "private")
// PutBucketAcl acl success
aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil)
aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(succBody))
aclSuccReq.Header.Set("X-Amz-Acl", "private")
// Invalid acl body case
errAclBodyReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(body))
errAclBodyReq.Header.Set("X-Amz-Grant-Read", "hello")
invAclOwnershipReq := httptest.NewRequest(http.MethodPut, "/my-bucket", nil)
invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
tests := []struct {
name string
app *fiber.App
@@ -761,42 +660,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", strings.NewReader(tagBody)),
},
wantErr: false,
statusCode: 204,
},
{
name: "Put-bucket-ownership-controls-invalid-ownership",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(invalidOwnershipBody)),
},
wantErr: false,
statusCode: 400,
},
{
name: "Put-bucket-ownership-controls-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(ownershipBody)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Put-object-lock-configuration-invalid-body",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Put-object-lock-configuration-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", strings.NewReader(objectLockBody)),
},
wantErr: false,
statusCode: 200,
},
{
@@ -869,7 +732,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
req: incorrectBucketOwner,
},
wantErr: false,
statusCode: 400,
statusCode: 403,
},
{
name: "Put-bucket-acl-success",
@@ -881,16 +744,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
statusCode: 200,
},
{
name: "Create-bucket-invalid-acl-ownership-combination",
app: app,
args: args{
req: invAclOwnershipReq,
},
wantErr: false,
statusCode: 400,
},
{
name: "Create-bucket-invalid-bucket-name",
name: "Put-bucket-invalid-bucket-name",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/aa", nil),
@@ -899,7 +753,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
statusCode: 400,
},
{
name: "Create-bucket-success",
name: "Put-bucket-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket", nil),
@@ -952,19 +806,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
</Tagging>
`
//retentionBody := `
//<Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
// <Mode>GOVERNANCE</Mode>
// <RetainUntilDate>2025-01-01T00:00:00Z</RetainUntilDate>
//</Retention>
//`
legalHoldBody := `
<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Status>ON</Status>
</LegalHold>
`
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
@@ -974,31 +815,22 @@ func TestS3ApiController_PutActions(t *testing.T) {
PutObjectAclFunc: func(context.Context, *s3.PutObjectAclInput) error {
return nil
},
CopyObjectFunc: func(context.Context, s3response.CopyObjectInput) (*s3.CopyObjectOutput, error) {
CopyObjectFunc: func(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
return &s3.CopyObjectOutput{
CopyObjectResult: &types.CopyObjectResult{},
}, nil
},
PutObjectFunc: func(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
return s3response.PutObjectOutput{}, nil
PutObjectFunc: func(context.Context, *s3.PutObjectInput) (string, error) {
return "ETag", nil
},
UploadPartFunc: func(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
return &s3.UploadPartOutput{}, nil
UploadPartFunc: func(context.Context, *s3.UploadPartInput) (string, error) {
return "hello", nil
},
PutObjectTaggingFunc: func(_ context.Context, bucket, object string, tags map[string]string) error {
return nil
},
UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
return s3response.CopyPartResult{}, nil
},
PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error {
return nil
},
PutObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
return nil
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
return s3response.CopyObjectResult{}, nil
},
},
}
@@ -1023,11 +855,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
cpySrcReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
cpySrcReq.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
// CopyObject invalid checksum algorithm
cpyInvChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
cpyInvChecksumAlgo.Header.Set("X-Amz-Copy-Source", "srcBucket/srcObject")
cpyInvChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
// PutObjectAcl success
aclReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
aclReq.Header.Set("X-Amz-Acl", "private")
@@ -1049,40 +876,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
invAclBodyGrtReq := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?acl", strings.NewReader(body))
invAclBodyGrtReq.Header.Set("X-Amz-Grant-Read", "hello")
// PutObject invalid checksum algorithm
invChecksumAlgo := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
// PutObject invalid base64 checksum
invBase64Checksum := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invBase64Checksum.Header.Set("X-Amz-Checksum-Crc32", "invalid_base64")
// PutObject invalid crc32
invCrc32 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invCrc32.Header.Set("X-Amz-Checksum-Crc32", "YXNkZmFkc2Zhc2Rm")
// PutObject invalid crc32c
invCrc32c := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invCrc32c.Header.Set("X-Amz-Checksum-Crc32c", "YXNkZmFkc2Zhc2RmYXNkZg==")
// PutObject invalid sha1
invSha1 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invSha1.Header.Set("X-Amz-Checksum-Sha1", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZg==")
// PutObject invalid sha256
invSha256 := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
invSha256.Header.Set("X-Amz-Checksum-Sha256", "YXNkZmFkc2Zhc2RmYXNkZnNkYWZkYXNmZGFzZmFkc2Zhc2Rm")
// PutObject multiple checksum headers
mulChecksumHdrs := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
mulChecksumHdrs.Header.Set("X-Amz-Checksum-Sha256", "d1SPCd/kZ2rAzbbLUC0n/bEaOSx70FNbXbIqoIxKuPY=")
mulChecksumHdrs.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
// PutObject checksum algorithm and header mismatch
checksumHdrMismatch := httptest.NewRequest(http.MethodPut, "/my-bucket/my-key", nil)
checksumHdrMismatch.Header.Set("X-Amz-Checksum-Algorithm", "SHA1")
checksumHdrMismatch.Header.Set("X-Amz-Checksum-Crc32c", "ww2FVQ==")
tests := []struct {
name string
app *fiber.App
@@ -1117,42 +910,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "put-object-retention-invalid-request",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", nil),
},
wantErr: false,
statusCode: 400,
},
//{
// name: "put-object-retention-success",
// app: app,
// args: args{
// req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", strings.NewReader(retentionBody)),
// },
// wantErr: false,
// statusCode: 200,
//},
{
name: "put-legal-hold-invalid-request",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "put-legal-hold-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", strings.NewReader(legalHoldBody)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Put-object-acl-invalid-acl",
app: app,
@@ -1225,15 +982,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Copy-object-invalid-checksum-algorithm",
app: app,
args: args{
req: cpyInvChecksumAlgo,
},
wantErr: false,
statusCode: 400,
},
{
name: "Copy-object-success",
app: app,
@@ -1276,18 +1024,12 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
DeleteBucketFunc: func(_ context.Context, bucket string) error {
DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
return nil
},
DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
return nil
},
DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
return nil
},
DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
return nil
},
},
}
@@ -1326,23 +1068,6 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
wantErr: false,
statusCode: 204,
},
{
name: "Delete-bucket-ownership-controls-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodDelete, "/my-bucket?ownershipControls", nil),
},
wantErr: false,
statusCode: 204,
}, {
name: "Delete-bucket-policy-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodDelete, "/my-bucket?policy", nil),
},
wantErr: false,
statusCode: 204,
},
}
for _, tt := range tests {
resp, err := tt.app.Test(tt.args.req)
@@ -1371,9 +1096,6 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
DeleteObjectsFunc: func(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
return s3response.DeleteResult{}, nil
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
},
},
}
@@ -1442,8 +1164,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
return &s3.DeleteObjectOutput{}, nil
DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
return nil
},
AbortMultipartUploadFunc: func(context.Context, *s3.AbortMultipartUploadInput) error {
return nil
@@ -1451,9 +1173,6 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
DeleteObjectTaggingFunc: func(_ context.Context, bucket, object string) error {
return nil
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
},
},
}
@@ -1473,11 +1192,8 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
return s3err.GetAPIError(7)
},
}}
@@ -1569,7 +1285,6 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
ctx.Locals("region", "us-east-1")
return ctx.Next()
})
@@ -1593,7 +1308,6 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
ctx.Locals("region", "us-east-1")
return ctx.Next()
})
@@ -1687,7 +1401,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
return acldata, nil
},
HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
return nil, s3err.GetAPIError(42)
},
},
}
@@ -1701,9 +1415,6 @@ func TestS3ApiController_HeadObject(t *testing.T) {
})
appErr.Head("/:bucket/:key/*", s3ApiControllerErr.HeadObject)
invChecksumMode := httptest.NewRequest(http.MethodHead, "/my-bucket/my-key", nil)
invChecksumMode.Header.Set("X-Amz-Checksum-Mode", "invalid_checksum_mode")
tests := []struct {
name string
app *fiber.App
@@ -1720,15 +1431,6 @@ func TestS3ApiController_HeadObject(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Head-object-invalid-checksum-mode",
app: app,
args: args{
req: invChecksumMode,
},
wantErr: false,
statusCode: 400,
},
{
name: "Head-object-error",
app: appErr,
@@ -1768,8 +1470,8 @@ func TestS3ApiController_CreateActions(t *testing.T) {
CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
return &s3.CompleteMultipartUploadOutput{}, nil
},
CreateMultipartUploadFunc: func(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
return s3response.InitiateMultipartUploadResult{}, nil
CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
return &s3.CreateMultipartUploadOutput{}, nil
},
SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) {
return func(w *bufio.Writer) {}
@@ -1784,19 +1486,6 @@ func TestS3ApiController_CreateActions(t *testing.T) {
</SelectObjectContentRequest>
`
completMpBody := `
<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Part>
<ETag>etag</ETag>
<PartNumber>1</PartNumber>
</Part>
</CompleteMultipartUpload>
`
completMpEmptyBody := `
<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></CompleteMultipartUpload>
`
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "valid access"})
ctx.Locals("isRoot", true)
@@ -1806,9 +1495,6 @@ func TestS3ApiController_CreateActions(t *testing.T) {
})
app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
invChecksumAlgo := httptest.NewRequest(http.MethodPost, "/my-bucket/my-key", nil)
invChecksumAlgo.Header.Set("X-Amz-Checksum-Algorithm", "invalid_checksum_algorithm")
tests := []struct {
name string
app *fiber.App
@@ -1820,11 +1506,20 @@ func TestS3ApiController_CreateActions(t *testing.T) {
name: "Restore-object-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", nil),
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", strings.NewReader(`<root><key>body</key></root>`)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Restore-object-error",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", nil),
},
wantErr: false,
statusCode: 500,
},
{
name: "Select-object-content-invalid-body",
app: app,
@@ -1852,33 +1547,15 @@ func TestS3ApiController_CreateActions(t *testing.T) {
wantErr: false,
statusCode: 400,
},
{
name: "Complete-multipart-upload-empty-parts",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpEmptyBody)),
},
wantErr: false,
statusCode: 400,
},
{
name: "Complete-multipart-upload-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(completMpBody)),
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?uploadId=23423", strings.NewReader(`<root><key>body</key></root>`)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Create-multipart-upload-invalid-checksum-algorithm",
app: app,
args: args{
req: invChecksumAlgo,
},
wantErr: false,
statusCode: 400,
},
{
name: "Create-multipart-upload-success",
app: app,

View File

@@ -33,9 +33,6 @@ var _ auth.IAMService = &IAMServiceMock{}
// ShutdownFunc: func() error {
// panic("mock out the Shutdown method")
// },
// UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
// panic("mock out the UpdateUserAccount method")
// },
// }
//
// // use mockedIAMService in code that requires auth.IAMService
@@ -58,9 +55,6 @@ type IAMServiceMock struct {
// ShutdownFunc mocks the Shutdown method.
ShutdownFunc func() error
// UpdateUserAccountFunc mocks the UpdateUserAccount method.
UpdateUserAccountFunc func(access string, props auth.MutableProps) error
// calls tracks calls to the methods.
calls struct {
// CreateAccount holds details about calls to the CreateAccount method.
@@ -84,20 +78,12 @@ type IAMServiceMock struct {
// Shutdown holds details about calls to the Shutdown method.
Shutdown []struct {
}
// UpdateUserAccount holds details about calls to the UpdateUserAccount method.
UpdateUserAccount []struct {
// Access is the access argument value.
Access string
// Props is the props argument value.
Props auth.MutableProps
}
}
lockCreateAccount sync.RWMutex
lockDeleteUserAccount sync.RWMutex
lockGetUserAccount sync.RWMutex
lockListUserAccounts sync.RWMutex
lockShutdown sync.RWMutex
lockUpdateUserAccount sync.RWMutex
}
// CreateAccount calls CreateAccountFunc.
@@ -249,39 +235,3 @@ func (mock *IAMServiceMock) ShutdownCalls() []struct {
mock.lockShutdown.RUnlock()
return calls
}
// UpdateUserAccount calls UpdateUserAccountFunc.
func (mock *IAMServiceMock) UpdateUserAccount(access string, props auth.MutableProps) error {
if mock.UpdateUserAccountFunc == nil {
panic("IAMServiceMock.UpdateUserAccountFunc: method is nil but IAMService.UpdateUserAccount was just called")
}
callInfo := struct {
Access string
Props auth.MutableProps
}{
Access: access,
Props: props,
}
mock.lockUpdateUserAccount.Lock()
mock.calls.UpdateUserAccount = append(mock.calls.UpdateUserAccount, callInfo)
mock.lockUpdateUserAccount.Unlock()
return mock.UpdateUserAccountFunc(access, props)
}
// UpdateUserAccountCalls gets all the calls that were made to UpdateUserAccount.
// Check the length with:
//
// len(mockedIAMService.UpdateUserAccountCalls())
func (mock *IAMServiceMock) UpdateUserAccountCalls() []struct {
Access string
Props auth.MutableProps
} {
var calls []struct {
Access string
Props auth.MutableProps
}
mock.lockUpdateUserAccount.RLock()
calls = mock.calls.UpdateUserAccount
mock.lockUpdateUserAccount.RUnlock()
return calls
}

View File

@@ -24,7 +24,6 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
@@ -32,7 +31,7 @@ var (
singlePath = regexp.MustCompile(`^/[^/]+/?$`)
)
func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fiber.Handler {
func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
path := ctx.Path()
@@ -49,20 +48,10 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
!ctx.Request().URI().QueryArgs().Has("acl") &&
!ctx.Request().URI().QueryArgs().Has("tagging") &&
!ctx.Request().URI().QueryArgs().Has("versioning") &&
!ctx.Request().URI().QueryArgs().Has("policy") &&
!ctx.Request().URI().QueryArgs().Has("object-lock") &&
!ctx.Request().URI().QueryArgs().Has("ownershipControls") &&
!ctx.Request().URI().QueryArgs().Has("cors") {
!ctx.Request().URI().QueryArgs().Has("policy") {
if err := auth.MayCreateBucket(acct, isRoot); err != nil {
return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
}
if readonly {
return controllers.SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrAccessDenied),
&controllers.MetaOpts{
Logger: logger,
Action: "CreateBucket",
})
}
return ctx.Next()
}
data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
@@ -75,11 +64,6 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}
// if owner is not set, set default owner to root account
if parsedAcl.Owner == "" {
parsedAcl.Owner = ctx.Locals("rootAccess").(string)
}
ctx.Locals("parsedAcl", parsedAcl)
return ctx.Next()
}

View File

@@ -1,59 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"strings"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func IsAdmin(logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != auth.RoleAdmin {
path := ctx.Path()
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAdminAccessDenied),
&controllers.MetaOpts{
Logger: logger,
Action: detectAction(path),
})
}
return ctx.Next()
}
}
func detectAction(path string) (action string) {
if strings.Contains(path, "create-user") {
action = metrics.ActionAdminCreateUser
} else if strings.Contains(path, "update-user") {
action = metrics.ActionAdminUpdateUser
} else if strings.Contains(path, "delete-user") {
action = metrics.ActionAdminDeleteUser
} else if strings.Contains(path, "list-user") {
action = metrics.ActionAdminListUsers
} else if strings.Contains(path, "list-buckets") {
action = metrics.ActionAdminListBuckets
} else if strings.Contains(path, "change-bucket-owner") {
action = metrics.ActionAdminChangeBucketOwner
}
return action
}

View File

@@ -25,7 +25,6 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
@@ -41,7 +40,7 @@ type RootUserConfig struct {
Secret string
}
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
@@ -55,12 +54,16 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
ctx.Locals("startTime", time.Now())
authorization := ctx.Get("Authorization")
if authorization == "" {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger)
}
authData, err := utils.ParseAuthorization(authorization)
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
if authData.Algorithm != "AWS4-HMAC-SHA256" {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger)
}
if authData.Region != region {
@@ -68,44 +71,42 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region),
HTTPStatusCode: http.StatusForbidden,
}, logger, mm)
}, logger)
}
ctx.Locals("isRoot", authData.Access == root.Access)
ctx.Locals("rootAccess", root.Access)
account, err := acct.getAccount(authData.Access)
if err == auth.ErrNoSuchUser {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
}
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
ctx.Locals("account", account)
// Check X-Amz-Date header
date := ctx.Get("X-Amz-Date")
if date == "" {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger)
}
// Parse the date and check the date validity
tdate, err := time.Parse(iso8601Format, date)
if err != nil {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger)
}
if date[:8] != authData.Date {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger)
}
// Validate the dates difference
err = utils.ValidateDate(tdate)
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
hashPayload := ctx.Get("X-Amz-Content-Sha256")
if utils.IsBigDataAction(ctx) {
// for streaming PUT actions, authorization is deferred
// until end of stream due to need to get length and
@@ -113,24 +114,10 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
return utils.NewAuthReader(ctx, r, authData, account.Secret, debug)
})
// wrap the io.Reader with ChunkReader if x-amz-content-sha256
// provide chunk encoding value
if utils.IsStreamingPayload(hashPayload) {
var err error
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
var cr io.Reader
cr, err = utils.NewChunkReader(ctx, r, authData, region, account.Secret, tdate)
return cr
})
if err != nil {
return sendResponse(ctx, err, logger, mm)
}
}
return ctx.Next()
}
hashPayload := ctx.Get("X-Amz-Content-Sha256")
if !utils.IsSpecialPayload(hashPayload) {
// Calculate the hash of the request payload
hashedPayload := sha256.Sum256(ctx.Body())
@@ -138,7 +125,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
// Compare the calculated hash with the hash provided
if hashPayload != hexPayload {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger)
}
}
@@ -147,13 +134,13 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
if contentLengthStr != "" {
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger)
}
}
err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
return ctx.Next()
@@ -177,6 +164,6 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
return a.iam.GetUserAccount(access)
}
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger, mm *metrics.Manager) error {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}

View File

@@ -0,0 +1,61 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"io"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3log"
)
// ProcessChunkedBody initializes the chunked upload stream if the
// request appears to be a chunked upload
func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string) fiber.Handler {
return func(ctx *fiber.Ctx) error {
decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
if decodedLength == "" {
return ctx.Next()
}
// TODO: validate content length
authData, err := utils.ParseAuthorization(ctx.Get("Authorization"))
if err != nil {
return sendResponse(ctx, err, logger)
}
acct := ctx.Locals("account").(auth.Account)
amzdate := ctx.Get("X-Amz-Date")
date, _ := time.Parse(iso8601Format, amzdate)
if utils.IsBigDataAction(ctx) {
var err error
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
var cr *utils.ChunkReader
cr, err = utils.NewChunkReader(ctx, r, authData, region, acct.Secret, date)
return cr
})
if err != nil {
return sendResponse(ctx, err, logger)
}
return ctx.Next()
}
return ctx.Next()
}
}

View File

@@ -45,7 +45,7 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
}
sum := md5.Sum(ctx.Body())
calculatedSum := utils.Base64SumString(sum[:])
calculatedSum := utils.Md5SumString(sum[:])
if incomingSum != calculatedSum {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})

View File

@@ -20,13 +20,12 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
@@ -39,18 +38,16 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
authData, err := utils.ParsePresignedURIParts(ctx)
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
ctx.Locals("isRoot", authData.Access == root.Access)
ctx.Locals("rootAccess", root.Access)
account, err := acct.getAccount(authData.Access)
if err == auth.ErrNoSuchUser {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
}
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
ctx.Locals("account", account)
@@ -64,7 +61,7 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug)
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
return ctx.Next()

View File

@@ -18,19 +18,19 @@ import (
"net/url"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
func DecodeURL(logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
unescp, err := url.QueryUnescape(string(ctx.Request().URI().PathOriginal()))
reqURL := ctx.Request().URI().String()
decoded, err := url.Parse(reqURL)
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
}
ctx.Path(unescp)
ctx.Path(decoded.Path)
return ctx.Next()
}
}

View File

@@ -18,9 +18,7 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3log"
)
@@ -29,29 +27,26 @@ type S3ApiRouter struct {
WithAdmSrv bool
}
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) {
s3ApiController := controllers.New(be, iam, logger, evs, mm, debug, readonly)
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, debug bool) {
s3ApiController := controllers.New(be, iam, logger, evs, debug)
if sa.WithAdmSrv {
adminController := controllers.NewAdminController(iam, be, aLogger)
adminController := controllers.NewAdminController(iam, be)
// CreateUser admin api
app.Patch("/create-user", middlewares.IsAdmin(logger), adminController.CreateUser)
app.Patch("/create-user", adminController.CreateUser)
// DeleteUsers admin api
app.Patch("/delete-user", middlewares.IsAdmin(logger), adminController.DeleteUser)
// UpdateUser admin api
app.Patch("update-user", middlewares.IsAdmin(logger), adminController.UpdateUser)
app.Patch("/delete-user", adminController.DeleteUser)
// ListUsers admin api
app.Patch("/list-users", middlewares.IsAdmin(logger), adminController.ListUsers)
app.Patch("/list-users", adminController.ListUsers)
// ChangeBucketOwner admin api
app.Patch("/change-bucket-owner", middlewares.IsAdmin(logger), adminController.ChangeBucketOwner)
app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
// ListBucketsAndOwners admin api
app.Patch("/list-buckets", middlewares.IsAdmin(logger), adminController.ListBuckets)
app.Patch("/list-buckets", adminController.ListBuckets)
}
// ListBuckets action

View File

@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, nil, nil, false, false)
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, false)
})
}
}

View File

@@ -22,36 +22,23 @@ import (
"github.com/gofiber/fiber/v2/middleware/logger"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3log"
)
type S3ApiServer struct {
app *fiber.App
backend backend.Backend
router *S3ApiRouter
port string
cert *tls.Certificate
quiet bool
debug bool
readonly bool
health string
app *fiber.App
backend backend.Backend
router *S3ApiRouter
port string
cert *tls.Certificate
quiet bool
debug bool
health string
}
func New(
app *fiber.App,
be backend.Backend,
root middlewares.RootUserConfig,
port, region string,
iam auth.IAMService,
l s3log.AuditLogger,
adminLogger s3log.AuditLogger,
evs s3event.S3EventSender,
mm *metrics.Manager,
opts ...Option,
) (*S3ApiServer, error) {
func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, evs s3event.S3EventSender, opts ...Option) (*S3ApiServer, error) {
server := &S3ApiServer{
app: app,
backend: be,
@@ -65,9 +52,7 @@ func New(
// Logging middlewares
if !server.quiet {
app.Use(logger.New(logger.Config{
Format: "${time} | ${status} | ${latency} | ${ip} | ${method} | ${path} | ${error} | ${queryParams}\n",
}))
app.Use(logger.New())
}
// Set up health endpoint if specified
if server.health != "" {
@@ -75,16 +60,17 @@ func New(
return ctx.SendStatus(http.StatusOK)
})
}
app.Use(middlewares.DecodeURL(l, mm))
app.Use(middlewares.DecodeURL(l))
app.Use(middlewares.RequestLogger(server.debug))
// Authentication middlewares
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug))
app.Use(middlewares.VerifyV4Signature(root, iam, l, mm, region, server.debug))
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, region, server.debug))
app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
app.Use(middlewares.ProcessChunkedBody(root, iam, l, region))
app.Use(middlewares.VerifyMD5Body(l))
app.Use(middlewares.AclParser(be, l, server.readonly))
app.Use(middlewares.AclParser(be, l))
server.router.Init(app, be, iam, l, adminLogger, evs, mm, server.debug, server.readonly)
server.router.Init(app, be, iam, l, evs, server.debug)
return server, nil
}
@@ -117,10 +103,6 @@ func WithHealth(health string) Option {
return func(s *S3ApiServer) { s.health = health }
}
func WithReadOnly() Option {
return func(s *S3ApiServer) { s.readonly = true }
}
func (sa *S3ApiServer) Serve() (err error) {
if sa.cert != nil {
return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)

View File

@@ -64,7 +64,7 @@ func TestNew(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil, nil, nil)
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil)
if (err != nil) != tt.wantErr {
t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
return

Some files were not shown because too many files have changed in this diff Show More