Compare commits

..

1 Commits

Author SHA1 Message Date
Ben McClelland
e9b41d53b6 feat: implement read only option for posix/scoutfs
Fixes #484. When read only option selected, backend will return
AccessDenied for any operations that would update data or metadata
in the backend. This include creating and deleting objects and
buckets as well as updating object metadata.

This will still allow admin to change bucket owner since read only
is mainly intended for s3 api only.
2024-04-08 10:39:22 -07:00
194 changed files with 4143 additions and 23039 deletions

View File

@@ -1,23 +1,27 @@
---
name: Bug Report
name: Bug report
about: Create a report to help us improve
title: '[Bug] - <Short Description>'
title: ''
labels: bug
assignees: ''
---
**Describe the bug**
<!-- A clear and concise description of what the bug is. -->
A clear and concise description of what the bug is.
**To Reproduce**
<!-- Steps to reproduce the behavior. -->
Steps to reproduce the behavior.
**Expected behavior**
<!-- A clear and concise description of what you expected to happen. -->
A clear and concise description of what you expected to happen.
**Server Version**
<!-- output of: './versitygw -version && uname -a' -->
output of
```
./versitygw -version
uname -a
```
**Additional context**
<!-- Describe s3 client and version if applicable.
Describe s3 client and version if applicable.

View File

@@ -1,14 +1,14 @@
---
name: Feature Request
name: Feature request
about: Suggest an idea for this project
title: '[Feature] - <Short Description>'
title: ''
labels: enhancement
assignees: ''
---
**Describe the solution you'd like**
<!-- A clear and concise description of what you want to happen. -->
A clear and concise description of what you want to happen.
**Additional context**
<!-- Add any other context or screenshots about the feature request here. -->
Add any other context or screenshots about the feature request here.

View File

@@ -1,33 +0,0 @@
---
name: Test Case Request
about: Request new test cases or additional test coverage
title: '[Test Case] - <Short Description>'
labels: 'testcase'
assignees: ''
---
## Description
<!-- Please provide a detailed description of the test case or test coverage request. -->
## Purpose
<!-- Explain why this test case is important and what it aims to achieve. -->
## Scope
<!-- Describe the scope of the test case, including any specific functionalities, features, or modules that should be tested. -->
## Acceptance Criteria
<!-- List the criteria that must be met for the test case to be considered complete. -->
1.
2.
3.
## Additional Context
<!-- Add any other context or screenshots about the feature request here. -->
## Resources
<!-- Provide any resources, documentation, or links that could help in writing the test case. -->
**Thank you for contributing to our project!**

View File

@@ -1,28 +0,0 @@
name: docker bats tests
on: pull_request
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build Docker Image
run: |
mv tests/.env.docker.default tests/.env.docker
mv tests/.secrets.default tests/.secrets
docker build --build-arg="GO_LIBRARY=go1.21.7.linux-amd64.tar.gz" \
--build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" --build-arg="MC_FOLDER=linux-amd64" \
--progress=plain -f Dockerfile_test_bats -t bats_test .
- name: Set up Docker Compose
run: sudo apt-get install -y docker-compose
- name: Run Docker Container
run: docker-compose -f docker-compose-bats.yml up posix_backend

View File

@@ -15,13 +15,6 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
@@ -50,7 +43,6 @@ jobs:
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64,linux/arm64
build-args: |
VERSION=${{ github.event.release.tag_name }}
TIME=${{ github.event.release.published_at }}

View File

@@ -1,16 +0,0 @@
name: shellcheck
on: pull_request
jobs:
build:
name: Run shellcheck
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Run checks
run: |
shellcheck --version
shellcheck -e SC1091 tests/*.sh tests/*/*.sh

View File

@@ -4,91 +4,16 @@ jobs:
build:
name: RunTests
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- set: 1
LOCAL_FOLDER: /tmp/gw1
BUCKET_ONE_NAME: versity-gwtest-bucket-one-1
BUCKET_TWO_NAME: versity-gwtest-bucket-two-1
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam1
AWS_ENDPOINT_URL: https://127.0.0.1:7070
RUN_SET: "s3cmd"
RECREATE_BUCKETS: "true"
PORT: 7070
BACKEND: "posix"
- set: 2
LOCAL_FOLDER: /tmp/gw2
BUCKET_ONE_NAME: versity-gwtest-bucket-one-2
BUCKET_TWO_NAME: versity-gwtest-bucket-two-2
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam2
AWS_ENDPOINT_URL: https://127.0.0.1:7071
RUN_SET: "s3"
RECREATE_BUCKETS: "true"
PORT: 7071
BACKEND: "posix"
- set: 3
LOCAL_FOLDER: /tmp/gw3
BUCKET_ONE_NAME: versity-gwtest-bucket-one-3
BUCKET_TWO_NAME: versity-gwtest-bucket-two-3
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam3
AWS_ENDPOINT_URL: https://127.0.0.1:7072
RUN_SET: "s3api"
RECREATE_BUCKETS: "true"
PORT: 7072
BACKEND: "posix"
- set: 4
LOCAL_FOLDER: /tmp/gw4
BUCKET_ONE_NAME: versity-gwtest-bucket-one-4
BUCKET_TWO_NAME: versity-gwtest-bucket-two-4
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam4
AWS_ENDPOINT_URL: https://127.0.0.1:7073
RUN_SET: "mc"
RECREATE_BUCKETS: "true"
PORT: 7073
BACKEND: "posix"
- set: 5
LOCAL_FOLDER: /tmp/gw5
BUCKET_ONE_NAME: versity-gwtest-bucket-one-5
BUCKET_TWO_NAME: versity-gwtest-bucket-two-5
IAM_TYPE: s3
USERS_BUCKET: versity-gwtest-iam
AWS_ENDPOINT_URL: https://127.0.0.1:7074
RUN_SET: "aws-user"
RECREATE_BUCKETS: "true"
PORT: 7074
BACKEND: "posix"
- set: 6
LOCAL_FOLDER: /tmp/gw6
BUCKET_ONE_NAME: versity-gwtest-bucket-one-6
BUCKET_TWO_NAME: versity-gwtest-bucket-two-6
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam6
AWS_ENDPOINT_URL: https://127.0.0.1:7075
RUN_SET: "aws"
RECREATE_BUCKETS: "false"
PORT: 7075
BACKEND: "posix"
- set: 7
LOCAL_FOLDER: /tmp/gw7
BUCKET_ONE_NAME: versity-gwtest-bucket-one-7
BUCKET_TWO_NAME: versity-gwtest-bucket-two-7
IAM_TYPE: folder
USERS_FOLDER: /tmp/iam7
AWS_ENDPOINT_URL: https://127.0.0.1:7076
RUN_SET: "aws"
RECREATE_BUCKETS: "true"
PORT: 7076
BACKEND: "s3"
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Install ShellCheck
run: sudo apt-get install shellcheck
- name: Run ShellCheck
run: shellcheck -S warning ./tests/*.sh
- name: Set up Go
uses: actions/setup-go@v5
with:
@@ -103,8 +28,6 @@ jobs:
run: |
git clone https://github.com/bats-core/bats-core.git
cd bats-core && ./install.sh $HOME
git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
- name: Install s3cmd
run: |
@@ -116,51 +39,34 @@ jobs:
chmod 755 /usr/local/bin/mc
- name: Build and run, posix backend
env:
LOCAL_FOLDER: ${{ matrix.LOCAL_FOLDER }}
BUCKET_ONE_NAME: ${{ matrix.BUCKET_ONE_NAME }}
BUCKET_TWO_NAME: ${{ matrix.BUCKET_TWO_NAME }}
USERS_FOLDER: ${{ matrix.USERS_FOLDER }}
USERS_BUCKET: ${{ matrix.USERS_BUCKET }}
IAM_TYPE: ${{ matrix.IAM_TYPE }}
AWS_ENDPOINT_URL: ${{ matrix.AWS_ENDPOINT_URL }}
RUN_SET: ${{ matrix.RUN_SET }}
PORT: ${{ matrix.PORT }}
AWS_PROFILE: versity
VERSITY_EXE: ${{ github.workspace }}/versitygw
RUN_VERSITYGW: true
BACKEND: ${{ matrix.BACKEND }}
RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
CERT: ${{ github.workspace }}/cert.pem
KEY: ${{ github.workspace }}/versitygw.pem
S3CMD_CONFIG: tests/s3cfg.local.default
MC_ALIAS: versity
LOG_LEVEL: 4
GOCOVERDIR: ${{ github.workspace }}/cover
USERNAME_ONE: ABCDEFG
PASSWORD_ONE: 1234567
USERNAME_TWO: HIJKLMN
PASSWORD_TWO: 8901234
TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
run: |
make testbin
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
export AWS_REGION=us-east-1
export AWS_ACCESS_KEY_ID_TWO=user
export AWS_SECRET_ACCESS_KEY_TWO=pass
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
aws configure set aws_region $AWS_REGION --profile versity
mkdir $LOCAL_FOLDER
mkdir /tmp/gw
export WORKSPACE=$GITHUB_WORKSPACE
openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
mkdir $GOCOVERDIR $USERS_FOLDER
if [[ $RECREATE_BUCKETS == "false" ]]; then
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
fi
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
mkdir cover
VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
#- name: Build and run, s3 backend
# run: |
# make testbin
# export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
# export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
# export AWS_REGION=us-east-1
# aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
# aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
# aws configure set aws_region $AWS_REGION --profile versity_s3
# export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
# export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
# export WORKSPACE=$GITHUB_WORKSPACE
# VERSITYGW_TEST_ENV=./tests/.env.s3.default GOCOVERDIR=/tmp/cover ./tests/run_all.sh
- name: Coverage report
run: |

17
.gitignore vendored
View File

@@ -47,18 +47,5 @@ tests/.secrets*
users.json
# env files for testing
**/.env*
**/!.env.default
# s3cmd config files (testing)
tests/s3cfg.local*
tests/!s3cfg.local.default
# keys
*.pem
# patches
*.patch
# grafana's local database (kept on filesystem for survival between instantiations)
metrics-exploration/grafana_data/**
.env*
!.env.default

View File

@@ -6,7 +6,7 @@ COPY go.mod ./
RUN go mod download
COPY ./ ./
COPY ./tests/certs/* /etc/pki/tls/certs/
COPY certs/* /etc/pki/tls/certs/
ARG IAM_DIR=/tmp/vgw
ARG SETUP_DIR=/tmp/vgw

View File

@@ -1,11 +1,8 @@
FROM ubuntu:latest
FROM --platform=linux/arm64 ubuntu:latest
ARG DEBIAN_FRONTEND=noninteractive
ARG SECRETS_FILE=tests/.secrets
ARG CONFIG_FILE=tests/.env.docker
ARG GO_LIBRARY=go1.21.7.linux-arm64.tar.gz
ARG AWS_CLI=awscli-exe-linux-aarch64.zip
ARG MC_FOLDER=linux-arm64
ENV TZ=Etc/UTC
RUN apt-get update && \
@@ -27,20 +24,20 @@ RUN apt-get update && \
WORKDIR /tmp
# Install AWS cli
RUN curl "https://awscli.amazonaws.com/${AWS_CLI}" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
# Install mc
RUN curl https://dl.min.io/client/mc/release/${MC_FOLDER}/mc \
RUN curl https://dl.min.io/client/mc/release/linux-arm64/mc \
--create-dirs \
-o /usr/local/minio-binaries/mc && \
chmod -R 755 /usr/local/minio-binaries
ENV PATH="/usr/local/minio-binaries":${PATH}
# Download Go 1.21 (adjust the version and platform as needed)
RUN wget https://golang.org/dl/${GO_LIBRARY}
RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
# Extract the downloaded archive
RUN tar -xvf $GO_LIBRARY -C /usr/local
RUN tar -xvf go1.21.7.linux-arm64.tar.gz -C /usr/local
# Set Go environment variables
ENV PATH="/usr/local/go/bin:${PATH}"
@@ -63,11 +60,9 @@ RUN git clone https://github.com/bats-core/bats-core.git && \
USER tester
COPY --chown=tester:tester . /home/tester
# add bats support libraries
RUN git clone https://github.com/bats-core/bats-support.git && rm -rf /home/tester/tests/bats-support && mv bats-support /home/tester/tests
RUN git clone https://github.com/ztombol/bats-assert.git && rm -rf /home/tester/tests/bats-assert && mv bats-assert /home/tester/tests
WORKDIR /home/tester
#RUN cp tests/.env.docker.s3.default tests/.env.docker.s3
RUN cp tests/s3cfg.local.default tests/s3cfg.local
RUN make
RUN . $SECRETS_FILE && \

View File

@@ -6,7 +6,7 @@
<a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
</picture>
[![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/versity/versitygw)](https://goreportcard.com/report/github.com/versity/versitygw) [![Go Reference](https://pkg.go.dev/badge/github.com/versity/versitygw.svg)](https://pkg.go.dev/github.com/versity/versitygw)
[![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)
### Binary release builds
Download [latest release](https://github.com/versity/versitygw/releases)
@@ -14,15 +14,8 @@ Download [latest release](https://github.com/versity/versitygw/releases)
|:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
| ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
### Use Cases
* Turn your local filesystem into an S3 server with a single command!
* Proxy S3 requests to S3 storage
* Simple to deploy S3 server with a single command
* Protocol compatibility in `posix` allows common access to files via posix or S3
* Simplified interface for adding new storage system support
### News
Check out latest wiki articles: [https://github.com/versity/versitygw/wiki/Articles](https://github.com/versity/versitygw/wiki/Articles)
* New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
### Mailing List
Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).
@@ -35,6 +28,12 @@ Ask questions in the [community discussions](https://github.com/versity/versityg
<br>
Contact [Versity Sales](https://www.versity.com/contact/) to discuss enterprise support.
### Use Cases
* Share filesystem directory via S3 protocol
* Proxy S3 requests to S3 storage
* Simple to deploy S3 server with a single command
* Protocol compatibility in `posix` allows common access to files via posix or S3
### Overview
Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
@@ -68,7 +67,7 @@ The command format is
```
versitygw [global options] command [command options] [arguments...]
```
The [global options](https://github.com/versity/versitygw/wiki/Global-Options) are specified before the backend type and the backend options are specified after.
The global options are specified before the backend type and the backend options are specified after.
***

View File

@@ -17,7 +17,6 @@ package auth
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
@@ -28,6 +27,7 @@ import (
)
type ACL struct {
ACL types.BucketCannedACL
Owner string
Grantees []Grantee
}
@@ -35,7 +35,6 @@ type ACL struct {
type Grantee struct {
Permission types.Permission
Access string
Type types.Type
}
type GetBucketAclOutput struct {
@@ -43,36 +42,12 @@ type GetBucketAclOutput struct {
AccessControlList AccessControlList
}
type PutBucketAclInput struct {
Bucket *string
ACL types.BucketCannedACL
AccessControlPolicy *AccessControlPolicy
GrantFullControl *string
GrantRead *string
GrantReadACP *string
GrantWrite *string
GrantWriteACP *string
type AccessControlList struct {
Grants []types.Grant `xml:"Grant"`
}
type AccessControlPolicy struct {
AccessControlList AccessControlList `xml:"AccessControlList"`
Owner *types.Owner
}
type AccessControlList struct {
Grants []Grant `xml:"Grant"`
}
type Grant struct {
Grantee *Grt
Permission types.Permission
}
type Grt struct {
XMLNS string `xml:"xmlns:xsi,attr"`
XMLXSI types.Type `xml:"xsi:type,attr"`
Type types.Type `xml:"Type"`
ID string `xml:"ID"`
Owner types.Owner
}
func ParseACL(data []byte) (ACL, error) {
@@ -93,19 +68,11 @@ func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
}
grants := []Grant{}
grants := []types.Grant{}
for _, elem := range acl.Grantees {
acs := elem.Access
grants = append(grants, Grant{
Grantee: &Grt{
XMLNS: "http://www.w3.org/2001/XMLSchema-instance",
XMLXSI: elem.Type,
ID: acs,
Type: elem.Type,
},
Permission: elem.Permission,
})
grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
}
return GetBucketAclOutput{
@@ -118,43 +85,20 @@ func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
}, nil
}
func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool) ([]byte, error) {
func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
if input == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
defaultGrantees := []Grantee{
{
Permission: types.PermissionFullControl,
Access: acl.Owner,
Type: types.TypeCanonicalUser,
},
if acl.Owner != *input.AccessControlPolicy.Owner.ID {
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
}
// if the ACL is specified, set the ACL, else replace the grantees
if input.ACL != "" {
switch input.ACL {
case types.BucketCannedACLPublicRead:
defaultGrantees = append(defaultGrantees, Grantee{
Permission: types.PermissionRead,
Access: "all-users",
Type: types.TypeGroup,
})
case types.BucketCannedACLPublicReadWrite:
defaultGrantees = append(defaultGrantees, []Grantee{
{
Permission: types.PermissionRead,
Access: "all-users",
Type: types.TypeGroup,
},
{
Permission: types.PermissionWrite,
Access: "all-users",
Type: types.TypeGroup,
},
}...)
}
acl.ACL = input.ACL
acl.Grantees = []Grantee{}
} else {
grantees := []Grantee{}
accs := []string{}
if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
@@ -163,71 +107,45 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
if input.GrantFullControl != nil && *input.GrantFullControl != "" {
fullControlList = splitUnique(*input.GrantFullControl, ",")
for _, str := range fullControlList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: types.PermissionFullControl,
Type: types.TypeCanonicalUser,
})
grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
}
}
if input.GrantRead != nil && *input.GrantRead != "" {
readList = splitUnique(*input.GrantRead, ",")
for _, str := range readList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: types.PermissionRead,
Type: types.TypeCanonicalUser,
})
grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
}
}
if input.GrantReadACP != nil && *input.GrantReadACP != "" {
readACPList = splitUnique(*input.GrantReadACP, ",")
for _, str := range readACPList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: types.PermissionReadAcp,
Type: types.TypeCanonicalUser,
})
grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
}
}
if input.GrantWrite != nil && *input.GrantWrite != "" {
writeList = splitUnique(*input.GrantWrite, ",")
for _, str := range writeList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: types.PermissionWrite,
Type: types.TypeCanonicalUser,
})
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
}
}
if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
writeACPList = splitUnique(*input.GrantWriteACP, ",")
for _, str := range writeACPList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: types.PermissionWriteAcp,
Type: types.TypeCanonicalUser,
})
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
}
}
accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
} else {
cache := make(map[string]bool)
for _, grt := range input.AccessControlPolicy.AccessControlList.Grants {
if grt.Grantee == nil || grt.Grantee.ID == "" || grt.Permission == "" {
for _, grt := range input.AccessControlPolicy.Grants {
if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
access := grt.Grantee.ID
defaultGrantees = append(defaultGrantees, Grantee{
Access: access,
Permission: grt.Permission,
Type: types.TypeCanonicalUser,
})
if _, ok := cache[access]; !ok {
cache[access] = true
accs = append(accs, access)
grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
if _, ok := cache[*grt.Grantee.ID]; !ok {
cache[*grt.Grantee.ID] = true
accs = append(accs, *grt.Grantee.ID)
}
}
}
@@ -240,9 +158,10 @@ func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool)
if len(accList) > 0 {
return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
}
}
acl.Grantees = defaultGrantees
acl.Grantees = grantees
acl.ACL = ""
}
result, err := json.Marshal(acl)
if err != nil {
@@ -287,33 +206,34 @@ func splitUnique(s, divider string) []string {
}
func verifyACL(acl ACL, access string, permission types.Permission) error {
grantee := Grantee{
Access: access,
Permission: permission,
Type: types.TypeCanonicalUser,
}
granteeFullCtrl := Grantee{
Access: access,
Permission: types.PermissionFullControl,
Type: types.TypeCanonicalUser,
}
granteeAllUsers := Grantee{
Access: "all-users",
Permission: permission,
Type: types.TypeGroup,
}
isFound := false
for _, grt := range acl.Grantees {
if grt == grantee || grt == granteeFullCtrl || grt == granteeAllUsers {
isFound = true
break
if acl.ACL != "" {
if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
}
if isFound {
return nil
} else {
if len(acl.Grantees) == 0 {
return nil
}
grantee := Grantee{Access: access, Permission: permission}
granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
isFound := false
for _, grt := range acl.Grantees {
if grt == grantee || grt == granteeFullCtrl {
isFound = true
break
}
}
if isFound {
return nil
}
}
return s3err.GetAPIError(s3err.ErrAccessDenied)
@@ -359,31 +279,32 @@ type AccessOptions struct {
Bucket string
Object string
Action Action
Readonly bool
}
func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
if opts.Readonly {
if opts.AclPermission == types.PermissionWrite || opts.AclPermission == types.PermissionWriteAcp {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
}
if opts.IsRoot {
return nil
}
if opts.Acc.Role == RoleAdmin {
return nil
}
policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
if policyErr != nil {
if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
return policyErr
}
} else {
return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
if opts.Acc.Access == opts.Acl.Owner {
return nil
}
policy, err := be.GetBucketPolicy(ctx, opts.Bucket)
if err != nil {
return err
}
// If bucket policy is not set and the ACL is default, only the owner has access
if len(policy) == 0 && opts.Acl.ACL == "" && len(opts.Acl.Grantees) == 0 {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
if err := verifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
return err
}
if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
return err
}

View File

@@ -16,22 +16,12 @@ package auth
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"github.com/versity/versitygw/s3err"
)
var (
errResourceMismatch = errors.New("Action does not apply to any resource(s) in statement")
//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
errInvalidResource = errors.New("Policy has invalid resource")
//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
errInvalidPrincipal = errors.New("Invalid principal in policy")
//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
errInvalidAction = errors.New("Policy has invalid action")
)
type BucketPolicy struct {
Statement []BucketPolicyItem `json:"Statement"`
}
@@ -85,14 +75,11 @@ func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
for action := range bpi.Actions {
isObjectAction := action.IsObjectAction()
if isObjectAction == nil {
break
if isObjectAction && !containsObjectAction {
return fmt.Errorf("unsupported object action '%v' on the specified resources", action)
}
if *isObjectAction && !containsObjectAction {
return errResourceMismatch
}
if !*isObjectAction && !containsBucketAction {
return errResourceMismatch
if !isObjectAction && !containsBucketAction {
return fmt.Errorf("unsupported bucket action '%v' on the specified resources", action)
}
}
@@ -121,11 +108,6 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
return getMalformedPolicyError(err)
}
if len(policy.Statement) == 0 {
//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
return getMalformedPolicyError(errors.New("Could not parse the policy: Statement is empty!"))
}
if err := policy.Validate(bucket, iam); err != nil {
return getMalformedPolicyError(err)
}
@@ -133,7 +115,12 @@ func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) err
return nil
}
func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
func verifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
// If bucket policy is not set
if len(policy) == 0 {
return nil
}
var bucketPolicy BucketPolicy
if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
return err
@@ -144,6 +131,7 @@ func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Act
resource += "/" + object
}
fmt.Println(access, action, resource)
if !bucketPolicy.isAllowed(access, action, resource) {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}

View File

@@ -16,113 +16,92 @@ package auth
import (
"encoding/json"
"fmt"
"strings"
)
type Action string
const (
GetBucketAclAction Action = "s3:GetBucketAcl"
CreateBucketAction Action = "s3:CreateBucket"
PutBucketAclAction Action = "s3:PutBucketAcl"
DeleteBucketAction Action = "s3:DeleteBucket"
PutBucketVersioningAction Action = "s3:PutBucketVersioning"
GetBucketVersioningAction Action = "s3:GetBucketVersioning"
PutBucketPolicyAction Action = "s3:PutBucketPolicy"
GetBucketPolicyAction Action = "s3:GetBucketPolicy"
DeleteBucketPolicyAction Action = "s3:DeleteBucketPolicy"
AbortMultipartUploadAction Action = "s3:AbortMultipartUpload"
ListMultipartUploadPartsAction Action = "s3:ListMultipartUploadParts"
ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
PutObjectAction Action = "s3:PutObject"
GetObjectAction Action = "s3:GetObject"
DeleteObjectAction Action = "s3:DeleteObject"
GetObjectAclAction Action = "s3:GetObjectAcl"
GetObjectAttributesAction Action = "s3:GetObjectAttributes"
PutObjectAclAction Action = "s3:PutObjectAcl"
RestoreObjectAction Action = "s3:RestoreObject"
GetBucketTaggingAction Action = "s3:GetBucketTagging"
PutBucketTaggingAction Action = "s3:PutBucketTagging"
GetObjectTaggingAction Action = "s3:GetObjectTagging"
PutObjectTaggingAction Action = "s3:PutObjectTagging"
DeleteObjectTaggingAction Action = "s3:DeleteObjectTagging"
ListBucketVersionsAction Action = "s3:ListBucketVersions"
ListBucketAction Action = "s3:ListBucket"
GetBucketObjectLockConfigurationAction Action = "s3:GetBucketObjectLockConfiguration"
PutBucketObjectLockConfigurationAction Action = "s3:PutBucketObjectLockConfiguration"
GetObjectLegalHoldAction Action = "s3:GetObjectLegalHold"
PutObjectLegalHoldAction Action = "s3:PutObjectLegalHold"
GetObjectRetentionAction Action = "s3:GetObjectRetention"
PutObjectRetentionAction Action = "s3:PutObjectRetention"
BypassGovernanceRetentionAction Action = "s3:BypassGovernanceRetention"
PutBucketOwnershipControlsAction Action = "s3:PutBucketOwnershipControls"
GetBucketOwnershipControlsAction Action = "s3:GetBucketOwnershipControls"
AllActions Action = "s3:*"
GetBucketAclAction Action = "s3:GetBucketAcl"
CreateBucketAction Action = "s3:CreateBucket"
PutBucketAclAction Action = "s3:PutBucketAcl"
DeleteBucketAction Action = "s3:DeleteBucket"
PutBucketVersioningAction Action = "s3:PutBucketVersioning"
GetBucketVersioningAction Action = "s3:GetBucketVersioning"
PutBucketPolicyAction Action = "s3:PutBucketPolicy"
GetBucketPolicyAction Action = "s3:GetBucketPolicy"
DeleteBucketPolicyAction Action = "s3:DeleteBucketPolicy"
AbortMultipartUploadAction Action = "s3:AbortMultipartUpload"
ListMultipartUploadPartsAction Action = "s3:ListMultipartUploadParts"
ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
PutObjectAction Action = "s3:PutObject"
GetObjectAction Action = "s3:GetObject"
DeleteObjectAction Action = "s3:DeleteObject"
GetObjectAclAction Action = "s3:GetObjectAcl"
GetObjectAttributesAction Action = "s3:GetObjectAttributes"
PutObjectAclAction Action = "s3:PutObjectAcl"
RestoreObjectAction Action = "s3:RestoreObject"
GetBucketTaggingAction Action = "s3:GetBucketTagging"
PutBucketTaggingAction Action = "s3:PutBucketTagging"
GetObjectTaggingAction Action = "s3:GetObjectTagging"
PutObjectTaggingAction Action = "s3:PutObjectTagging"
DeleteObjectTaggingAction Action = "s3:DeleteObjectTagging"
ListBucketVersionsAction Action = "s3:ListBucketVersions"
ListBucketAction Action = "s3:ListBucket"
AllActions Action = "s3:*"
)
var supportedActionList = map[Action]struct{}{
GetBucketAclAction: {},
CreateBucketAction: {},
PutBucketAclAction: {},
DeleteBucketAction: {},
PutBucketVersioningAction: {},
GetBucketVersioningAction: {},
PutBucketPolicyAction: {},
GetBucketPolicyAction: {},
DeleteBucketPolicyAction: {},
AbortMultipartUploadAction: {},
ListMultipartUploadPartsAction: {},
ListBucketMultipartUploadsAction: {},
PutObjectAction: {},
GetObjectAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
PutObjectAclAction: {},
RestoreObjectAction: {},
GetBucketTaggingAction: {},
PutBucketTaggingAction: {},
GetObjectTaggingAction: {},
PutObjectTaggingAction: {},
DeleteObjectTaggingAction: {},
ListBucketVersionsAction: {},
ListBucketAction: {},
PutBucketObjectLockConfigurationAction: {},
GetObjectLegalHoldAction: {},
PutObjectLegalHoldAction: {},
GetObjectRetentionAction: {},
PutObjectRetentionAction: {},
BypassGovernanceRetentionAction: {},
PutBucketOwnershipControlsAction: {},
GetBucketOwnershipControlsAction: {},
AllActions: {},
GetBucketAclAction: {},
CreateBucketAction: {},
PutBucketAclAction: {},
DeleteBucketAction: {},
PutBucketVersioningAction: {},
GetBucketVersioningAction: {},
PutBucketPolicyAction: {},
GetBucketPolicyAction: {},
DeleteBucketPolicyAction: {},
AbortMultipartUploadAction: {},
ListMultipartUploadPartsAction: {},
ListBucketMultipartUploadsAction: {},
PutObjectAction: {},
GetObjectAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
PutObjectAclAction: {},
RestoreObjectAction: {},
GetBucketTaggingAction: {},
PutBucketTaggingAction: {},
GetObjectTaggingAction: {},
PutObjectTaggingAction: {},
DeleteObjectTaggingAction: {},
ListBucketVersionsAction: {},
ListBucketAction: {},
AllActions: {},
}
var supportedObjectActionList = map[Action]struct{}{
AbortMultipartUploadAction: {},
ListMultipartUploadPartsAction: {},
PutObjectAction: {},
GetObjectAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
PutObjectAclAction: {},
RestoreObjectAction: {},
GetObjectTaggingAction: {},
PutObjectTaggingAction: {},
DeleteObjectTaggingAction: {},
GetObjectLegalHoldAction: {},
PutObjectLegalHoldAction: {},
GetObjectRetentionAction: {},
PutObjectRetentionAction: {},
BypassGovernanceRetentionAction: {},
AllActions: {},
AbortMultipartUploadAction: {},
ListMultipartUploadPartsAction: {},
PutObjectAction: {},
GetObjectAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
PutObjectAclAction: {},
RestoreObjectAction: {},
GetObjectTaggingAction: {},
PutObjectTaggingAction: {},
DeleteObjectTaggingAction: {},
AllActions: {},
}
// Validates Action: it should either wildcard match with supported actions list or be in it
func (a Action) IsValid() error {
if !strings.HasPrefix(string(a), "s3:") {
return errInvalidAction
return fmt.Errorf("invalid action: %v", a)
}
if a == AllActions {
@@ -137,39 +116,31 @@ func (a Action) IsValid() error {
}
}
return errInvalidAction
return fmt.Errorf("invalid wildcard usage: %v prefix is not in the supported actions list", pattern)
}
_, found := supportedActionList[a]
if !found {
return errInvalidAction
return fmt.Errorf("unsupported action: %v", a)
}
return nil
}
func getBoolPtr(bl bool) *bool {
return &bl
}
// Checks if the action is object action
// nil points to 's3:*'
func (a Action) IsObjectAction() *bool {
if a == AllActions {
return nil
}
func (a Action) IsObjectAction() bool {
if a[len(a)-1] == '*' {
pattern := strings.TrimSuffix(string(a), "*")
for act := range supportedObjectActionList {
if strings.HasPrefix(string(act), pattern) {
return getBoolPtr(true)
return true
}
}
return getBoolPtr(false)
return false
}
_, found := supportedObjectActionList[a]
return &found
return found
}
func (a Action) WildCardMatch(act Action) bool {
@@ -188,7 +159,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
var err error
if err = json.Unmarshal(data, &ss); err == nil {
if len(ss) == 0 {
return errInvalidAction
return fmt.Errorf("actions can't be empty")
}
*a = make(Actions)
for _, s := range ss {
@@ -201,7 +172,7 @@ func (a *Actions) UnmarshalJSON(data []byte) error {
var s string
if err = json.Unmarshal(data, &s); err == nil {
if s == "" {
return errInvalidAction
return fmt.Errorf("actions can't be empty")
}
*a = make(Actions)
err = a.Add(s)

View File

@@ -30,6 +30,5 @@ func (bpat BucketPolicyAccessType) Validate() error {
return nil
}
//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
return fmt.Errorf("Invalid effect: %v", bpat)
return fmt.Errorf("invalid effect: %v", bpat)
}

View File

@@ -16,6 +16,7 @@ package auth
import (
"encoding/json"
"fmt"
)
type Principals map[string]struct{}
@@ -27,50 +28,23 @@ func (p Principals) Add(key string) {
// Override UnmarshalJSON method to decode both []string and string properties
func (p *Principals) UnmarshalJSON(data []byte) error {
ss := []string{}
var s string
var k struct {
AWS string
}
var err error
if err = json.Unmarshal(data, &ss); err == nil {
if len(ss) == 0 {
return errInvalidPrincipal
return fmt.Errorf("principals can't be empty")
}
*p = make(Principals)
for _, s := range ss {
p.Add(s)
}
return nil
} else if err = json.Unmarshal(data, &s); err == nil {
if s == "" {
return errInvalidPrincipal
}
*p = make(Principals)
p.Add(s)
return nil
} else if err = json.Unmarshal(data, &k); err == nil {
if k.AWS == "" {
return errInvalidPrincipal
}
*p = make(Principals)
p.Add(k.AWS)
return nil
} else {
var sk struct {
AWS []string
}
if err = json.Unmarshal(data, &sk); err == nil {
if len(sk.AWS) == 0 {
return errInvalidPrincipal
var s string
if err = json.Unmarshal(data, &s); err == nil {
if s == "" {
return fmt.Errorf("principals can't be empty")
}
*p = make(Principals)
for _, s := range sk.AWS {
p.Add(s)
}
p.Add(s)
}
}
@@ -97,7 +71,7 @@ func (p Principals) Validate(iam IAMService) error {
if len(p) == 1 {
return nil
}
return errInvalidPrincipal
return fmt.Errorf("principals should either contain * or user access keys")
}
accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
@@ -105,7 +79,7 @@ func (p Principals) Validate(iam IAMService) error {
return err
}
if len(accs) > 0 {
return errInvalidPrincipal
return fmt.Errorf("user accounts don't exist: %v", accs)
}
return nil

View File

@@ -16,6 +16,7 @@ package auth
import (
"encoding/json"
"fmt"
"strings"
)
@@ -29,7 +30,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
var err error
if err = json.Unmarshal(data, &ss); err == nil {
if len(ss) == 0 {
return errInvalidResource
return fmt.Errorf("resources can't be empty")
}
*r = make(Resources)
for _, s := range ss {
@@ -42,7 +43,7 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
var s string
if err = json.Unmarshal(data, &s); err == nil {
if s == "" {
return errInvalidResource
return fmt.Errorf("resources can't be empty")
}
*r = make(Resources)
err = r.Add(s)
@@ -59,7 +60,12 @@ func (r *Resources) UnmarshalJSON(data []byte) error {
func (r Resources) Add(rc string) error {
ok, pattern := isValidResource(rc)
if !ok {
return errInvalidResource
return fmt.Errorf("invalid resource: %v", rc)
}
_, found := r[pattern]
if found {
return fmt.Errorf("duplicate resource: %v", rc)
}
r[pattern] = struct{}{}
@@ -93,7 +99,7 @@ func (r Resources) ContainsBucketPattern() bool {
func (r Resources) Validate(bucket string) error {
for resource := range r {
if !strings.HasPrefix(resource, bucket) {
return errInvalidResource
return fmt.Errorf("incorrect bucket name in %v", resource)
}
}

View File

@@ -30,30 +30,12 @@ const (
// Account is a gateway IAM account
type Account struct {
Access string `json:"access"`
Secret string `json:"secret"`
Role Role `json:"role"`
UserID int `json:"userID"`
GroupID int `json:"groupID"`
}
// Mutable props, which could be changed when updating an IAM account
type MutableProps struct {
Secret *string `json:"secret"`
UserID *int `json:"userID"`
GroupID *int `json:"groupID"`
}
func updateAcc(acc *Account, props MutableProps) {
if props.Secret != nil {
acc.Secret = *props.Secret
}
if props.GroupID != nil {
acc.GroupID = *props.GroupID
}
if props.UserID != nil {
acc.UserID = *props.UserID
}
Access string `json:"access"`
Secret string `json:"secret"`
Role Role `json:"role"`
UserID int `json:"userID"`
GroupID int `json:"groupID"`
ProjectID int `json:"projectID"`
}
// IAMService is the interface for all IAM service implementations
@@ -62,51 +44,33 @@ func updateAcc(acc *Account, props MutableProps) {
type IAMService interface {
CreateAccount(account Account) error
GetUserAccount(access string) (Account, error)
UpdateUserAccount(access string, props MutableProps) error
DeleteUserAccount(access string) error
ListUserAccounts() ([]Account, error)
Shutdown() error
}
var (
// ErrUserExists is returned when the user already exists
ErrUserExists = errors.New("user already exists")
// ErrNoSuchUser is returned when the user does not exist
ErrNoSuchUser = errors.New("user not found")
)
var ErrNoSuchUser = errors.New("user not found")
type Opts struct {
RootAccount Account
Dir string
LDAPServerURL string
LDAPBindDN string
LDAPPassword string
LDAPQueryBase string
LDAPObjClasses string
LDAPAccessAtr string
LDAPSecretAtr string
LDAPRoleAtr string
LDAPUserIdAtr string
LDAPGroupIdAtr string
VaultEndpointURL string
VaultSecretStoragePath string
VaultMountPath string
VaultRootToken string
VaultRoleId string
VaultRoleSecret string
VaultServerCert string
VaultClientCert string
VaultClientCertKey string
S3Access string
S3Secret string
S3Region string
S3Bucket string
S3Endpoint string
S3DisableSSlVerfiy bool
S3Debug bool
CacheDisable bool
CacheTTL int
CachePrune int
Dir string
LDAPServerURL string
LDAPBindDN string
LDAPPassword string
LDAPQueryBase string
LDAPObjClasses string
LDAPAccessAtr string
LDAPSecretAtr string
LDAPRoleAtr string
S3Access string
S3Secret string
S3Region string
S3Bucket string
S3Endpoint string
S3DisableSSlVerfiy bool
S3Debug bool
CacheDisable bool
CacheTTL int
CachePrune int
}
func New(o *Opts) (IAMService, error) {
@@ -115,23 +79,18 @@ func New(o *Opts) (IAMService, error) {
switch {
case o.Dir != "":
svc, err = NewInternal(o.RootAccount, o.Dir)
svc, err = NewInternal(o.Dir)
fmt.Printf("initializing internal IAM with %q\n", o.Dir)
case o.LDAPServerURL != "":
svc, err = NewLDAPService(o.RootAccount, o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
o.LDAPGroupIdAtr, o.LDAPObjClasses)
svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
o.LDAPObjClasses)
fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
case o.S3Endpoint != "":
svc, err = NewS3(o.RootAccount, o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
fmt.Printf("initializing S3 IAM with '%v/%v'\n",
o.S3Endpoint, o.S3Bucket)
case o.VaultEndpointURL != "":
svc, err = NewVaultIAMService(o.RootAccount, o.VaultEndpointURL, o.VaultSecretStoragePath,
o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
default:
// if no iam options selected, default to the single user mode
fmt.Println("No IAM service configured, enabling single account mode")

View File

@@ -66,21 +66,6 @@ func (i *icache) get(k string) (Account, bool) {
return v.value, true
}
func (i *icache) update(k string, props MutableProps) {
i.Lock()
defer i.Unlock()
item, found := i.items[k]
if found {
updateAcc(&item.value, props)
// refresh the expiration date
item.exp = time.Now().Add(i.expire)
i.items[k] = item
}
}
func (i *icache) Delete(k string) {
i.Lock()
delete(i.items, k)
@@ -181,16 +166,6 @@ func (c *IAMCache) DeleteUserAccount(access string) error {
return nil
}
func (c *IAMCache) UpdateUserAccount(access string, props MutableProps) error {
err := c.service.UpdateUserAccount(access, props)
if err != nil {
return err
}
c.iamcache.update(access, props)
return nil
}
// ListUserAccounts is a passthrough to the underlying service and
// does not make use of the cache
func (c *IAMCache) ListUserAccounts() ([]Account, error) {

View File

@@ -22,7 +22,6 @@ import (
"os"
"path/filepath"
"sort"
"sync"
"time"
)
@@ -33,15 +32,7 @@ const (
// IAMServiceInternal manages the internal IAM service
type IAMServiceInternal struct {
// This mutex will help with racing updates to the IAM data
// from multiple requests to this gateway instance, but
// will not help with racing updates to multiple load balanced
// gateway instances. This is a limitation of the internal
// IAM service. All account updates should be sent to a single
// gateway instance if possible.
sync.RWMutex
dir string
rootAcc Account
dir string
}
// UpdateAcctFunc accepts the current data and returns the new data to be stored
@@ -55,10 +46,9 @@ type iAMConfig struct {
var _ IAMService = &IAMServiceInternal{}
// NewInternal creates a new instance for the Internal IAM service
func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
func NewInternal(dir string) (*IAMServiceInternal, error) {
i := &IAMServiceInternal{
dir: dir,
rootAcc: rootAcc,
dir: dir,
}
err := i.initIAM()
@@ -72,13 +62,6 @@ func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
// CreateAccount creates a new IAM account. Returns an error if the account
// already exists.
func (s *IAMServiceInternal) CreateAccount(account Account) error {
if account.Access == s.rootAcc.Access {
return ErrUserExists
}
s.Lock()
defer s.Unlock()
return s.storeIAM(func(data []byte) ([]byte, error) {
conf, err := parseIAM(data)
if err != nil {
@@ -87,7 +70,7 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
_, ok := conf.AccessAccounts[account.Access]
if ok {
return nil, ErrUserExists
return nil, fmt.Errorf("account already exists")
}
conf.AccessAccounts[account.Access] = account
@@ -103,13 +86,6 @@ func (s *IAMServiceInternal) CreateAccount(account Account) error {
// GetUserAccount retrieves account info for the requested user. Returns
// ErrNoSuchUser if the account does not exist.
func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
if access == s.rootAcc.Access {
return s.rootAcc, nil
}
s.RLock()
defer s.RUnlock()
conf, err := s.getIAM()
if err != nil {
return Account{}, fmt.Errorf("get iam data: %w", err)
@@ -123,41 +99,9 @@ func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
return acct, nil
}
// UpdateUserAccount updates the specified user account fields. Returns
// ErrNoSuchUser if the account does not exist.
func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error {
s.Lock()
defer s.Unlock()
return s.storeIAM(func(data []byte) ([]byte, error) {
conf, err := parseIAM(data)
if err != nil {
return nil, fmt.Errorf("get iam data: %w", err)
}
acc, found := conf.AccessAccounts[access]
if !found {
return nil, ErrNoSuchUser
}
updateAcc(&acc, props)
conf.AccessAccounts[access] = acc
b, err := json.Marshal(conf)
if err != nil {
return nil, fmt.Errorf("failed to serialize iam: %w", err)
}
return b, nil
})
}
// DeleteUserAccount deletes the specified user account. Does not check if
// account exists.
func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
s.Lock()
defer s.Unlock()
return s.storeIAM(func(data []byte) ([]byte, error) {
conf, err := parseIAM(data)
if err != nil {
@@ -177,9 +121,6 @@ func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
// ListUserAccounts lists all the user accounts stored.
func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
s.RLock()
defer s.RUnlock()
conf, err := s.getIAM()
if err != nil {
return []Account{}, fmt.Errorf("get iam data: %w", err)
@@ -194,11 +135,12 @@ func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
var accs []Account
for _, k := range keys {
accs = append(accs, Account{
Access: k,
Secret: conf.AccessAccounts[k].Secret,
Role: conf.AccessAccounts[k].Role,
UserID: conf.AccessAccounts[k].UserID,
GroupID: conf.AccessAccounts[k].GroupID,
Access: k,
Secret: conf.AccessAccounts[k].Secret,
Role: conf.AccessAccounts[k].Role,
UserID: conf.AccessAccounts[k].UserID,
GroupID: conf.AccessAccounts[k].GroupID,
ProjectID: conf.AccessAccounts[k].ProjectID,
})
}
@@ -247,10 +189,6 @@ func parseIAM(b []byte) (iAMConfig, error) {
return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
}
if conf.AccessAccounts == nil {
conf.AccessAccounts = make(map[string]Account)
}
return conf, nil
}

View File

@@ -1,22 +1,7 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"fmt"
"strconv"
"strings"
"github.com/go-ldap/ldap/v3"
@@ -29,19 +14,15 @@ type LdapIAMService struct {
accessAtr string
secretAtr string
roleAtr string
groupIdAtr string
userIdAtr string
rootAcc Account
}
var _ IAMService = &LdapIAMService{}
func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
return nil, fmt.Errorf("required parameters list not fully provided")
}
conn, err := ldap.DialURL(url)
conn, err := ldap.Dial("tcp", url)
if err != nil {
return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
}
@@ -57,23 +38,15 @@ func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAt
accessAtr: accAtr,
secretAtr: secAtr,
roleAtr: roleAtr,
userIdAtr: userIdAtr,
groupIdAtr: groupIdAtr,
rootAcc: rootAcc,
}, nil
}
func (ld *LdapIAMService) CreateAccount(account Account) error {
if ld.rootAcc.Access == account.Access {
return ErrUserExists
}
userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
userEntry.Attribute("objectClass", ld.objClasses)
userEntry.Attribute(ld.accessAtr, []string{account.Access})
userEntry.Attribute(ld.secretAtr, []string{account.Secret})
userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
userEntry.Attribute(ld.groupIdAtr, []string{fmt.Sprint(account.GroupID)})
userEntry.Attribute(ld.userIdAtr, []string{fmt.Sprint(account.UserID)})
err := ld.conn.Add(userEntry)
if err != nil {
@@ -84,9 +57,6 @@ func (ld *LdapIAMService) CreateAccount(account Account) error {
}
func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
if access == ld.rootAcc.Access {
return ld.rootAcc, nil
}
searchRequest := ldap.NewSearchRequest(
ld.queryBase,
ldap.ScopeWholeSubtree,
@@ -95,7 +65,7 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
0,
false,
fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.userIdAtr, ld.groupIdAtr},
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
nil,
)
@@ -104,48 +74,14 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
return Account{}, err
}
if len(result.Entries) == 0 {
return Account{}, ErrNoSuchUser
}
entry := result.Entries[0]
groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
if err != nil {
return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.groupIdAtr))
}
userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
if err != nil {
return Account{}, fmt.Errorf("invalid entry value for group-id: %v", entry.GetAttributeValue(ld.userIdAtr))
}
return Account{
Access: entry.GetAttributeValue(ld.accessAtr),
Secret: entry.GetAttributeValue(ld.secretAtr),
Role: Role(entry.GetAttributeValue(ld.roleAtr)),
GroupID: groupId,
UserID: userId,
Access: entry.GetAttributeValue(ld.accessAtr),
Secret: entry.GetAttributeValue(ld.secretAtr),
Role: Role(entry.GetAttributeValue(ld.roleAtr)),
}, nil
}
func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error {
req := ldap.NewModifyRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
if props.Secret != nil {
req.Replace(ld.secretAtr, []string{*props.Secret})
}
if props.GroupID != nil {
req.Replace(ld.groupIdAtr, []string{fmt.Sprint(*props.GroupID)})
}
if props.UserID != nil {
req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
}
err := ld.conn.Modify(req)
//TODO: Handle non existing user case
if err != nil {
return err
}
return nil
}
func (ld *LdapIAMService) DeleteUserAccount(access string) error {
delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
@@ -170,7 +106,7 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
0,
false,
fmt.Sprintf("(&%v)", searchFilter),
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.groupIdAtr, ld.userIdAtr},
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
nil,
)
@@ -181,20 +117,10 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
result := []Account{}
for _, el := range resp.Entries {
groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
if err != nil {
return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.groupIdAtr))
}
userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
if err != nil {
return nil, fmt.Errorf("invalid entry value for group-id: %v", el.GetAttributeValue(ld.userIdAtr))
}
result = append(result, Account{
Access: el.GetAttributeValue(ld.accessAtr),
Secret: el.GetAttributeValue(ld.secretAtr),
Role: Role(el.GetAttributeValue(ld.roleAtr)),
GroupID: groupId,
UserID: userId,
Access: el.GetAttributeValue(ld.accessAtr),
Secret: el.GetAttributeValue(ld.secretAtr),
Role: Role(el.GetAttributeValue(ld.roleAtr)),
})
}

View File

@@ -24,7 +24,6 @@ import (
"io"
"net/http"
"sort"
"sync"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
@@ -42,14 +41,6 @@ import (
// coming from iAMConfig and iamFile in iam_internal.
type IAMServiceS3 struct {
// This mutex will help with racing updates to the IAM data
// from multiple requests to this gateway instance, but
// will not help with racing updates to multiple load balanced
// gateway instances. This is a limitation of the internal
// IAM service. All account updates should be sent to a single
// gateway instance if possible.
sync.RWMutex
access string
secret string
region string
@@ -57,13 +48,12 @@ type IAMServiceS3 struct {
endpoint string
sslSkipVerify bool
debug bool
rootAcc Account
client *s3.Client
}
var _ IAMService = &IAMServiceS3{}
func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
if access == "" {
return nil, fmt.Errorf("must provide s3 IAM service access key")
}
@@ -88,7 +78,6 @@ func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, ssl
endpoint: endpoint,
sslSkipVerify: sslSkipVerify,
debug: debug,
rootAcc: rootAcc,
}
cfg, err := i.getConfig()
@@ -96,25 +85,11 @@ func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, ssl
return nil, fmt.Errorf("init s3 IAM: %v", err)
}
if endpoint != "" {
i.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = &endpoint
})
return i, nil
}
i.client = s3.NewFromConfig(cfg)
return i, nil
}
func (s *IAMServiceS3) CreateAccount(account Account) error {
if s.rootAcc.Access == account.Access {
return ErrUserExists
}
s.Lock()
defer s.Unlock()
conf, err := s.getAccounts()
if err != nil {
return err
@@ -122,7 +97,7 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
_, ok := conf.AccessAccounts[account.Access]
if ok {
return ErrUserExists
return fmt.Errorf("account already exists")
}
conf.AccessAccounts[account.Access] = account
@@ -130,13 +105,6 @@ func (s *IAMServiceS3) CreateAccount(account Account) error {
}
func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
if access == s.rootAcc.Access {
return s.rootAcc, nil
}
s.RLock()
defer s.RUnlock()
conf, err := s.getAccounts()
if err != nil {
return Account{}, err
@@ -150,30 +118,7 @@ func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
return acct, nil
}
func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error {
s.Lock()
defer s.Unlock()
conf, err := s.getAccounts()
if err != nil {
return err
}
acc, ok := conf.AccessAccounts[access]
if !ok {
return ErrNoSuchUser
}
updateAcc(&acc, props)
conf.AccessAccounts[access] = acc
return s.storeAccts(conf)
}
func (s *IAMServiceS3) DeleteUserAccount(access string) error {
s.Lock()
defer s.Unlock()
conf, err := s.getAccounts()
if err != nil {
return err
@@ -189,9 +134,6 @@ func (s *IAMServiceS3) DeleteUserAccount(access string) error {
}
func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
s.RLock()
defer s.RUnlock()
conf, err := s.getAccounts()
if err != nil {
return nil, err
@@ -206,17 +148,28 @@ func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
var accs []Account
for _, k := range keys {
accs = append(accs, Account{
Access: k,
Secret: conf.AccessAccounts[k].Secret,
Role: conf.AccessAccounts[k].Role,
UserID: conf.AccessAccounts[k].UserID,
GroupID: conf.AccessAccounts[k].GroupID,
Access: k,
Secret: conf.AccessAccounts[k].Secret,
Role: conf.AccessAccounts[k].Role,
UserID: conf.AccessAccounts[k].UserID,
GroupID: conf.AccessAccounts[k].GroupID,
ProjectID: conf.AccessAccounts[k].ProjectID,
})
}
return accs, nil
}
// ResolveEndpoint is used for on prem or non-aws endpoints
func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
URL: s.endpoint,
SigningRegion: s.region,
HostnameImmutable: true,
}, nil
}
func (s *IAMServiceS3) Shutdown() error {
return nil
}
@@ -235,6 +188,11 @@ func (s *IAMServiceS3) getConfig() (aws.Config, error) {
config.WithHTTPClient(client),
}
if s.endpoint != "" {
opts = append(opts,
config.WithEndpointResolverWithOptions(s))
}
if s.debug {
opts = append(opts,
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
@@ -252,15 +210,15 @@ func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
})
if err != nil {
// if the error is object not exists,
// init empty accounts struct and return that
// init empty accounts stuct and return that
var nsk *types.NoSuchKey
if errors.As(err, &nsk) {
return iAMConfig{AccessAccounts: map[string]Account{}}, nil
return iAMConfig{}, nil
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
if apiErr.ErrorCode() == "NotFound" {
return iAMConfig{AccessAccounts: map[string]Account{}}, nil
return iAMConfig{}, nil
}
}

View File

@@ -32,12 +32,7 @@ func (IAMServiceSingle) CreateAccount(account Account) error {
// GetUserAccount no accounts in single tenant mode
func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
return Account{}, ErrNoSuchUser
}
// UpdateUserAccount no accounts in single tenant mode
func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
return ErrNotSupported
return Account{}, ErrNotSupported
}
// DeleteUserAccount no accounts in single tenant mode

View File

@@ -1,256 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
vault "github.com/hashicorp/vault-client-go"
"github.com/hashicorp/vault-client-go/schema"
)
type VaultIAMService struct {
client *vault.Client
reqOpts []vault.RequestOption
secretStoragePath string
rootAcc Account
}
var _ IAMService = &VaultIAMService{}
func NewVaultIAMService(rootAcc Account, endpoint, secretStoragePath, mountPath, rootToken, roleID, roleSecret, serverCert, clientCert, clientCertKey string) (IAMService, error) {
opts := []vault.ClientOption{
vault.WithAddress(endpoint),
// set request timeout to 10 secs
vault.WithRequestTimeout(10 * time.Second),
}
if serverCert != "" {
tls := vault.TLSConfiguration{}
tls.ServerCertificate.FromBytes = []byte(serverCert)
if clientCert != "" {
if clientCertKey == "" {
return nil, fmt.Errorf("client certificate and client certificate should both be specified")
}
tls.ClientCertificate.FromBytes = []byte(clientCert)
tls.ClientCertificateKey.FromBytes = []byte(clientCertKey)
}
opts = append(opts, vault.WithTLS(tls))
}
client, err := vault.New(opts...)
if err != nil {
return nil, fmt.Errorf("init vault client: %w", err)
}
reqOpts := []vault.RequestOption{}
// if mount path is not specified, it defaults to "approle"
if mountPath != "" {
reqOpts = append(reqOpts, vault.WithMountPath(mountPath))
}
// Authentication
switch {
case rootToken != "":
err := client.SetToken(rootToken)
if err != nil {
return nil, fmt.Errorf("root token authentication failure: %w", err)
}
case roleID != "":
if roleSecret == "" {
return nil, fmt.Errorf("role id and role secret must both be specified")
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
resp, err := client.Auth.AppRoleLogin(ctx, schema.AppRoleLoginRequest{
RoleId: roleID,
SecretId: roleSecret,
}, reqOpts...)
cancel()
if err != nil {
return nil, fmt.Errorf("approle authentication failure: %w", err)
}
if err := client.SetToken(resp.Auth.ClientToken); err != nil {
return nil, fmt.Errorf("approle authentication set token failure: %w", err)
}
default:
return nil, fmt.Errorf("vault authentication requires either roleid/rolesecret or root token")
}
return &VaultIAMService{
client: client,
reqOpts: reqOpts,
secretStoragePath: secretStoragePath,
rootAcc: rootAcc,
}, nil
}
func (vt *VaultIAMService) CreateAccount(account Account) error {
if vt.rootAcc.Access == account.Access {
return ErrUserExists
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
_, err := vt.client.Secrets.KvV2Write(ctx, vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
Data: map[string]any{
account.Access: account,
},
Options: map[string]interface{}{
"cas": 0,
},
}, vt.reqOpts...)
cancel()
if err != nil {
if strings.Contains(err.Error(), "check-and-set") {
return ErrUserExists
}
return err
}
return nil
}
func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
if vt.rootAcc.Access == access {
return vt.rootAcc, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
resp, err := vt.client.Secrets.KvV2Read(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
cancel()
if err != nil {
return Account{}, err
}
acc, err := parseVaultUserAccount(resp.Data.Data, access)
if err != nil {
return Account{}, err
}
return acc, nil
}
func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error {
//TODO: We need something like a transaction here ?
acc, err := vt.GetUserAccount(access)
if err != nil {
return err
}
updateAcc(&acc, props)
err = vt.DeleteUserAccount(access)
if err != nil {
return err
}
err = vt.CreateAccount(acc)
if err != nil {
return err
}
return nil
}
func (vt *VaultIAMService) DeleteUserAccount(access string) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
_, err := vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(ctx, vt.secretStoragePath+"/"+access, vt.reqOpts...)
cancel()
if err != nil {
return err
}
return nil
}
func (vt *VaultIAMService) ListUserAccounts() ([]Account, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
resp, err := vt.client.Secrets.KvV2List(ctx, vt.secretStoragePath, vt.reqOpts...)
cancel()
if err != nil {
if vault.IsErrorStatus(err, 404) {
return []Account{}, nil
}
return nil, err
}
accs := []Account{}
for _, acss := range resp.Data.Keys {
acc, err := vt.GetUserAccount(acss)
if err != nil {
return nil, err
}
accs = append(accs, acc)
}
return accs, nil
}
// the client doesn't have explicit shutdown, as it uses http.Client
func (vt *VaultIAMService) Shutdown() error {
return nil
}
var errInvalidUser error = errors.New("invalid user account entry in secrets engine")
func parseVaultUserAccount(data map[string]interface{}, access string) (acc Account, err error) {
usrAcc, ok := data[access].(map[string]interface{})
if !ok {
return acc, errInvalidUser
}
acss, ok := usrAcc["access"].(string)
if !ok {
return acc, errInvalidUser
}
secret, ok := usrAcc["secret"].(string)
if !ok {
return acc, errInvalidUser
}
role, ok := usrAcc["role"].(string)
if !ok {
return acc, errInvalidUser
}
userIdJson, ok := usrAcc["userID"].(json.Number)
if !ok {
return acc, errInvalidUser
}
userId, err := userIdJson.Int64()
if err != nil {
return acc, errInvalidUser
}
groupIdJson, ok := usrAcc["groupID"].(json.Number)
if !ok {
return acc, errInvalidUser
}
groupId, err := groupIdJson.Int64()
if err != nil {
return acc, errInvalidUser
}
return Account{
Access: acss,
Secret: secret,
Role: Role(role),
UserID: int(userId),
GroupID: int(groupId),
}, nil
}

View File

@@ -1,258 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"context"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
)
type BucketLockConfig struct {
Enabled bool
DefaultRetention *types.DefaultRetention
CreatedAt *time.Time
}
func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
var lockConfig types.ObjectLockConfiguration
if err := xml.Unmarshal(input, &lockConfig); err != nil {
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
if lockConfig.ObjectLockEnabled != "" && lockConfig.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
config := BucketLockConfig{
Enabled: lockConfig.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
}
if lockConfig.Rule != nil && lockConfig.Rule.DefaultRetention != nil {
retention := lockConfig.Rule.DefaultRetention
if retention.Mode != types.ObjectLockRetentionModeCompliance && retention.Mode != types.ObjectLockRetentionModeGovernance {
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
if retention.Years != nil && retention.Days != nil {
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
if retention.Days != nil && *retention.Days <= 0 {
return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
}
if retention.Years != nil && *retention.Years <= 0 {
return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
}
config.DefaultRetention = retention
now := time.Now()
config.CreatedAt = &now
}
return json.Marshal(config)
}
func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfiguration, error) {
var config BucketLockConfig
if err := json.Unmarshal(input, &config); err != nil {
return nil, fmt.Errorf("parse object lock config: %w", err)
}
result := &types.ObjectLockConfiguration{
Rule: &types.ObjectLockRule{
DefaultRetention: config.DefaultRetention,
},
}
if config.Enabled {
result.ObjectLockEnabled = types.ObjectLockEnabledEnabled
}
return result, nil
}
func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
var retention types.ObjectLockRetention
if err := xml.Unmarshal(input, &retention); err != nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
if retention.RetainUntilDate == nil || retention.RetainUntilDate.Before(time.Now()) {
return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
}
switch retention.Mode {
case types.ObjectLockRetentionModeCompliance:
case types.ObjectLockRetentionModeGovernance:
default:
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
return json.Marshal(retention)
}
func ParseObjectLockRetentionOutput(input []byte) (*types.ObjectLockRetention, error) {
var retention types.ObjectLockRetention
if err := json.Unmarshal(input, &retention); err != nil {
return nil, fmt.Errorf("parse object lock retention: %w", err)
}
return &retention, nil
}
func ParseObjectLegalHoldOutput(status *bool) *types.ObjectLockLegalHold {
if status == nil {
return nil
}
if *status {
return &types.ObjectLockLegalHold{
Status: types.ObjectLockLegalHoldStatusOn,
}
}
return &types.ObjectLockLegalHold{
Status: types.ObjectLockLegalHoldStatusOff,
}
}
func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []string, bypass bool, be backend.Backend) error {
data, err := be.GetObjectLockConfiguration(ctx, bucket)
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
return nil
}
return err
}
var bucketLockConfig BucketLockConfig
if err := json.Unmarshal(data, &bucketLockConfig); err != nil {
return fmt.Errorf("parse object lock config: %w", err)
}
if !bucketLockConfig.Enabled {
return nil
}
checkDefaultRetention := false
if bucketLockConfig.DefaultRetention != nil && bucketLockConfig.CreatedAt != nil {
expirationDate := *bucketLockConfig.CreatedAt
if bucketLockConfig.DefaultRetention.Days != nil {
expirationDate = expirationDate.AddDate(0, 0, int(*bucketLockConfig.DefaultRetention.Days))
}
if bucketLockConfig.DefaultRetention.Years != nil {
expirationDate = expirationDate.AddDate(int(*bucketLockConfig.DefaultRetention.Years), 0, 0)
}
if expirationDate.After(time.Now()) {
checkDefaultRetention = true
}
}
for _, obj := range objects {
checkRetention := true
retentionData, err := be.GetObjectRetention(ctx, bucket, obj, "")
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
continue
}
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
checkRetention = false
}
if err != nil && checkRetention {
return err
}
if checkRetention {
retention, err := ParseObjectLockRetentionOutput(retentionData)
if err != nil {
return err
}
if retention.Mode != "" && retention.RetainUntilDate != nil {
if retention.RetainUntilDate.After(time.Now()) {
switch retention.Mode {
case types.ObjectLockRetentionModeGovernance:
if !bypass {
return s3err.GetAPIError(s3err.ErrObjectLocked)
} else {
policy, err := be.GetBucketPolicy(ctx, bucket)
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
if err != nil {
return err
}
err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
if err != nil {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
}
case types.ObjectLockRetentionModeCompliance:
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
}
}
}
checkLegalHold := true
status, err := be.GetObjectLegalHold(ctx, bucket, obj, "")
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
checkLegalHold = false
} else {
return err
}
}
if checkLegalHold && *status {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
if checkDefaultRetention {
switch bucketLockConfig.DefaultRetention.Mode {
case types.ObjectLockRetentionModeGovernance:
if !bypass {
return s3err.GetAPIError(s3err.ErrObjectLocked)
} else {
policy, err := be.GetBucketPolicy(ctx, bucket)
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
if err != nil {
return err
}
err = VerifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
if err != nil {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
}
case types.ObjectLockRetentionModeCompliance:
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
}
}
return nil
}

View File

@@ -87,13 +87,13 @@ func TestStandaloneSign(t *testing.T) {
actual := req.Header.Get("Authorization")
if e, a := c.ExpSig, actual; e != a {
t.Errorf("expected %v, but received %v", e, a)
t.Errorf("expected %v, but recieved %v", e, a)
}
if e, a := c.OrigURI, req.URL.Path; e != a {
t.Errorf("expected %v, but received %v", e, a)
t.Errorf("expected %v, but recieved %v", e, a)
}
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
t.Errorf("expected %v, but received %v", e, a)
t.Errorf("expected %v, but recieved %v", e, a)
}
}
}
@@ -127,13 +127,13 @@ func TestStandaloneSign_RawPath(t *testing.T) {
actual := req.Header.Get("Authorization")
if e, a := c.ExpSig, actual; e != a {
t.Errorf("expected %v, but received %v", e, a)
t.Errorf("expected %v, but recieved %v", e, a)
}
if e, a := c.OrigURI, req.URL.Path; e != a {
t.Errorf("expected %v, but received %v", e, a)
t.Errorf("expected %v, but recieved %v", e, a)
}
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
t.Errorf("expected %v, but received %v", e, a)
t.Errorf("expected %v, but recieved %v", e, a)
}
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -18,9 +18,9 @@ import (
"bufio"
"context"
"fmt"
"io"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
"github.com/versity/versitygw/s3select"
@@ -43,12 +43,9 @@ type Backend interface {
PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
DeleteBucketPolicy(_ context.Context, bucket string) error
PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
DeleteBucketOwnershipControls(_ context.Context, bucket string) error
// multipart operations
CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
@@ -59,12 +56,12 @@ type Backend interface {
// standard object operations
PutObject(context.Context, *s3.PutObjectInput) (string, error)
HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
DeleteObject(context.Context, *s3.DeleteObjectInput) error
DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
@@ -84,16 +81,8 @@ type Backend interface {
PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
DeleteObjectTagging(_ context.Context, bucket, object string) error
// object lock operations
PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error
GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error)
PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error
GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error)
PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error
GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error)
// non AWS actions
ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error
ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
}
@@ -141,18 +130,9 @@ func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]b
func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
return types.ObjectOwnershipBucketOwnerEnforced, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -179,23 +159,23 @@ func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string
func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
return s3response.GetObjectAttributesResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -249,26 +229,7 @@ func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectRetention(_ context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket string, acl []byte) error {
func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {

View File

@@ -18,9 +18,7 @@ import (
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"net/http"
"os"
"io/fs"
"strconv"
"strings"
"time"
@@ -30,6 +28,11 @@ import (
"github.com/versity/versitygw/s3response"
)
var (
// RFC3339TimeFormat RFC3339 time format
RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
)
func IsValidBucketName(name string) bool { return true }
type ByBucketName []s3response.ListAllMyBucketsEntry
@@ -58,9 +61,9 @@ var (
// ParseRange parses input range header and returns startoffset, length, and
// error. If no endoffset specified, then length is set to -1.
func ParseRange(size int64, acceptRange string) (int64, int64, error) {
func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
if acceptRange == "" {
return 0, size, nil
return 0, fi.Size(), nil
}
rangeKv := strings.Split(acceptRange, "=")
@@ -89,21 +92,13 @@ func ParseRange(size int64, acceptRange string) (int64, int64, error) {
return 0, 0, errInvalidRange
}
if endOffset <= startOffset {
if endOffset < startOffset {
return 0, 0, errInvalidRange
}
return startOffset, endOffset - startOffset + 1, nil
}
func CreateExceedingRangeErr(objSize int64) s3err.APIError {
return s3err.APIError{
Code: "InvalidArgument",
Description: fmt.Sprintf("Range specified is not valid for source object of size: %d", objSize),
HTTPStatusCode: http.StatusBadRequest,
}
}
func GetMultipartMD5(parts []types.CompletedPart) string {
var partsEtagBytes []byte
for _, part := range parts {
@@ -125,16 +120,3 @@ func md5String(data []byte) string {
sum := md5.Sum(data)
return hex.EncodeToString(sum[:])
}
type FileSectionReadCloser struct {
R io.Reader
F *os.File
}
func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
return f.R.Read(p)
}
func (f *FileSectionReadCloser) Close() error {
return f.F.Close()
}

View File

@@ -1,40 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package meta
// MetadataStorer defines the interface for managing metadata.
// When object == "", the operation is on the bucket.
type MetadataStorer interface {
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
// Returns the value of the attribute, or an error if the attribute does not exist.
RetrieveAttribute(bucket, object, attribute string) ([]byte, error)
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
// If attribute already exists, new attribute should replace existing.
// Returns an error if the operation fails.
StoreAttribute(bucket, object, attribute string, value []byte) error
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
// Returns an error if the operation fails.
DeleteAttribute(bucket, object, attribute string) error
// ListAttributes lists all attributes for an object or a bucket.
// Returns list of attribute names, or an error if the operation fails.
ListAttributes(bucket, object string) ([]string, error)
// DeleteAttributes removes all attributes for an object or a bucket.
// Returns an error if the operation fails.
DeleteAttributes(bucket, object string) error
}

View File

@@ -1,101 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package meta
import (
"errors"
"fmt"
"path/filepath"
"strings"
"syscall"
"github.com/pkg/xattr"
)
const (
xattrPrefix = "user."
)
var (
// ErrNoSuchKey is returned when the key does not exist.
ErrNoSuchKey = errors.New("no such key")
)
type XattrMeta struct{}
// RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
func (x XattrMeta) RetrieveAttribute(bucket, object, attribute string) ([]byte, error) {
b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
if errors.Is(err, xattr.ENOATTR) {
return nil, ErrNoSuchKey
}
return b, err
}
// StoreAttribute stores the value of a specific attribute for an object in a bucket.
func (x XattrMeta) StoreAttribute(bucket, object, attribute string, value []byte) error {
return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
}
// DeleteAttribute removes the value of a specific attribute for an object in a bucket.
func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
err := xattr.Remove(filepath.Join(bucket, object), xattrPrefix+attribute)
if errors.Is(err, xattr.ENOATTR) {
return ErrNoSuchKey
}
return err
}
// DeleteAttributes is not implemented for xattr since xattrs
// are automatically removed when the file is deleted.
func (x XattrMeta) DeleteAttributes(bucket, object string) error {
return nil
}
// ListAttributes lists all attributes for an object in a bucket.
func (x XattrMeta) ListAttributes(bucket, object string) ([]string, error) {
attrs, err := xattr.List(filepath.Join(bucket, object))
if err != nil {
return nil, err
}
attributes := make([]string, 0, len(attrs))
for _, attr := range attrs {
if !isUserAttr(attr) {
continue
}
attributes = append(attributes, strings.TrimPrefix(attr, xattrPrefix))
}
return attributes, nil
}
func isUserAttr(attr string) bool {
return strings.HasPrefix(attr, xattrPrefix)
}
// Test is a helper function to test if xattrs are supported.
func (x XattrMeta) Test(path string) error {
// check for platform support
if !xattr.XATTR_SUPPORTED {
return fmt.Errorf("xattrs are not supported on this platform")
}
// check if the filesystem supports xattrs
_, err := xattr.Get(path, "user.test")
if errors.Is(err, syscall.ENOTSUP) {
return fmt.Errorf("xattrs are not supported on this filesystem")
}
return nil
}

View File

@@ -1,7 +1,6 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Copyright 2024 Versity Software
// MkdirAll borrowed from stdlib to add ability to set ownership
// as directories are created
@@ -30,7 +29,7 @@ var (
// Any newly created directory is set to provided uid/gid ownership.
// If path is already a directory, MkdirAll does nothing
// and returns nil.
// Any directory created will be set to provided uid/gid ownership
// Any directoy created will be set to provided uid/gid ownership
// if doChown is true.
func MkdirAll(path string, uid, gid int, doChown bool) error {
// Fast path: if we can tell whether path is a directory or file, stop with success or error.

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,24 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build !freebsd && !openbsd && !netbsd
// +build !freebsd,!openbsd,!netbsd
package posix
import "syscall"
var (
errNoData = syscall.ENODATA
)

View File

@@ -50,7 +50,7 @@ var (
defaultFilePerm uint32 = 0644
)
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool) (*tmpfile, error) {
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
uid, gid, doChown := p.getChownIDs(acct)
// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
@@ -81,7 +81,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
gid: gid,
}
// falloc is best effort, its fine if this fails
if size > 0 && dofalloc {
if size > 0 {
tmp.falloc()
}
@@ -111,7 +111,7 @@ func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Accou
}
// falloc is best effort, its fine if this fails
if size > 0 && dofalloc {
if size > 0 {
tmp.falloc()
}
@@ -221,7 +221,3 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
func (tmp *tmpfile) cleanup() {
tmp.f.Close()
}
func (tmp *tmpfile) File() *os.File {
return tmp.f
}

View File

@@ -0,0 +1,24 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build freebsd || openbsd || netbsd
// +build freebsd openbsd netbsd
package posix
import "syscall"
var (
errNoData = syscall.ENOATTR
)

View File

@@ -36,7 +36,7 @@ type tmpfile struct {
size int64
}
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool) (*tmpfile, error) {
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
uid, gid, doChown := p.getChownIDs(acct)
// Create a temp file for upload while in progress (see link comments below).
@@ -112,7 +112,3 @@ func (tmp *tmpfile) Write(b []byte) (int, error) {
func (tmp *tmpfile) cleanup() {
tmp.f.Close()
}
func (tmp *tmpfile) File() *os.File {
return tmp.f
}

View File

@@ -33,12 +33,6 @@ func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
return nil, err
}
if s.endpoint != "" {
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = &s.endpoint
}), nil
}
return s3.NewFromConfig(cfg), nil
}
@@ -56,6 +50,11 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
config.WithHTTPClient(client),
}
if s.endpoint != "" {
opts = append(opts,
config.WithEndpointResolverWithOptions(s))
}
if s.disableChecksum {
opts = append(opts,
config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
@@ -68,3 +67,13 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
return config.LoadDefaultConfig(ctx, opts...)
}
// ResolveEndpoint is used for on prem or non-aws endpoints
func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
URL: s.endpoint,
SigningRegion: s.awsRegion,
HostnameImmutable: true,
}, nil
}

View File

@@ -33,7 +33,6 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
@@ -55,8 +54,6 @@ type S3Proxy struct {
debug bool
}
var _ backend.Backend = &S3Proxy{}
func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
s := &S3Proxy{
access: access,
@@ -130,48 +127,9 @@ func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput)
return handleError(err)
}
func (s *S3Proxy) PutBucketOwnershipControls(ctx context.Context, bucket string, ownership types.ObjectOwnership) error {
_, err := s.client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{
Bucket: &bucket,
OwnershipControls: &types.OwnershipControls{
Rules: []types.OwnershipControlsRule{
{
ObjectOwnership: ownership,
},
},
},
})
return handleError(err)
}
func (s *S3Proxy) GetBucketOwnershipControls(ctx context.Context, bucket string) (types.ObjectOwnership, error) {
var ownship types.ObjectOwnership
resp, err := s.client.GetBucketOwnershipControls(ctx, &s3.GetBucketOwnershipControlsInput{
Bucket: &bucket,
})
if err != nil {
return ownship, handleError(err)
}
return resp.OwnershipControls.Rules[0].ObjectOwnership, nil
}
func (s *S3Proxy) DeleteBucketOwnershipControls(ctx context.Context, bucket string) error {
_, err := s.client.DeleteBucketOwnershipControls(ctx, &s3.DeleteBucketOwnershipControlsInput{
Bucket: &bucket,
})
return handleError(err)
}
func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
out, err := s.client.CreateMultipartUpload(ctx, input)
if err != nil {
return s3response.InitiateMultipartUploadResult{}, handleError(err)
}
return s3response.InitiateMultipartUploadResult{
Bucket: *out.Bucket,
Key: *out.Key,
UploadId: *out.UploadId,
}, nil
return out, handleError(err)
}
func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
@@ -203,8 +161,8 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
ID: *u.Owner.ID,
DisplayName: *u.Owner.DisplayName,
},
StorageClass: u.StorageClass,
Initiated: *u.Initiated,
StorageClass: string(u.StorageClass),
Initiated: u.Initiated.Format(backend.RFC3339TimeFormat),
})
}
@@ -241,7 +199,7 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
for _, p := range output.Parts {
parts = append(parts, s3response.Part{
PartNumber: int(*p.PartNumber),
LastModified: *p.LastModified,
LastModified: p.LastModified.Format(backend.RFC3339TimeFormat),
ETag: *p.ETag,
Size: *p.Size,
})
@@ -270,7 +228,7 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
ID: *output.Owner.ID,
DisplayName: *output.Owner.DisplayName,
},
StorageClass: output.StorageClass,
StorageClass: string(output.StorageClass),
PartNumberMarker: pnm,
NextPartNumberMarker: npmn,
MaxParts: int(*output.MaxParts),
@@ -322,50 +280,24 @@ func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
return out, handleError(err)
}
func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
output, err := s.client.GetObject(ctx, input)
if err != nil {
return nil, handleError(err)
}
defer output.Body.Close()
_, err = io.Copy(w, output.Body)
if err != nil {
return nil, err
}
return output, nil
}
func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
out, err := s.client.GetObjectAttributes(ctx, input)
parts := s3response.ObjectParts{}
objParts := out.ObjectParts
if objParts != nil {
if objParts.PartNumberMarker != nil {
partNumberMarker, err := strconv.Atoi(*objParts.PartNumberMarker)
if err != nil {
parts.PartNumberMarker = partNumberMarker
}
if objParts.NextPartNumberMarker != nil {
nextPartNumberMarker, err := strconv.Atoi(*objParts.NextPartNumberMarker)
if err != nil {
parts.NextPartNumberMarker = nextPartNumberMarker
}
}
if objParts.IsTruncated != nil {
parts.IsTruncated = *objParts.IsTruncated
}
if objParts.MaxParts != nil {
parts.MaxParts = int(*objParts.MaxParts)
}
parts.Parts = objParts.Parts
}
}
return s3response.GetObjectAttributesResult{
ETag: out.ETag,
LastModified: out.LastModified,
ObjectSize: out.ObjectSize,
StorageClass: out.StorageClass,
VersionId: out.VersionId,
ObjectParts: &parts,
}, handleError(err)
return out, handleError(err)
}
func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
@@ -373,47 +305,14 @@ func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s
return out, handleError(err)
}
func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
out, err := s.client.ListObjects(ctx, input)
if err != nil {
return s3response.ListObjectsResult{}, handleError(err)
}
contents := convertObjects(out.Contents)
return s3response.ListObjectsResult{
CommonPrefixes: out.CommonPrefixes,
Contents: contents,
Delimiter: out.Delimiter,
IsTruncated: out.IsTruncated,
Marker: out.Marker,
MaxKeys: out.MaxKeys,
Name: out.Name,
NextMarker: out.NextMarker,
Prefix: out.Prefix,
}, nil
return out, handleError(err)
}
func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
out, err := s.client.ListObjectsV2(ctx, input)
if err != nil {
return s3response.ListObjectsV2Result{}, handleError(err)
}
contents := convertObjects(out.Contents)
return s3response.ListObjectsV2Result{
CommonPrefixes: out.CommonPrefixes,
Contents: contents,
Delimiter: out.Delimiter,
IsTruncated: out.IsTruncated,
ContinuationToken: out.ContinuationToken,
MaxKeys: out.MaxKeys,
Name: out.Name,
NextContinuationToken: out.NextContinuationToken,
Prefix: out.Prefix,
KeyCount: out.KeyCount,
}, nil
return out, handleError(err)
}
func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
@@ -537,135 +436,8 @@ func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string
return handleError(err)
}
func (s *S3Proxy) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
_, err := s.client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
Bucket: &bucket,
Policy: backend.GetStringPtr(string(policy)),
})
return handleError(err)
}
func (s *S3Proxy) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
policy, err := s.client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
Bucket: &bucket,
})
if err != nil {
return nil, handleError(err)
}
result := []byte{}
if policy.Policy != nil {
result = []byte(*policy.Policy)
}
return result, nil
}
func (s *S3Proxy) DeleteBucketPolicy(ctx context.Context, bucket string) error {
_, err := s.client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
Bucket: &bucket,
})
return handleError(err)
}
func (s *S3Proxy) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
cfg, err := auth.ParseBucketLockConfigurationOutput(config)
if err != nil {
return err
}
_, err = s.client.PutObjectLockConfiguration(ctx, &s3.PutObjectLockConfigurationInput{
Bucket: &bucket,
ObjectLockConfiguration: cfg,
})
return handleError(err)
}
func (s *S3Proxy) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) {
resp, err := s.client.GetObjectLockConfiguration(ctx, &s3.GetObjectLockConfigurationInput{
Bucket: &bucket,
})
if err != nil {
return nil, handleError(err)
}
config := auth.BucketLockConfig{
Enabled: resp.ObjectLockConfiguration.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
DefaultRetention: resp.ObjectLockConfiguration.Rule.DefaultRetention,
}
return json.Marshal(config)
}
func (s *S3Proxy) PutObjectRetention(ctx context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
ret, err := auth.ParseObjectLockRetentionOutput(retention)
if err != nil {
return err
}
_, err = s.client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
Bucket: &bucket,
Key: &object,
VersionId: &versionId,
Retention: ret,
BypassGovernanceRetention: &bypass,
})
return handleError(err)
}
func (s *S3Proxy) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) {
resp, err := s.client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{
Bucket: &bucket,
Key: &object,
VersionId: &versionId,
})
if err != nil {
return nil, handleError(err)
}
return json.Marshal(resp.Retention)
}
func (s *S3Proxy) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
var st types.ObjectLockLegalHoldStatus
if status {
st = types.ObjectLockLegalHoldStatusOn
} else {
st = types.ObjectLockLegalHoldStatusOff
}
_, err := s.client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{
Bucket: &bucket,
Key: &object,
VersionId: &versionId,
LegalHold: &types.ObjectLockLegalHold{
Status: st,
},
})
return handleError(err)
}
func (s *S3Proxy) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) {
resp, err := s.client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{
Bucket: &bucket,
Key: &object,
VersionId: &versionId,
})
if err != nil {
return nil, handleError(err)
}
status := resp.LegalHold.Status == types.ObjectLockLegalHoldStatusOn
return &status, nil
}
func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket string, acl []byte) error {
var acll auth.ACL
if err := json.Unmarshal(acl, &acll); err != nil {
return fmt.Errorf("unmarshal acl: %w", err)
}
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, acll.Owner), nil)
func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, newOwner), nil)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
@@ -695,7 +467,7 @@ func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket string, acl []by
return err
}
defer resp.Body.Close()
return fmt.Errorf("%v", string(body))
return fmt.Errorf(string(body))
}
return nil
@@ -771,21 +543,3 @@ func base64Decode(encoded string) ([]byte, error) {
}
return decoded, nil
}
func convertObjects(objs []types.Object) []s3response.Object {
result := make([]s3response.Object, len(objs))
for _, obj := range objs {
result = append(result, s3response.Object{
ETag: obj.ETag,
Key: obj.Key,
LastModified: obj.LastModified,
Owner: obj.Owner,
Size: obj.Size,
RestoreStatus: obj.RestoreStatus,
StorageClass: obj.StorageClass,
})
}
return result
}

View File

@@ -26,24 +26,21 @@ import (
"path/filepath"
"strings"
"syscall"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/pkg/xattr"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/backend/posix"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
type ScoutfsOpts struct {
ChownUID bool
ChownGID bool
ReadOnly bool
GlacierMode bool
BucketLinks bool
}
type ScoutFS struct {
@@ -51,9 +48,6 @@ type ScoutFS struct {
rootfd *os.File
rootdir string
// bucket/object metadata storage facility
meta meta.MetadataStorer
// glaciermode enables the following behavior:
// GET object: if file offline, return invalid object state
// HEAD object: if file offline, set obj storage class to GLACIER
@@ -70,6 +64,9 @@ type ScoutFS struct {
chownuid bool
chowngid bool
// read only mode prevents any backend modifications
readonly bool
// euid/egid are the effective uid/gid of the running versitygw process
// used to determine if chowning is needed
euid int
@@ -82,13 +79,8 @@ const (
metaTmpDir = ".sgwtmp"
metaTmpMultipartDir = metaTmpDir + "/multipart"
tagHdr = "X-Amz-Tagging"
metaHdr = "X-Amz-Meta"
contentTypeHdr = "content-type"
contentEncHdr = "content-encoding"
emptyMD5 = "d41d8cd98f00b204e9800998ecf8427e"
etagkey = "etag"
objectRetentionKey = "object-retention"
objectLegalHoldKey = "object-legal-hold"
etagkey = "user.etag"
)
var (
@@ -99,12 +91,11 @@ var (
const (
// ScoutFS special xattr types
systemPrefix = "scoutfs.hide."
onameAttr = systemPrefix + "objname"
flagskey = systemPrefix + "sam_flags"
stagecopykey = systemPrefix + "sam_stagereq"
fsBlocksize = 4096
)
const (
@@ -156,6 +147,10 @@ func (s *ScoutFS) getChownIDs(acct auth.Account) (int, int, bool) {
// ioctl to not have to read and copy the part data to the final object. This
// saves a read and write cycle for all mutlipart uploads.
func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
if s.readonly {
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
}
acct, ok := ctx.Value("account").(auth.Account)
if !ok {
acct = auth.Account{}
@@ -192,20 +187,18 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
return nil, err
}
objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum))
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
// check all parts ok
last := len(parts) - 1
partsize := int64(0)
var totalsize int64
for i, part := range parts {
if part.PartNumber == nil || *part.PartNumber < 1 {
for i, p := range parts {
if p.PartNumber == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
fullPartPath := filepath.Join(bucket, partObjPath)
fi, err := os.Lstat(fullPartPath)
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))
fi, err := os.Lstat(partPath)
if err != nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
@@ -213,25 +206,23 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
if i == 0 {
partsize = fi.Size()
}
// partsize must be a multiple of the filesystem blocksize
// except for last part
if i < last && partsize%fsBlocksize != 0 {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
totalsize += fi.Size()
// all parts except the last need to be the same size
if i < last && partsize != fi.Size() {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
// non-last part sizes need to be multiples of 4k for move blocks
// TODO: fallback to no move blocks if not 4k aligned?
if i == 0 && i < last && fi.Size()%4096 != 0 {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
b, err := s.meta.RetrieveAttribute(bucket, partObjPath, etagkey)
b, err := xattr.Get(partPath, "user.etag")
etag := string(b)
if err != nil {
etag = ""
}
if parts[i].ETag == nil || etag != *parts[i].ETag {
if etag != *parts[i].ETag {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
}
@@ -247,16 +238,10 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
}
defer f.cleanup()
for _, part := range parts {
if part.PartNumber == nil || *part.PartNumber < 1 {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
partObjPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part.PartNumber))
fullPartPath := filepath.Join(bucket, partObjPath)
pf, err := os.Open(fullPartPath)
for _, p := range parts {
pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)))
if err != nil {
return nil, fmt.Errorf("open part %v: %v", *part.PartNumber, err)
return nil, fmt.Errorf("open part %v: %v", *p.PartNumber, err)
}
// scoutfs move data is a metadata only operation that moves the data
@@ -265,13 +250,13 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
err = moveData(pf, f.f)
pf.Close()
if err != nil {
return nil, fmt.Errorf("move blocks part %v: %v", *part.PartNumber, err)
return nil, fmt.Errorf("move blocks part %v: %v", *p.PartNumber, err)
}
}
userMetaData := make(map[string]string)
upiddir := filepath.Join(objdir, uploadID)
cType, _ := s.loadUserMetaData(bucket, upiddir, userMetaData)
loadUserMetaData(upiddir, userMetaData)
objname := filepath.Join(bucket, object)
dir := filepath.Dir(objname)
@@ -288,7 +273,7 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
}
for k, v := range userMetaData {
err = s.meta.StoreAttribute(bucket, object, fmt.Sprintf("%v.%v", metaHdr, k), []byte(v))
err = xattr.Set(objname, "user."+k, []byte(v))
if err != nil {
// cleanup object if returning error
os.Remove(objname)
@@ -296,58 +281,10 @@ func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.Complet
}
}
// load and set tagging
tagging, err := s.meta.RetrieveAttribute(bucket, upiddir, tagHdr)
if err == nil {
if err := s.meta.StoreAttribute(bucket, object, tagHdr, tagging); err != nil {
// cleanup object
os.Remove(objname)
return nil, fmt.Errorf("set object tagging: %w", err)
}
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object tagging: %w", err)
}
// set content-type
if cType != "" {
if err := s.meta.StoreAttribute(bucket, object, contentTypeHdr, []byte(cType)); err != nil {
// cleanup object
os.Remove(objname)
return nil, fmt.Errorf("set object content type: %w", err)
}
}
// load and set legal hold
lHold, err := s.meta.RetrieveAttribute(bucket, upiddir, objectLegalHoldKey)
if err == nil {
if err := s.meta.StoreAttribute(bucket, object, objectLegalHoldKey, lHold); err != nil {
// cleanup object
os.Remove(objname)
return nil, fmt.Errorf("set object legal hold: %w", err)
}
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object legal hold: %w", err)
}
// load and set retention
ret, err := s.meta.RetrieveAttribute(bucket, upiddir, objectRetentionKey)
if err == nil {
if err := s.meta.StoreAttribute(bucket, object, objectRetentionKey, ret); err != nil {
// cleanup object
os.Remove(objname)
return nil, fmt.Errorf("set object retention: %w", err)
}
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return nil, fmt.Errorf("get object retention: %w", err)
}
// Calculate s3 compatible md5sum for complete multipart.
s3MD5 := backend.GetMultipartMD5(parts)
err = s.meta.StoreAttribute(bucket, object, etagkey, []byte(s3MD5))
err = xattr.Set(objname, "user.etag", []byte(s3MD5))
if err != nil {
// cleanup object if returning error
os.Remove(objname)
@@ -381,107 +318,61 @@ func (s *ScoutFS) checkUploadIDExists(bucket, object, uploadID string) ([32]byte
return sum, nil
}
// fll out the user metadata map with the metadata for the object
// and return the content type and encoding
func (s *ScoutFS) loadUserMetaData(bucket, object string, m map[string]string) (string, string) {
ents, err := s.meta.ListAttributes(bucket, object)
func loadUserMetaData(path string, m map[string]string) (contentType, contentEncoding string) {
ents, err := xattr.List(path)
if err != nil || len(ents) == 0 {
return "", ""
return
}
for _, e := range ents {
if !isValidMeta(e) {
continue
}
b, err := s.meta.RetrieveAttribute(bucket, object, e)
b, err := xattr.Get(path, e)
if err == errNoData {
m[strings.TrimPrefix(e, "user.")] = ""
continue
}
if err != nil {
continue
}
if b == nil {
m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = ""
continue
}
m[strings.TrimPrefix(e, fmt.Sprintf("%v.", metaHdr))] = string(b)
m[strings.TrimPrefix(e, "user.")] = string(b)
}
var contentType, contentEncoding string
b, _ := s.meta.RetrieveAttribute(bucket, object, contentTypeHdr)
b, err := xattr.Get(path, "user.content-type")
contentType = string(b)
if err != nil {
contentType = ""
}
if contentType != "" {
m[contentTypeHdr] = contentType
m["content-type"] = contentType
}
b, _ = s.meta.RetrieveAttribute(bucket, object, contentEncHdr)
b, err = xattr.Get(path, "user.content-encoding")
contentEncoding = string(b)
if err != nil {
contentEncoding = ""
}
if contentEncoding != "" {
m[contentEncHdr] = contentEncoding
m["content-encoding"] = contentEncoding
}
return contentType, contentEncoding
return
}
func isValidMeta(val string) bool {
if strings.HasPrefix(val, metaHdr) {
if strings.HasPrefix(val, "user.X-Amz-Meta") {
return true
}
if strings.EqualFold(val, "Expires") {
if strings.EqualFold(val, "user.Expires") {
return true
}
return false
}
func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
bucket := *input.Bucket
object := *input.Key
if input.PartNumber != nil {
uploadId, sum, err := s.retrieveUploadId(bucket, object)
if err != nil {
return nil, err
}
ents, err := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return nil, fmt.Errorf("read parts: %w", err)
}
partPath := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId, fmt.Sprintf("%v", *input.PartNumber))
part, err := os.Stat(filepath.Join(bucket, partPath))
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
}
if err != nil {
return nil, fmt.Errorf("stat part: %w", err)
}
b, err := s.meta.RetrieveAttribute(bucket, partPath, etagkey)
etag := string(b)
if err != nil {
etag = ""
}
partsCount := int32(len(ents))
size := part.Size()
return &s3.HeadObjectOutput{
LastModified: backend.GetTimePtr(part.ModTime()),
ETag: &etag,
PartsCount: &partsCount,
ContentLength: &size,
}, nil
}
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -491,30 +382,18 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
}
objPath := filepath.Join(bucket, object)
fi, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
}
if err != nil {
return nil, fmt.Errorf("stat object: %w", err)
}
if strings.HasSuffix(object, "/") && !fi.IsDir() {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
userMetaData := make(map[string]string)
contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
if fi.IsDir() {
// this is the media type for directories in AWS and Nextcloud
contentType = "application/x-directory"
}
b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
b, err := xattr.Get(objPath, etagkey)
etag := string(b)
if err != nil {
etag = ""
@@ -553,55 +432,19 @@ func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s
contentLength := fi.Size()
var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
status, err := s.Posix.GetObjectLegalHold(ctx, bucket, object, "")
if err == nil {
if *status {
objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
} else {
objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff
}
}
var objectLockMode types.ObjectLockMode
var objectLockRetainUntilDate *time.Time
retention, err := s.Posix.GetObjectRetention(ctx, bucket, object, "")
if err == nil {
var config types.ObjectLockRetention
if err := json.Unmarshal(retention, &config); err == nil {
objectLockMode = types.ObjectLockMode(config.Mode)
objectLockRetainUntilDate = config.RetainUntilDate
}
}
return &s3.HeadObjectOutput{
ContentLength: &contentLength,
ContentType: &contentType,
ContentEncoding: &contentEncoding,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
StorageClass: stclass,
Restore: &requestOngoing,
ObjectLockLegalHoldStatus: objectLockLegalHoldStatus,
ObjectLockMode: objectLockMode,
ObjectLockRetainUntilDate: objectLockRetainUntilDate,
ContentLength: &contentLength,
ContentType: &contentType,
ContentEncoding: &contentEncoding,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
StorageClass: stclass,
Restore: &requestOngoing,
}, nil
}
func (s *ScoutFS) retrieveUploadId(bucket, object string) (string, [32]byte, error) {
sum := sha256.Sum256([]byte(object))
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
entries, err := os.ReadDir(objdir)
if err != nil || len(entries) == 0 {
return "", [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
return entries[0].Name(), sum, nil
}
func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
bucket := *input.Bucket
object := *input.Key
acceptRange := *input.Range
@@ -615,23 +458,15 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
}
objPath := filepath.Join(bucket, object)
fi, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
}
if err != nil {
return nil, fmt.Errorf("stat object: %w", err)
}
if strings.HasSuffix(object, "/") && !fi.IsDir() {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
startOffset, length, err := backend.ParseRange(fi.Size(), acceptRange)
startOffset, length, err := backend.ParseRange(fi, acceptRange)
if err != nil {
return nil, err
}
@@ -678,14 +513,19 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
if err != nil {
return nil, fmt.Errorf("open object: %w", err)
}
defer f.Close()
rdr := io.NewSectionReader(f, startOffset, length)
_, err = io.Copy(writer, rdr)
if err != nil {
return nil, fmt.Errorf("copy data: %w", err)
}
userMetaData := make(map[string]string)
contentType, contentEncoding := s.loadUserMetaData(bucket, object, userMetaData)
contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
b, err := s.meta.RetrieveAttribute(bucket, object, etagkey)
b, err := xattr.Get(objPath, etagkey)
etag := string(b)
if err != nil {
etag = ""
@@ -709,7 +549,6 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.Ge
TagCount: &tagCount,
StorageClass: types.StorageClassStandard,
ContentRange: &contentRange,
Body: &backend.FileSectionReadCloser{R: rdr, F: f},
}, nil
}
@@ -734,9 +573,9 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error)
return tags, nil
}
func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
if input.Bucket == nil {
return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
bucket := *input.Bucket
prefix := ""
@@ -758,20 +597,20 @@ func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return s3response.ListObjectsResult{}, fmt.Errorf("stat bucket: %w", err)
return nil, fmt.Errorf("stat bucket: %w", err)
}
fileSystem := os.DirFS(bucket)
results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
s.fileToObj(bucket), []string{metaTmpDir})
if err != nil {
return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err)
return nil, fmt.Errorf("walk %v: %w", bucket, err)
}
return s3response.ListObjectsResult{
return &s3.ListObjectsOutput{
CommonPrefixes: results.CommonPrefixes,
Contents: results.Objects,
Delimiter: &delim,
@@ -784,9 +623,9 @@ func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (
}, nil
}
func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
if input.Bucket == nil {
return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
bucket := *input.Bucket
prefix := ""
@@ -808,20 +647,20 @@ func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return s3response.ListObjectsV2Result{}, fmt.Errorf("stat bucket: %w", err)
return nil, fmt.Errorf("stat bucket: %w", err)
}
fileSystem := os.DirFS(bucket)
results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, int32(maxkeys),
results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
s.fileToObj(bucket), []string{metaTmpDir})
if err != nil {
return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err)
return nil, fmt.Errorf("walk %v: %w", bucket, err)
}
return s3response.ListObjectsV2Result{
return &s3.ListObjectsV2Output{
CommonPrefixes: results.CommonPrefixes,
Contents: results.Objects,
Delimiter: &delim,
@@ -835,58 +674,53 @@ func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu
}
func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
return func(path string, d fs.DirEntry) (s3response.Object, error) {
return func(path string, d fs.DirEntry) (types.Object, error) {
objPath := filepath.Join(bucket, path)
if d.IsDir() {
// directory object only happens if directory empty
// check to see if this is a directory object by checking etag
etagBytes, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) {
return s3response.Object{}, backend.ErrSkipObj
etagBytes, err := xattr.Get(objPath, etagkey)
if isNoAttr(err) || errors.Is(err, fs.ErrNotExist) {
return types.Object{}, backend.ErrSkipObj
}
if err != nil {
return s3response.Object{}, fmt.Errorf("get etag: %w", err)
return types.Object{}, fmt.Errorf("get etag: %w", err)
}
etag := string(etagBytes)
fi, err := d.Info()
if errors.Is(err, fs.ErrNotExist) {
return s3response.Object{}, backend.ErrSkipObj
return types.Object{}, backend.ErrSkipObj
}
if err != nil {
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
key := path + "/"
mtime := fi.ModTime()
return s3response.Object{
return types.Object{
ETag: &etag,
Key: &key,
LastModified: &mtime,
StorageClass: types.ObjectStorageClassStandard,
LastModified: backend.GetTimePtr(fi.ModTime()),
}, nil
}
// file object, get object info and fill out object data
b, err := s.meta.RetrieveAttribute(bucket, path, etagkey)
etagBytes, err := xattr.Get(objPath, etagkey)
if errors.Is(err, fs.ErrNotExist) {
return s3response.Object{}, backend.ErrSkipObj
return types.Object{}, backend.ErrSkipObj
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
return s3response.Object{}, fmt.Errorf("get etag: %w", err)
if err != nil && !isNoAttr(err) {
return types.Object{}, fmt.Errorf("get etag: %w", err)
}
// note: meta.ErrNoSuchKey will return etagBytes = []byte{}
// so this will just set etag to "" if its not already set
etag := string(b)
etag := string(etagBytes)
fi, err := d.Info()
if errors.Is(err, fs.ErrNotExist) {
return s3response.Object{}, backend.ErrSkipObj
return types.Object{}, backend.ErrSkipObj
}
if err != nil {
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
sc := types.ObjectStorageClassStandard
@@ -895,10 +729,10 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
// If so, we will return the InvalidObjectState error.
st, err := statMore(objPath)
if errors.Is(err, fs.ErrNotExist) {
return s3response.Object{}, backend.ErrSkipObj
return types.Object{}, backend.ErrSkipObj
}
if err != nil {
return s3response.Object{}, fmt.Errorf("stat more: %w", err)
return types.Object{}, fmt.Errorf("stat more: %w", err)
}
if st.Offline_blocks != 0 {
sc = types.ObjectStorageClassGlacier
@@ -906,12 +740,11 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
}
size := fi.Size()
mtime := fi.ModTime()
return s3response.Object{
return types.Object{
ETag: &etag,
Key: &path,
LastModified: &mtime,
LastModified: backend.GetTimePtr(fi.ModTime()),
Size: &size,
StorageClass: sc,
}, nil
@@ -994,9 +827,15 @@ func fSetNewGlobalFlags(objname string, flags uint64) error {
}
func isNoAttr(err error) bool {
if err == nil {
return false
}
xerr, ok := err.(*xattr.Error)
if ok && xerr.Err == xattr.ENOATTR {
return true
}
if err == errNoData {
return true
}
return false
}

View File

@@ -23,23 +23,21 @@ import (
"os"
"path/filepath"
"strconv"
"syscall"
"golang.org/x/sys/unix"
"github.com/versity/scoutfs-go"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/backend/posix"
)
func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
metastore := meta.XattrMeta{}
p, err := posix.New(rootdir, metastore, posix.PosixOpts{
ChownUID: opts.ChownUID,
ChownGID: opts.ChownGID,
BucketLinks: opts.BucketLinks,
p, err := posix.New(rootdir, posix.PosixOpts{
ChownUID: opts.ChownUID,
ChownGID: opts.ChownGID,
ReadOnly: opts.ReadOnly,
})
if err != nil {
return nil, err
@@ -51,13 +49,12 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
}
return &ScoutFS{
Posix: p,
rootfd: f,
rootdir: rootdir,
meta: metastore,
chownuid: opts.ChownUID,
chowngid: opts.ChownGID,
glaciermode: opts.GlacierMode,
Posix: p,
rootfd: f,
rootdir: rootdir,
chownuid: opts.ChownUID,
chowngid: opts.ChownGID,
readonly: opts.ReadOnly,
}, nil
}
@@ -104,6 +101,11 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
gid: gid,
}
// falloc is best effort, its fine if this fails
if size > 0 {
tmp.falloc()
}
if doChown {
err := f.Chown(uid, gid)
if err != nil {
@@ -114,6 +116,14 @@ func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Acc
return tmp, nil
}
func (tmp *tmpfile) falloc() error {
err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
if err != nil {
return fmt.Errorf("fallocate: %v", err)
}
return nil
}
func (tmp *tmpfile) link() error {
// We use Linkat/Rename as the atomic operation for object puts. The
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict

View File

@@ -42,6 +42,7 @@ func (s *ScoutFS) openTmpFile(_, _, _ string, _ int64, _ auth.Account) (*tmpfile
_ = s.chowngid
_ = s.euid
_ = s.egid
_ = s.readonly
return nil, errNotSupported
}

View File

@@ -0,0 +1,24 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build !freebsd && !openbsd && !netbsd
// +build !freebsd,!openbsd,!netbsd
package scoutfs
import "syscall"
var (
errNoData = syscall.ENODATA
)

View File

@@ -0,0 +1,24 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build freebsd || openbsd || netbsd
// +build freebsd openbsd netbsd
package scoutfs
import "syscall"
var (
errNoData = syscall.ENOATTR
)

View File

@@ -15,7 +15,6 @@
package backend
import (
"context"
"errors"
"fmt"
"io/fs"
@@ -24,25 +23,24 @@ import (
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3response"
)
type WalkResults struct {
CommonPrefixes []types.CommonPrefix
Objects []s3response.Object
Objects []types.Object
Truncated bool
NextMarker string
}
type GetObjFunc func(path string, d fs.DirEntry) (s3response.Object, error)
type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
var ErrSkipObj = errors.New("skip this object")
// Walk walks the supplied fs.FS and returns results compatible with list
// objects responses
func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
cpmap := make(map[string]struct{})
var objects []s3response.Object
var objects []types.Object
var pastMarker bool
if marker == "" {
@@ -57,9 +55,6 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
if err != nil {
return err
}
if ctx.Err() != nil {
return ctx.Err()
}
// Ignore the root directory
if path == "." {
return nil
@@ -96,10 +91,7 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
if err != nil {
return fmt.Errorf("readdir %q: %w", path, err)
}
path += string(os.PathSeparator)
if len(ents) == 0 && delimiter == "" {
if len(ents) == 0 {
dirobj, err := getObj(path, d)
if err == ErrSkipObj {
return nil
@@ -108,13 +100,9 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
return fmt.Errorf("directory to object %q: %w", path, err)
}
objects = append(objects, dirobj)
return nil
}
if len(ents) != 0 {
return nil
}
return nil
}
if !pastMarker {
@@ -192,16 +180,9 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin
// Common prefixes are a set, so should not have duplicates.
// These are abstractly a "directory", so need to include the
// delimiter at the end.
cpref := prefix + before + delimiter
if cpref == marker {
pastMarker = true
return nil
}
cpmap[cpref] = struct{}{}
cpmap[prefix+before+delimiter] = struct{}{}
if (len(objects) + len(cpmap)) == int(max) {
newMarker = cpref
truncated = true
return fs.SkipAll
pastMax = true
}
return nil

View File

@@ -15,19 +15,15 @@
package backend_test
import (
"context"
"crypto/md5"
"encoding/hex"
"fmt"
"io/fs"
"sync"
"testing"
"testing/fstest"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3response"
)
type walkTest struct {
@@ -36,20 +32,19 @@ type walkTest struct {
getobj backend.GetObjFunc
}
func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
func getObj(path string, d fs.DirEntry) (types.Object, error) {
if d.IsDir() {
etag := getMD5(path)
fi, err := d.Info()
if err != nil {
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
mtime := fi.ModTime()
return s3response.Object{
return types.Object{
ETag: &etag,
Key: &path,
LastModified: &mtime,
LastModified: backend.GetTimePtr(fi.ModTime()),
}, nil
}
@@ -57,16 +52,15 @@ func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
fi, err := d.Info()
if err != nil {
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
size := fi.Size()
mtime := fi.ModTime()
return s3response.Object{
return types.Object{
ETag: &etag,
Key: &path,
LastModified: &mtime,
LastModified: backend.GetTimePtr(fi.ModTime()),
Size: &size,
}, nil
}
@@ -92,7 +86,7 @@ func TestWalk(t *testing.T) {
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetStringPtr("photos/"),
}},
Objects: []s3response.Object{{
Objects: []types.Object{{
Key: backend.GetStringPtr("sample.jpg"),
}},
},
@@ -107,14 +101,14 @@ func TestWalk(t *testing.T) {
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetStringPtr("test/"),
}},
Objects: []s3response.Object{},
Objects: []types.Object{},
},
getobj: getObj,
},
}
for _, tt := range tests {
res, err := backend.Walk(context.Background(), tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
if err != nil {
t.Fatalf("walk: %v", err)
}
@@ -174,7 +168,7 @@ func printCommonPrefixes(list []types.CommonPrefix) string {
return res + "]"
}
func compareObjects(a, b []s3response.Object) bool {
func compareObjects(a, b []types.Object) bool {
if len(a) == 0 && len(b) == 0 {
return true
}
@@ -190,7 +184,7 @@ func compareObjects(a, b []s3response.Object) bool {
return false
}
func containsObject(c s3response.Object, list []s3response.Object) bool {
func containsObject(c types.Object, list []types.Object) bool {
for _, cp := range list {
if *c.Key == *cp.Key {
return true
@@ -199,7 +193,7 @@ func containsObject(c s3response.Object, list []s3response.Object) bool {
return false
}
func printObjects(list []s3response.Object) string {
func printObjects(list []types.Object) string {
res := "["
for _, cp := range list {
if res == "[" {
@@ -210,50 +204,3 @@ func printObjects(list []s3response.Object) string {
}
return res + "]"
}
type slowFS struct {
fstest.MapFS
}
const (
readDirPause = 100 * time.Millisecond
// walkTimeOut should be less than the tree traversal time
// which is the readdirPause time * the number of directories
walkTimeOut = 500 * time.Millisecond
)
func (s *slowFS) ReadDir(name string) ([]fs.DirEntry, error) {
time.Sleep(readDirPause)
return s.MapFS.ReadDir(name)
}
func TestWalkStop(t *testing.T) {
s := &slowFS{MapFS: fstest.MapFS{
"/a/b/c/d/e/f/g/h/i/g/k/l/m/n": &fstest.MapFile{},
}}
ctx, cancel := context.WithTimeout(context.Background(), walkTimeOut)
defer cancel()
var err error
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
_, err = backend.Walk(ctx, s, "", "/", "", 1000,
func(path string, d fs.DirEntry) (s3response.Object, error) {
return s3response.Object{}, nil
}, []string{})
}()
select {
case <-time.After(1 * time.Second):
t.Fatalf("walk is not terminated in time")
case <-ctx.Done():
}
wg.Wait()
if err != ctx.Err() {
t.Fatalf("unexpected error: %v", err)
}
}

View File

@@ -17,7 +17,6 @@ package main
import (
"bytes"
"crypto/sha256"
"crypto/tls"
"encoding/hex"
"encoding/json"
"fmt"
@@ -38,7 +37,6 @@ var (
adminAccess string
adminSecret string
adminEndpoint string
allowInsecure bool
)
func adminCommand() *cli.Command {
@@ -80,33 +78,10 @@ func adminCommand() *cli.Command {
Usage: "groupID for the new user",
Aliases: []string{"gi"},
},
},
},
{
Name: "update-user",
Usage: "Updates a user account",
Action: updateUser,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "access",
Usage: "user access key id to be updated",
Required: true,
Aliases: []string{"a"},
},
&cli.StringFlag{
Name: "secret",
Usage: "secret access key for the new user",
Aliases: []string{"s"},
},
&cli.IntFlag{
Name: "user-id",
Usage: "userID for the new user",
Aliases: []string{"ui"},
},
&cli.IntFlag{
Name: "group-id",
Usage: "groupID for the new user",
Aliases: []string{"gi"},
Name: "project-id",
Usage: "projectID for the new user",
Aliases: []string{"pi"},
},
},
},
@@ -179,40 +154,27 @@ func adminCommand() *cli.Command {
Required: true,
Destination: &adminEndpoint,
},
&cli.BoolFlag{
Name: "allow-insecure",
Usage: "disable tls certificate verification for the admin endpoint",
EnvVars: []string{"ADMIN_ALLOW_INSECURE"},
Aliases: []string{"ai"},
Destination: &allowInsecure,
},
},
}
}
func initHTTPClient() *http.Client {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: allowInsecure},
}
return &http.Client{Transport: tr}
}
func createUser(ctx *cli.Context) error {
access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
userID, groupID := ctx.Int("user-id"), ctx.Int("group-id")
userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
if access == "" || secret == "" {
return fmt.Errorf("invalid input parameters for the new user access/secret keys")
return fmt.Errorf("invalid input parameters for the new user")
}
if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
return fmt.Errorf("invalid input parameter for role: %v", role)
}
acc := auth.Account{
Access: access,
Secret: secret,
Role: auth.Role(role),
UserID: userID,
GroupID: groupID,
Access: access,
Secret: secret,
Role: auth.Role(role),
UserID: userID,
GroupID: groupID,
ProjectID: projectID,
}
accJson, err := json.Marshal(acc)
@@ -237,22 +199,18 @@ func createUser(ctx *cli.Context) error {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode >= 400 {
return fmt.Errorf("%s", body)
}
defer resp.Body.Close()
fmt.Printf("%s\n", body)
@@ -262,7 +220,7 @@ func createUser(ctx *cli.Context) error {
func deleteUser(ctx *cli.Context) error {
access := ctx.String("access")
if access == "" {
return fmt.Errorf("invalid input parameter for the user access key")
return fmt.Errorf("invalid input parameter for the new user")
}
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
@@ -282,80 +240,19 @@ func deleteUser(ctx *cli.Context) error {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode >= 400 {
return fmt.Errorf("%s", body)
}
fmt.Printf("%s\n", body)
return nil
}
func updateUser(ctx *cli.Context) error {
access, secret, userId, groupId := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id")
props := auth.MutableProps{}
if ctx.IsSet("secret") {
props.Secret = &secret
}
if ctx.IsSet("user-id") {
props.UserID = &userId
}
if ctx.IsSet("group-id") {
props.GroupID = &groupId
}
propsJSON, err := json.Marshal(props)
if err != nil {
return fmt.Errorf("failed to parse user attributes: %w", err)
}
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsJSON))
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256(propsJSON)
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode >= 400 {
return fmt.Errorf("%s", body)
}
fmt.Printf("%s\n", body)
return nil
@@ -379,18 +276,18 @@ func listUsers(ctx *cli.Context) error {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return fmt.Errorf("%s", body)
@@ -418,10 +315,10 @@ const (
func printAcctTable(accs []auth.Account) {
w := new(tabwriter.Writer)
w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID")
fmt.Fprintln(w, "-------\t----\t------\t-------")
fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
for _, acc := range accs {
fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID)
fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
}
fmt.Fprintln(w)
w.Flush()
@@ -446,22 +343,18 @@ func changeBucketOwner(ctx *cli.Context) error {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode >= 400 {
return fmt.Errorf("%s", body)
}
defer resp.Body.Close()
fmt.Println(string(body))
@@ -498,18 +391,18 @@ func listBuckets(ctx *cli.Context) error {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return fmt.Errorf("%s", body)

View File

@@ -8,7 +8,6 @@ import (
"sync"
"testing"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/backend/posix"
"github.com/versity/versitygw/tests/integration"
)
@@ -57,7 +56,7 @@ func initPosix(ctx context.Context) {
log.Fatalf("make temp directory: %v", err)
}
be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{})
be, err := posix.New(tempdir, posix.PosixOpts{})
if err != nil {
log.Fatalf("init posix: %v", err)
}

View File

@@ -19,17 +19,14 @@ import (
"crypto/tls"
"fmt"
"log"
"net"
"net/http"
_ "net/http/pprof"
"os"
"strings"
"github.com/gofiber/fiber/v2"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3event"
@@ -37,43 +34,33 @@ import (
)
var (
port, admPort string
rootUserAccess string
rootUserSecret string
region string
admCertFile, admKeyFile string
certFile, keyFile string
kafkaURL, kafkaTopic, kafkaKey string
natsURL, natsTopic string
eventWebhookURL string
eventConfigFilePath string
logWebhookURL, accessLog string
adminLogFile string
healthPath string
debug bool
pprof string
quiet bool
readonly bool
iamDir string
ldapURL, ldapBindDN, ldapPassword string
ldapQueryBase, ldapObjClasses string
ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
ldapUserIdAtr, ldapGroupIdAtr string
vaultEndpointURL, vaultSecretStoragePath string
vaultMountPath, vaultRootToken string
vaultRoleId, vaultRoleSecret string
vaultServerCert, vaultClientCert string
vaultClientCertKey string
s3IamAccess, s3IamSecret string
s3IamRegion, s3IamBucket string
s3IamEndpoint string
s3IamSslNoVerify, s3IamDebug bool
iamCacheDisable bool
iamCacheTTL int
iamCachePrune int
metricsService string
statsdServers string
dogstatsServers string
port, admPort string
rootUserAccess string
rootUserSecret string
region string
admCertFile, admKeyFile string
certFile, keyFile string
kafkaURL, kafkaTopic, kafkaKey string
natsURL, natsTopic string
eventWebhookURL string
eventConfigFilePath string
logWebhookURL string
accessLog string
healthPath string
debug bool
pprof string
quiet bool
iamDir string
ldapURL, ldapBindDN, ldapPassword string
ldapQueryBase, ldapObjClasses string
ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
s3IamAccess, s3IamSecret string
s3IamRegion, s3IamBucket string
s3IamEndpoint string
s3IamSslNoVerify, s3IamDebug bool
iamCacheDisable bool
iamCacheTTL int
iamCachePrune int
)
var (
@@ -114,13 +101,10 @@ func main() {
func initApp() *cli.App {
return &cli.App{
Usage: "Versity S3 Gateway",
Description: `The Versity S3 Gateway is an S3 protocol translator that allows an S3 client
to access the supported backend storage as if it was a native S3 service.
VersityGW is an open-source project licensed under the Apache 2.0 License. The
source code is hosted on GitHub at https://github.com/versity/versitygw, and
documentation can be found in the GitHub wiki.`,
Copyright: "Copyright (c) 2023-2024 Versity Software",
Name: "versitygw",
Usage: "Start S3 gateway service with specified backend storage.",
Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
to access the supported backend storage as if it was a native S3 service.`,
Action: func(ctx *cli.Context) error {
return ctx.App.Command("help").Run(ctx)
},
@@ -228,12 +212,6 @@ func initFlags() []cli.Flag {
EnvVars: []string{"LOGFILE", "VGW_ACCESS_LOG"},
Destination: &accessLog,
},
&cli.StringFlag{
Name: "admin-access-log",
Usage: "enable admin server access logging to specified file",
EnvVars: []string{"LOGFILE", "VGW_ADMIN_ACCESS_LOG"},
Destination: &adminLogFile,
},
&cli.StringFlag{
Name: "log-webhook-url",
Usage: "webhook url to send the audit logs",
@@ -343,72 +321,6 @@ func initFlags() []cli.Flag {
EnvVars: []string{"VGW_IAM_LDAP_ROLE_ATR"},
Destination: &ldapRoleAtr,
},
&cli.StringFlag{
Name: "iam-ldap-user-id-atr",
Usage: "ldap server user id attribute name",
EnvVars: []string{"VGW_IAM_LDAP_USER_ID_ATR"},
Destination: &ldapUserIdAtr,
},
&cli.StringFlag{
Name: "iam-ldap-group-id-atr",
Usage: "ldap server user group id attribute name",
EnvVars: []string{"VGW_IAM_LDAP_GROUP_ID_ATR"},
Destination: &ldapGroupIdAtr,
},
&cli.StringFlag{
Name: "iam-vault-endpoint-url",
Usage: "vault server url",
EnvVars: []string{"VGW_IAM_VAULT_ENDPOINT_URL"},
Destination: &vaultEndpointURL,
},
&cli.StringFlag{
Name: "iam-vault-secret-storage-path",
Usage: "vault server secret storage path",
EnvVars: []string{"VGW_IAM_VAULT_SECRET_STORAGE_PATH"},
Destination: &vaultSecretStoragePath,
},
&cli.StringFlag{
Name: "iam-vault-mount-path",
Usage: "vault server mount path",
EnvVars: []string{"VGW_IAM_VAULT_MOUNT_PATH"},
Destination: &vaultMountPath,
},
&cli.StringFlag{
Name: "iam-vault-root-token",
Usage: "vault server root token",
EnvVars: []string{"VGW_IAM_VAULT_ROOT_TOKEN"},
Destination: &vaultRootToken,
},
&cli.StringFlag{
Name: "iam-vault-role-id",
Usage: "vault server user role id",
EnvVars: []string{"VGW_IAM_VAULT_ROLE_ID"},
Destination: &vaultRoleId,
},
&cli.StringFlag{
Name: "iam-vault-role-secret",
Usage: "vault server user role secret",
EnvVars: []string{"VGW_IAM_VAULT_ROLE_SECRET"},
Destination: &vaultRoleSecret,
},
&cli.StringFlag{
Name: "iam-vault-server_cert",
Usage: "vault server TLS certificate",
EnvVars: []string{"VGW_IAM_VAULT_SERVER_CERT"},
Destination: &vaultServerCert,
},
&cli.StringFlag{
Name: "iam-vault-client_cert",
Usage: "vault client TLS certificate",
EnvVars: []string{"VGW_IAM_VAULT_CLIENT_CERT"},
Destination: &vaultClientCert,
},
&cli.StringFlag{
Name: "iam-vault-client_cert_key",
Usage: "vault client TLS certificate key",
EnvVars: []string{"VGW_IAM_VAULT_CLIENT_CERT_KEY"},
Destination: &vaultClientCertKey,
},
&cli.StringFlag{
Name: "s3-iam-access",
Usage: "s3 IAM access key",
@@ -479,33 +391,6 @@ func initFlags() []cli.Flag {
EnvVars: []string{"VGW_HEALTH"},
Destination: &healthPath,
},
&cli.BoolFlag{
Name: "readonly",
Usage: "allow only read operations across all the gateway",
EnvVars: []string{"VGW_READ_ONLY"},
Destination: &readonly,
},
&cli.StringFlag{
Name: "metrics-service-name",
Usage: "service name tag for metrics, hostname if blank",
EnvVars: []string{"VGW_METRICS_SERVICE_NAME"},
Aliases: []string{"msn"},
Destination: &metricsService,
},
&cli.StringFlag{
Name: "metrics-statsd-servers",
Usage: "StatsD server urls comma separated. e.g. 'statsd1.example.com:8125,statsd2.example.com:8125'",
EnvVars: []string{"VGW_METRICS_STATSD_SERVERS"},
Aliases: []string{"mss"},
Destination: &statsdServers,
},
&cli.StringFlag{
Name: "metrics-dogstatsd-servers",
Usage: "DogStatsD server urls comma separated. e.g. '127.0.0.1:8125,dogstats.example.com:8125'",
EnvVars: []string{"VGW_METRICS_DOGSTATS_SERVERS"},
Aliases: []string{"mds"},
Destination: &dogstatsServers,
},
}
}
@@ -523,12 +408,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
}
app := fiber.New(fiber.Config{
AppName: "versitygw",
ServerHeader: "VERSITYGW",
StreamRequestBody: true,
DisableKeepalive: true,
Network: fiber.NetworkTCP,
DisableStartupMessage: true,
AppName: "versitygw",
ServerHeader: "VERSITYGW",
StreamRequestBody: true,
DisableKeepalive: true,
})
var opts []s3api.Option
@@ -559,15 +442,10 @@ func runGateway(ctx context.Context, be backend.Backend) error {
if healthPath != "" {
opts = append(opts, s3api.WithHealth(healthPath))
}
if readonly {
opts = append(opts, s3api.WithReadOnly())
}
admApp := fiber.New(fiber.Config{
AppName: "versitygw",
ServerHeader: "VERSITYGW",
Network: fiber.NetworkTCP,
DisableStartupMessage: true,
AppName: "versitygw",
ServerHeader: "VERSITYGW",
})
var admOpts []s3api.AdminOpt
@@ -588,64 +466,38 @@ func runGateway(ctx context.Context, be backend.Backend) error {
}
iam, err := auth.New(&auth.Opts{
RootAccount: auth.Account{
Access: rootUserAccess,
Secret: rootUserSecret,
Role: auth.RoleAdmin,
},
Dir: iamDir,
LDAPServerURL: ldapURL,
LDAPBindDN: ldapBindDN,
LDAPPassword: ldapPassword,
LDAPQueryBase: ldapQueryBase,
LDAPObjClasses: ldapObjClasses,
LDAPAccessAtr: ldapAccessAtr,
LDAPSecretAtr: ldapSecAtr,
LDAPRoleAtr: ldapRoleAtr,
LDAPUserIdAtr: ldapUserIdAtr,
LDAPGroupIdAtr: ldapGroupIdAtr,
VaultEndpointURL: vaultEndpointURL,
VaultSecretStoragePath: vaultSecretStoragePath,
VaultMountPath: vaultMountPath,
VaultRootToken: vaultRootToken,
VaultRoleId: vaultRoleId,
VaultRoleSecret: vaultRoleSecret,
VaultServerCert: vaultServerCert,
VaultClientCert: vaultClientCert,
VaultClientCertKey: vaultClientCertKey,
S3Access: s3IamAccess,
S3Secret: s3IamSecret,
S3Region: s3IamRegion,
S3Bucket: s3IamBucket,
S3Endpoint: s3IamEndpoint,
S3DisableSSlVerfiy: s3IamSslNoVerify,
S3Debug: s3IamDebug,
CacheDisable: iamCacheDisable,
CacheTTL: iamCacheTTL,
CachePrune: iamCachePrune,
Dir: iamDir,
LDAPServerURL: ldapURL,
LDAPBindDN: ldapBindDN,
LDAPPassword: ldapPassword,
LDAPQueryBase: ldapQueryBase,
LDAPObjClasses: ldapObjClasses,
LDAPAccessAtr: ldapAccessAtr,
LDAPSecretAtr: ldapSecAtr,
LDAPRoleAtr: ldapRoleAtr,
S3Access: s3IamAccess,
S3Secret: s3IamSecret,
S3Region: s3IamRegion,
S3Bucket: s3IamBucket,
S3Endpoint: s3IamEndpoint,
S3DisableSSlVerfiy: s3IamSslNoVerify,
S3Debug: s3IamDebug,
CacheDisable: iamCacheDisable,
CacheTTL: iamCacheTTL,
CachePrune: iamCachePrune,
})
if err != nil {
return fmt.Errorf("setup iam: %w", err)
}
loggers, err := s3log.InitLogger(&s3log.LogConfig{
LogFile: accessLog,
WebhookURL: logWebhookURL,
AdminLogFile: adminLogFile,
logger, err := s3log.InitLogger(&s3log.LogConfig{
LogFile: accessLog,
WebhookURL: logWebhookURL,
})
if err != nil {
return fmt.Errorf("setup logger: %w", err)
}
metricsManager, err := metrics.NewManager(ctx, metrics.Config{
ServiceName: metricsService,
StatsdServers: statsdServers,
DogStatsdServers: dogstatsServers,
})
if err != nil {
return fmt.Errorf("init metrics manager: %w", err)
}
evSender, err := s3event.InitEventSender(&s3event.EventConfig{
KafkaURL: kafkaURL,
KafkaTopic: kafkaTopic,
@@ -662,16 +514,12 @@ func runGateway(ctx context.Context, be backend.Backend) error {
srv, err := s3api.New(app, be, middlewares.RootUserConfig{
Access: rootUserAccess,
Secret: rootUserSecret,
}, port, region, iam, loggers.S3Logger, loggers.AdminLogger, evSender, metricsManager, opts...)
}, port, region, iam, logger, evSender, opts...)
if err != nil {
return fmt.Errorf("init gateway: %v", err)
}
admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, loggers.AdminLogger, admOpts...)
if !quiet {
printBanner(port, admPort, certFile != "", admCertFile != "")
}
admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
c := make(chan error, 2)
go func() { c <- srv.Serve() }()
@@ -688,17 +536,10 @@ Loop:
case err = <-c:
break Loop
case <-sigHup:
if loggers.S3Logger != nil {
err = loggers.S3Logger.HangUp()
if logger != nil {
err = logger.HangUp()
if err != nil {
err = fmt.Errorf("HUP s3 logger: %w", err)
break Loop
}
}
if loggers.AdminLogger != nil {
err = loggers.AdminLogger.HangUp()
if err != nil {
err = fmt.Errorf("HUP admin logger: %w", err)
err = fmt.Errorf("HUP logger: %w", err)
break Loop
}
}
@@ -716,22 +557,13 @@ Loop:
fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
}
if loggers.S3Logger != nil {
err := loggers.S3Logger.Shutdown()
if logger != nil {
err := logger.Shutdown()
if err != nil {
if saveErr == nil {
saveErr = err
}
fmt.Fprintf(os.Stderr, "shutdown s3 logger: %v\n", err)
}
}
if loggers.AdminLogger != nil {
err := loggers.AdminLogger.Shutdown()
if err != nil {
if saveErr == nil {
saveErr = err
}
fmt.Fprintf(os.Stderr, "shutdown admin logger: %v\n", err)
fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
}
}
@@ -745,183 +577,5 @@ Loop:
}
}
if metricsManager != nil {
metricsManager.Close()
}
return saveErr
}
func printBanner(port, admPort string, ssl, admSsl bool) {
interfaces, err := getMatchingIPs(port)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to match local IP addresses: %v\n", err)
return
}
var admInterfaces []string
if admPort != "" {
admInterfaces, err = getMatchingIPs(admPort)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to match admin port local IP addresses: %v\n", err)
return
}
}
title := "VersityGW"
version := fmt.Sprintf("Version %v, Build %v", Version, Build)
urls := []string{}
hst, prt, err := net.SplitHostPort(port)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
return
}
for _, ip := range interfaces {
url := fmt.Sprintf("http://%s:%s", ip, prt)
if ssl {
url = fmt.Sprintf("https://%s:%s", ip, prt)
}
urls = append(urls, url)
}
if hst == "" {
hst = "0.0.0.0"
}
boundHost := fmt.Sprintf("(bound on host %s and port %s)", hst, prt)
lines := []string{
centerText(title),
centerText(version),
centerText(boundHost),
centerText(""),
}
if len(admInterfaces) > 0 {
lines = append(lines,
leftText("S3 service listening on:"),
)
} else {
lines = append(lines,
leftText("Admin/S3 service listening on:"),
)
}
for _, url := range urls {
lines = append(lines, leftText(" "+url))
}
if len(admInterfaces) > 0 {
lines = append(lines,
centerText(""),
leftText("Admin service listening on:"),
)
_, prt, err := net.SplitHostPort(admPort)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to parse port: %v\n", err)
return
}
for _, ip := range admInterfaces {
url := fmt.Sprintf("http://%s:%s", ip, prt)
if admSsl {
url = fmt.Sprintf("https://%s:%s", ip, prt)
}
lines = append(lines, leftText(" "+url))
}
}
// Print the top border
fmt.Println("┌" + strings.Repeat("─", columnWidth-2) + "┐")
// Print each line
for _, line := range lines {
fmt.Printf("│%-*s│\n", columnWidth-2, line)
}
// Print the bottom border
fmt.Println("└" + strings.Repeat("─", columnWidth-2) + "┘")
}
// getMatchingIPs returns all IP addresses for local system interfaces that
// match the input address specification.
func getMatchingIPs(spec string) ([]string, error) {
// Split the input spec into IP and port
host, _, err := net.SplitHostPort(spec)
if err != nil {
return nil, fmt.Errorf("parse address/port: %v", err)
}
// Handle cases where IP is omitted (e.g., ":1234")
if host == "" {
host = "0.0.0.0"
}
ipaddr, err := net.ResolveIPAddr("ip", host)
if err != nil {
return nil, err
}
parsedInputIP := ipaddr.IP
var result []string
// Get all network interfaces
interfaces, err := net.Interfaces()
if err != nil {
return nil, err
}
for _, iface := range interfaces {
// Get all addresses associated with the interface
addrs, err := iface.Addrs()
if err != nil {
return nil, err
}
for _, addr := range addrs {
// Parse the address to get the IP part
ipAddr, _, err := net.ParseCIDR(addr.String())
if err != nil {
return nil, err
}
if ipAddr.IsLinkLocalUnicast() {
continue
}
if ipAddr.IsInterfaceLocalMulticast() {
continue
}
if ipAddr.IsLinkLocalMulticast() {
continue
}
// Check if the IP matches the input specification
if parsedInputIP.Equal(net.IPv4(0, 0, 0, 0)) || parsedInputIP.Equal(ipAddr) {
result = append(result, ipAddr.String())
}
}
}
return result, nil
}
const columnWidth = 70
func centerText(text string) string {
padding := (columnWidth - 2 - len(text)) / 2
if padding < 0 {
padding = 0
}
return strings.Repeat(" ", padding) + text
}
func leftText(text string) string {
if len(text) > columnWidth-2 {
return text
}
return text + strings.Repeat(" ", columnWidth-2-len(text))
}

View File

@@ -18,13 +18,12 @@ import (
"fmt"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/backend/posix"
)
var (
chownuid, chowngid bool
bucketlinks bool
readonly bool
)
func posixCommand() *cli.Command {
@@ -56,10 +55,10 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
Destination: &chowngid,
},
&cli.BoolFlag{
Name: "bucketlinks",
Usage: "allow symlinked directories at bucket level to be treated as buckets",
EnvVars: []string{"VGW_BUCKET_LINKS"},
Destination: &bucketlinks,
Name: "readonly",
Usage: "allow only read operations to backend",
EnvVars: []string{"VGW_READ_ONLY"},
Destination: &readonly,
},
},
}
@@ -70,16 +69,10 @@ func runPosix(ctx *cli.Context) error {
return fmt.Errorf("no directory provided for operation")
}
gwroot := (ctx.Args().Get(0))
err := meta.XattrMeta{}.Test(gwroot)
if err != nil {
return fmt.Errorf("posix xattr check: %v", err)
}
be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
ChownUID: chownuid,
ChownGID: chowngid,
BucketLinks: bucketlinks,
be, err := posix.New(ctx.Args().Get(0), posix.PosixOpts{
ChownUID: chownuid,
ChownGID: chowngid,
ReadOnly: readonly,
})
if err != nil {
return fmt.Errorf("init posix: %v", err)

View File

@@ -23,6 +23,10 @@ import (
var (
glacier bool
// defined in posix.go:
// chownuid, chowngid bool
// readonly bool
)
func scoutfsCommand() *cli.Command {
@@ -64,10 +68,10 @@ move interfaces as well as support for tiered filesystems.`,
Destination: &chowngid,
},
&cli.BoolFlag{
Name: "bucketlinks",
Usage: "allow symlinked directories at bucket level to be treated as buckets",
EnvVars: []string{"VGW_BUCKET_LINKS"},
Destination: &bucketlinks,
Name: "readonly",
Usage: "allow only read operations to backend",
EnvVars: []string{"VGW_READ_ONLY"},
Destination: &readonly,
},
},
}
@@ -82,7 +86,7 @@ func runScoutfs(ctx *cli.Context) error {
opts.GlacierMode = glacier
opts.ChownUID = chownuid
opts.ChownGID = chowngid
opts.BucketLinks = bucketlinks
opts.ReadOnly = readonly
be, err := scoutfs.New(ctx.Args().Get(0), opts)
if err != nil {

View File

@@ -54,24 +54,22 @@ func generateEventFiltersConfig(ctx *cli.Context) error {
}
config := s3event.EventFilter{
s3event.EventObjectCreated: true,
s3event.EventObjectCreatedPut: true,
s3event.EventObjectCreatedPost: true,
s3event.EventObjectCreatedCopy: true,
s3event.EventCompleteMultipartUpload: true,
s3event.EventObjectRemoved: true,
s3event.EventObjectRemovedDelete: true,
s3event.EventObjectRemovedDeleteObjects: true,
s3event.EventObjectTagging: true,
s3event.EventObjectTaggingPut: true,
s3event.EventObjectTaggingDelete: true,
s3event.EventObjectAclPut: true,
s3event.EventObjectRestore: true,
s3event.EventObjectRestorePost: true,
s3event.EventObjectRestoreCompleted: true,
s3event.EventObjectCreated: true,
s3event.EventObjectCreatedPut: true,
s3event.EventObjectCreatedPost: true,
s3event.EventObjectCreatedCopy: true,
s3event.EventCompleteMultipartUpload: true,
s3event.EventObjectDeleted: true,
s3event.EventObjectTagging: true,
s3event.EventObjectTaggingPut: true,
s3event.EventObjectTaggingDelete: true,
s3event.EventObjectAclPut: true,
s3event.EventObjectRestore: true,
s3event.EventObjectRestorePost: true,
s3event.EventObjectRestoreCompleted: true,
}
configBytes, err := json.MarshalIndent(config, "", " ")
configBytes, err := json.Marshal(config)
if err != nil {
return fmt.Errorf("parse event config: %w", err)
}

View File

@@ -1,36 +0,0 @@
version: '3'
services:
no_certs:
build:
context: .
dockerfile: Dockerfile_test_bats
args:
- CONFIG_FILE=tests/.env.nocerts
static_buckets:
build:
context: .
dockerfile: Dockerfile_test_bats
args:
- CONFIG_FILE=tests/.env.static
posix_backend:
build:
context: .
dockerfile: Dockerfile_test_bats
args:
- CONFIG_FILE=tests/.env.default
image: bats_test
s3_backend:
build:
context: .
dockerfile: Dockerfile_test_bats
args:
- CONFIG_FILE=tests/.env.s3
- SECRETS_FILE=tests/.secrets.s3
direct:
build:
context: .
dockerfile: Dockerfile_test_bats
args:
- CONFIG_FILE=tests/.env.direct
- SECRETS_FILE=tests/.secrets.direct

View File

@@ -1,64 +0,0 @@
services:
telegraf:
image: telegraf
container_name: telegraf
restart: always
volumes:
- ./metrics-exploration/telegraf.conf:/etc/telegraf/telegraf.conf:ro
depends_on:
- influxdb
links:
- influxdb
ports:
- '8125:8125/udp'
influxdb:
image: influxdb
container_name: influxdb
restart: always
environment:
- DOCKER_INFLUXDB_INIT_MODE=setup
- DOCKER_INFLUXDB_INIT_USERNAME=admin
- DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
- DOCKER_INFLUXDB_INIT_ORG=myorg
- DOCKER_INFLUXDB_INIT_BUCKET=metrics
- DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
ports:
- '8086:8086'
volumes:
- influxdb_data:/var/lib/influxdb
grafana:
image: grafana/grafana
container_name: grafana-server
restart: always
depends_on:
- influxdb
environment:
- GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_INSTALL_PLUGINS=
links:
- influxdb
ports:
- '3000:3000'
volumes:
- ./metrics-exploration/grafana_data:/var/lib/grafana
- ./metrics-exploration/provisioning:/etc/grafana/provisioning
versitygw:
image: versity/versitygw:latest
container_name: versitygw
ports:
- "7070:7070"
environment:
- ROOT_ACCESS_KEY=user
- ROOT_SECRET_KEY=password
- VGW_METRICS_STATSD_SERVERS=telegraf:8125
depends_on:
- telegraf
command: >
posix /tmp/vgw
volumes:
influxdb_data: {}

View File

@@ -31,7 +31,7 @@ services:
hostname: azurite
command: "azurite --oauth basic --cert /tests/certs/azurite.pem --key /tests/certs/azurite-key.pem --blobHost 0.0.0.0"
volumes:
- ./tests/certs:/tests/certs
- ./certs:/certs
azuritegw:
build:
context: .

View File

@@ -153,14 +153,6 @@ ROOT_SECRET_ACCESS_KEY=
# specified, all configured bucket events will be sent to the webhook.
#VGW_EVENT_WEBHOOK_URL=
# Bucket events can be filtered for any of the above event types. The
# VGW_EVENT_FILTER option specifies a config file that contains the event
# filter rules. The event filter rules are used to determine which events are
# sent to the configured event services. Run:
# versitygw utils gen-event-filter-config --path .
# to generate a default rules file "event_config.json" in the current directory.
#VGW_EVENT_FILTER=
#######################
# Debug / Diagnostics #
#######################
@@ -190,40 +182,7 @@ ROOT_SECRET_ACCESS_KEY=
# as a dedicated IAM service.
#VGW_IAM_DIR=
# The Vault options will enable the Vault IAM service with accounts stored in
# the HashiCorp Vault service. The Vault URL is the address and port of the
# Vault server with the format <IP/host>:<port>. A root taken can be used for
# testing, but it is recommended to use the role based authentication in
# production. The Vault server certificate, client certificate, and client
# certificate key are optional, and will default to not verifying the server
# certificate and not using client certificates. The Vault server certificate
# is used to verify the Vault server, and the client certificate and key are
# used to authenticate the gateway to the Vault server. See wiki documentation
# for an example of using Vault in dev mode with the gateway.
#VGW_IAM_VAULT_ENDPOINT_URL=
#VGW_IAM_VAULT_SECRET_STORAGE_PATH=
#VGW_IAM_VAULT_MOUNT_PATH=
#VGW_IAM_VAULT_ROOT_TOKEN=
#VGW_IAM_VAULT_ROLE_ID=
#VGW_IAM_VAULT_ROLE_SECRET=
#VGW_IAM_VAULT_SERVER_CERT=
#VGW_IAM_VAULT_CLIENT_CERT=
#VGW_IAM_VAULT_CLIENT_CERT_KEY=
# The VGW_S3 IAM service is similar to the internal IAM service, but instead
# stores the account information JSON encoded in an S3 object. This should use
# a bucket that is not accessible to general users when using s3 backend to
# prevent access to account credentials. This IAM service is added for
# convenience, but is not considered as secure or scalable as a dedicated IAM
# service.
#VGW_S3_IAM_ACCESS_KEY=
#VGW_S3_IAM_SECRET_KEY=
#VGW_S3_IAM_REGION=
#VGW_S3_IAM_ENDPOINT=
#VGW_S3_IAM_BUCKET=
#VGW_S3_IAM_NO_VERIFY=
# The LDAP options will enable the LDAP IAM service with accounts stored in an
# The ldap options will enable the LDAP IAM service with accounts stored in an
# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
# secret credentials and role respectively. The other options are used to
@@ -237,8 +196,19 @@ ROOT_SECRET_ACCESS_KEY=
#VGW_IAM_LDAP_ACCESS_ATR=
#VGW_IAM_LDAP_SECRET_ATR=
#VGW_IAM_LDAP_ROLE_ATR=
#VGW_IAM_LDAP_USER_ID_ATR=
#VGW_IAM_LDAP_GROUP_ID_ATR=
# The VGW_S3 IAM service is similar to the internal IAM service, but instead
# stores the account information JSON encoded in an S3 object. This should use
# a bucket that is not accessible to general users when using s3 backend to
# prevent access to account credentials. This IAM service is added for
# convenience, but is not considered as secure or scalable as a dedicated IAM
# service.
#VGW_S3_IAM_ACCESS_KEY=
#VGW_S3_IAM_SECRET_KEY=
#VGW_S3_IAM_REGION=
#VGW_S3_IAM_ENDPOINT=
#VGW_S3_IAM_BUCKET=
#VGW_S3_IAM_NO_VERIFY=
###############
# IAM caching #
@@ -258,29 +228,6 @@ ROOT_SECRET_ACCESS_KEY=
#VGW_IAM_CACHE_TTL=120
#VGW_IAM_CACHE_PRUNE=3600
###########
# Metrics #
###########
# The metrics service name is a tag that is added to all metrics to help
# identify the source of the metrics. This is especially useful when multiple
# gateways are running. The default is the hostname of the system.
#VGW_METRICS_SERVICE_NAME=$HOSTNAME
# The metrics service will send metrics to the configured statsd servers. The
# servers are specified as a comma separated list of host:port pairs. The
# default is to not send metrics to any statsd servers. The gateway uses
# InfluxDB flavor of statsd metrics tags for the StatsD metrics type.
#VGW_METRICS_STATSD_SERVERS=
# The metrics service will send metrics to the configured dogstatsd servers.
# The servers are specified as a comma separated list of host:port pairs. The
# default is to not send metrics to any dogstatsd servers. Generally
# DataDog recommends installing a local agent to collect metrics and forward
# them to the DataDog service. In this case the option value would be the
# local agent address: 127.0.0.1:8125.
#VGW_METRICS_DOGSTATS_SERVERS=
######################################
# VersityGW Backend Specific Options #
######################################
@@ -307,9 +254,11 @@ ROOT_SECRET_ACCESS_KEY=
#VGW_CHOWN_UID=false
#VGW_CHOWN_GID=false
# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
# to directories at the top level gateway directory as buckets.
#VGW_BUCKET_LINKS=false
# The VGW_READ_ONLY option will disable all write operations to the backend
# filesystem. This is useful for creating a read-only gateway for clients.
# This will prevent any PUT, POST, DELETE, and multipart upload operations
# for objects and buckets as well as preventing updating metadata for objects.
#VGW_READ_ONLY=false
###########
# scoutfs #
@@ -342,9 +291,11 @@ ROOT_SECRET_ACCESS_KEY=
#VGW_CHOWN_UID=false
#VGW_CHOWN_GID=false
# The VGW_BUCKET_LINKS option will enable the gateway to treat symbolic links
# to directories at the top level gateway directory as buckets.
#VGW_BUCKET_LINKS=false
# The VGW_READ_ONLY option will disable all write operations to the backend
# filesystem. This is useful for creating a read-only gateway for clients.
# This will prevent any PUT, POST, DELETE, and multipart upload operations
# for objects and buckets as well as preventing updating metadata for objects.
#VGW_READ_ONLY=false
######
# s3 #

88
go.mod
View File

@@ -1,80 +1,70 @@
module github.com/versity/versitygw
go 1.21.0
go 1.21
require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0
github.com/DataDog/datadog-go/v5 v5.5.0
github.com/aws/aws-sdk-go-v2 v1.30.4
github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1
github.com/aws/smithy-go v1.20.4
github.com/go-ldap/ldap/v3 v3.4.8
github.com/gofiber/fiber/v2 v2.52.5
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
github.com/aws/aws-sdk-go-v2 v1.26.1
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
github.com/aws/smithy-go v1.20.2
github.com/go-ldap/ldap/v3 v3.4.6
github.com/gofiber/fiber/v2 v2.52.4
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
github.com/hashicorp/vault-client-go v0.4.3
github.com/nats-io/nats.go v1.37.0
github.com/pkg/xattr v0.4.10
github.com/nats-io/nats.go v1.34.1
github.com/pkg/xattr v0.4.9
github.com/segmentio/kafka-go v0.4.47
github.com/smira/go-statsd v1.3.3
github.com/urfave/cli/v2 v2.27.4
github.com/valyala/fasthttp v1.55.0
github.com/urfave/cli/v2 v2.27.1
github.com/valyala/fasthttp v1.52.0
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
golang.org/x/sys v0.24.0
golang.org/x/sys v0.19.0
)
require (
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/nats-io/nkeys v0.4.7 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
golang.org/x/crypto v0.26.0 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/time v0.6.0 // indirect
golang.org/x/crypto v0.22.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/text v0.14.0 // indirect
)
require (
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
github.com/aws/aws-sdk-go-v2/config v1.27.31
github.com/aws/aws-sdk-go-v2/credentials v1.17.30
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.15
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
github.com/aws/aws-sdk-go-v2/config v1.27.11
github.com/aws/aws-sdk-go-v2/credentials v1.17.11
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
)

249
go.sum
View File

@@ -1,117 +1,86 @@
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 h1:Be6KInmFEKV81c0pOAEbRYehLMwmmGI1exuFj248AMk=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0/go.mod h1:WCPBHsOXfBVnivScjs2ypRfimjEW0qPVLGgJkZlrIOA=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU=
github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw=
github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8=
github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw=
github.com/aws/aws-sdk-go-v2/config v1.27.31 h1:kxBoRsjhT3pq0cKthgj6RU6bXTm/2SgdoUMyrVw0rAI=
github.com/aws/aws-sdk-go-v2/config v1.27.31/go.mod h1:z04nZdSWFPaDwK3DdJOG2r+scLQzMYuJeW0CujEm9FM=
github.com/aws/aws-sdk-go-v2/credentials v1.17.30 h1:aau/oYFtibVovr2rDt8FHlU17BTicFEMAi29V1U+L5Q=
github.com/aws/aws-sdk-go-v2/credentials v1.17.30/go.mod h1:BPJ/yXV92ZVq6G8uYvbU0gSl8q94UB63nMT5ctNO38g=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJCkMC0lMy6FaCD51jm6ayE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.15 h1:ijB7hr56MngOiELJe0C5aQRaBQ11LveNgWFyG02AUto=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.15/go.mod h1:0QEmQSSWMVfiAk93l1/ayR9DQ9+jwni7gHS2NARZXB0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGxcASiBd2MOp8m/dMc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16/go.mod h1:YHk6owoSwrIsok+cAH9PENCOGoH5PU2EllX4vLtSrsY=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 h1:GckUnpm4EJOAio1c8o25a+b3lVfwVzC9gnSBqiiNmZM=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18/go.mod h1:Br6+bxfG33Dk3ynmkhsW2Z/t9D4+lRqdLDNCKi85w0U=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 h1:tJ5RnkHCiSH0jyd6gROjlJtNwov0eGYNz8s8nFcR0jQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18/go.mod h1:++NHzT+nAF7ZPrHPsA+ENvsXkOO8wEu+C6RXltAG4/c=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 h1:jg16PhLPUiHIj8zYIW6bqzeQSuHVEiWnGA0Brz5Xv2I=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16/go.mod h1:Uyk1zE1VVdsHSU7096h/rwnXDzOzYQVl+FNPhPw7ShY=
github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1 h1:mx2ucgtv+MWzJesJY9Ig/8AFHgoE5FwLXwUVgW/FGdI=
github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1/go.mod h1:BSPI0EfnYUuNHPS0uqIo5VrRwzie+Fp+YhQOUs16sKI=
github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 h1:zCsFCKvbj25i7p1u94imVoO447I/sFv8qq+lGJhRN0c=
github.com/aws/aws-sdk-go-v2/service/sso v1.22.5/go.mod h1:ZeDX1SnKsVlejeuz41GiajjZpRSWR7/42q/EyA/QEiM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 h1:SKvPgvdvmiTWoi0GAJ7AsJfOz3ngVkD/ERbs5pUnHNI=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5/go.mod h1:20sz31hv/WsPa3HhU3hfrIet2kxM4Pe0r20eBZ20Tac=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 h1:OMsEmCyz2i89XwRwPouAJvhj81wINh+4UK+k/0Yo/q8=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.5/go.mod h1:vmSqFK+BVIwVpDAGZB3CoCXHzurt4qBE8lf+I/kRTh0=
github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ=
github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk=
github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY=
github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg=
github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=
github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8=
github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -119,12 +88,10 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/nats-io/nats.go v1.37.0 h1:07rauXbVnnJvv1gfIyghFEo6lUcYRY0WXc3x7x0vUxE=
github.com/nats-io/nats.go v1.37.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@@ -134,9 +101,8 @@ github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -144,30 +110,20 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smira/go-statsd v1.3.3 h1:WnMlmGTyMpzto+HvOJWRPoLaLlk5EGfzsnlQBcvj4yI=
github.com/smira/go-statsd v1.3.3/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8=
github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
@@ -178,47 +134,30 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -228,18 +167,16 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -247,21 +184,17 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -1,19 +0,0 @@
# Versity Gateway Dashboard
This project is a dashboard that visualizes data in the six metrics emitted by the Versity Gateway, displayed in Grafana.
The Versity Gateway emits metrics in the statsd format. We used Telegraf as the bridge from statsd to influxdb.
This implementation uses the influxql query language.
## Usage
From the root of this repository, run `docker compose -f docker-compose-metrics.yml up` to start the stack.
To shut it down, run `docker compose -f docker-compose-metrics.yml down -v`.
The Grafana database is explicitly not destroyed when shutting down containers. The influxdb one, however, is.
The dashbaord is automatically provisioned at container bring up and is visible at http://localhost:3000 with username: `admin` and password: `admin`.
To use the gateway and generate metrics, `source metrics-exploration/aws_env_setup.sh` and start using your aws cli as usual.

View File

@@ -1,4 +0,0 @@
export AWS_SECRET_ACCESS_KEY=password
export AWS_ACCESS_KEY_ID=user
export AWS_ENDPOINT_URL=http://127.0.0.1:7070
export AWS_REGION=us-east-1

View File

@@ -1,64 +0,0 @@
services:
telegraf:
image: telegraf
container_name: telegraf
restart: always
volumes:
- ./telegraf.conf:/etc/telegraf/telegraf.conf:ro
depends_on:
- influxdb
links:
- influxdb
ports:
- '8125:8125/udp'
influxdb:
image: influxdb
container_name: influxdb
restart: always
environment:
- DOCKER_INFLUXDB_INIT_MODE=setup
- DOCKER_INFLUXDB_INIT_USERNAME=admin
- DOCKER_INFLUXDB_INIT_PASSWORD=adminpass
- DOCKER_INFLUXDB_INIT_ORG=myorg
- DOCKER_INFLUXDB_INIT_BUCKET=metrics
- DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token
ports:
- '8086:8086'
volumes:
- influxdb_data:/var/lib/influxdb
grafana:
image: grafana/grafana
container_name: grafana-server
restart: always
depends_on:
- influxdb
environment:
- GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_INSTALL_PLUGINS=
links:
- influxdb
ports:
- '3000:3000'
volumes:
- ./grafana_data:/var/lib/grafana
- ./provisioning:/etc/grafana/provisioning
versitygw:
image: versity/versitygw:latest
container_name: versitygw
ports:
- "7070:7070"
environment:
- ROOT_ACCESS_KEY=user
- ROOT_SECRET_KEY=password
- VGW_METRICS_STATSD_SERVERS=telegraf:8125
depends_on:
- telegraf
command: >
posix /tmp/vgw
volumes:
influxdb_data: {}

View File

@@ -1,25 +0,0 @@
apiVersion: 1
providers:
# <string> an unique provider name. Required
- name: 'influxql'
# <int> Org id. Default to 1
orgId: 1
# <string> name of the dashboard folder.
folder: 'influxql'
# <string> folder UID. will be automatically generated if not specified
folderUid: ''
# <string> provider type. Default to 'file'
type: file
# <bool> disable dashboard deletion
disableDeletion: false
# <int> how often Grafana will scan for changed dashboards
updateIntervalSeconds: 10
# <bool> allow updating provisioned dashboards from the UI
allowUiUpdates: true
options:
# <string, required> path to dashboard files on disk. Required when using the 'file' type
path: /etc/grafana/provisioning/dashboards/influxql
# <bool> use folder names from filesystem to create folders in Grafana
foldersFromFilesStructure: true

View File

@@ -1,13 +0,0 @@
apiVersion: 1
datasources:
- name: influxdb
type: influxdb
isDefault: true
access: proxy
url: http://influxdb:8086
jsonData:
dbName: 'metrics'
httpHeaderName1: 'Authorization'
secureJsonData:
httpHeaderValue1: 'Token my-super-secret-auth-token'

View File

@@ -1,34 +0,0 @@
[global_tags]
[agent]
debug = true
quiet = false
interval = "60s"
round_interval = true
metric_batch_size = 1000
metric_buffer_limit = 10000
collection_jitter = "0s"
flush_interval = "10s"
flush_jitter = "0s"
precision = ""
hostname = "versitygw"
omit_hostname = false
[[outputs.file]]
files = ["stdout"]
[[outputs.influxdb_v2]]
urls = ["http://influxdb:8086"]
timeout = "5s"
token = "my-super-secret-auth-token"
organization = "myorg"
bucket = "metrics"
[[inputs.statsd]]
protocol = "udp4"
service_address = ":8125"
percentiles = [90]
metric_separator = "_"
datadog_extensions = false
allowed_pending_messages = 10000
percentile_limit = 1000

View File

@@ -1,6 +0,0 @@
#!/usr/bin/env bash
. ./aws_env_setup.sh
aws s3 mb s3://test
aws s3 cp docker-compose.yml s3://test/test.yaml

View File

@@ -1,261 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package metrics
type Action struct {
Name string
Service string
}
var (
ActionMap map[string]Action
)
var (
ActionUndetected = "ActionUnDetected"
ActionAbortMultipartUpload = "s3_AbortMultipartUpload"
ActionCompleteMultipartUpload = "s3_CompleteMultipartUpload"
ActionCopyObject = "s3_CopyObject"
ActionCreateBucket = "s3_CreateBucket"
ActionCreateMultipartUpload = "s3_CreateMultipartUpload"
ActionDeleteBucket = "s3_DeleteBucket"
ActionDeleteBucketPolicy = "s3_DeleteBucketPolicy"
ActionDeleteBucketTagging = "s3_DeleteBucketTagging"
ActionDeleteObject = "s3_DeleteObject"
ActionDeleteObjectTagging = "s3_DeleteObjectTagging"
ActionDeleteObjects = "s3_DeleteObjects"
ActionGetBucketAcl = "s3_GetBucketAcl"
ActionGetBucketPolicy = "s3_GetBucketPolicy"
ActionGetBucketTagging = "s3_GetBucketTagging"
ActionGetBucketVersioning = "s3_GetBucketVersioning"
ActionGetObject = "s3_GetObject"
ActionGetObjectAcl = "s3_GetObjectAcl"
ActionGetObjectAttributes = "s3_GetObjectAttributes"
ActionGetObjectLegalHold = "s3_GetObjectLegalHold"
ActionGetObjectLockConfiguration = "s3_GetObjectLockConfiguration"
ActionGetObjectRetention = "s3_GetObjectRetention"
ActionGetObjectTagging = "s3_GetObjectTagging"
ActionHeadBucket = "s3_HeadBucket"
ActionHeadObject = "s3_HeadObject"
ActionListAllMyBuckets = "s3_ListAllMyBuckets"
ActionListMultipartUploads = "s3_ListMultipartUploads"
ActionListObjectVersions = "s3_ListObjectVersions"
ActionListObjects = "s3_ListObjects"
ActionListObjectsV2 = "s3_ListObjectsV2"
ActionListParts = "s3_ListParts"
ActionPutBucketAcl = "s3_PutBucketAcl"
ActionPutBucketPolicy = "s3_PutBucketPolicy"
ActionPutBucketTagging = "s3_PutBucketTagging"
ActionPutBucketVersioning = "s3_PutBucketVersioning"
ActionPutObject = "s3_PutObject"
ActionPutObjectAcl = "s3_PutObjectAcl"
ActionPutObjectLegalHold = "s3_PutObjectLegalHold"
ActionPutObjectLockConfiguration = "s3_PutObjectLockConfiguration"
ActionPutObjectRetention = "s3_PutObjectRetention"
ActionPutObjectTagging = "s3_PutObjectTagging"
ActionRestoreObject = "s3_RestoreObject"
ActionSelectObjectContent = "s3_SelectObjectContent"
ActionUploadPart = "s3_UploadPart"
ActionUploadPartCopy = "s3_UploadPartCopy"
ActionPutBucketOwnershipControls = "s3_PutBucketOwnershipControls"
ActionGetBucketOwnershipControls = "s3_GetBucketOwnershipControls"
ActionDeleteBucketOwnershipControls = "s3_DeleteBucketOwnershipControls"
)
func init() {
ActionMap = make(map[string]Action)
ActionMap[ActionUndetected] = Action{
Name: "ActionUnDetected",
Service: "unknown",
}
ActionMap[ActionAbortMultipartUpload] = Action{
Name: "AbortMultipartUpload",
Service: "s3",
}
ActionMap[ActionCompleteMultipartUpload] = Action{
Name: "CompleteMultipartUpload",
Service: "s3",
}
ActionMap[ActionCopyObject] = Action{
Name: "CopyObject",
Service: "s3",
}
ActionMap[ActionCreateBucket] = Action{
Name: "CreateBucket",
Service: "s3",
}
ActionMap[ActionCreateMultipartUpload] = Action{
Name: "CreateMultipartUpload",
Service: "s3",
}
ActionMap[ActionDeleteBucket] = Action{
Name: "DeleteBucket",
Service: "s3",
}
ActionMap[ActionDeleteBucketPolicy] = Action{
Name: "DeleteBucketPolicy",
Service: "s3",
}
ActionMap[ActionDeleteBucketTagging] = Action{
Name: "DeleteBucketTagging",
Service: "s3",
}
ActionMap[ActionDeleteObject] = Action{
Name: "DeleteObject",
Service: "s3",
}
ActionMap[ActionDeleteObjectTagging] = Action{
Name: "DeleteObjectTagging",
Service: "s3",
}
ActionMap[ActionDeleteObjects] = Action{
Name: "DeleteObjects",
Service: "s3",
}
ActionMap[ActionGetBucketAcl] = Action{
Name: "GetBucketAcl",
Service: "s3",
}
ActionMap[ActionGetBucketPolicy] = Action{
Name: "GetBucketPolicy",
Service: "s3",
}
ActionMap[ActionGetBucketTagging] = Action{
Name: "GetBucketTagging",
Service: "s3",
}
ActionMap[ActionGetBucketVersioning] = Action{
Name: "GetBucketVersioning",
Service: "s3",
}
ActionMap[ActionGetObject] = Action{
Name: "GetObject",
Service: "s3",
}
ActionMap[ActionGetObjectAcl] = Action{
Name: "GetObjectAcl",
Service: "s3",
}
ActionMap[ActionGetObjectAttributes] = Action{
Name: "GetObjectAttributes",
Service: "s3",
}
ActionMap[ActionGetObjectLegalHold] = Action{
Name: "GetObjectLegalHold",
Service: "s3",
}
ActionMap[ActionGetObjectLockConfiguration] = Action{
Name: "GetObjectLockConfiguration",
Service: "s3",
}
ActionMap[ActionGetObjectRetention] = Action{
Name: "GetObjectRetention",
Service: "s3",
}
ActionMap[ActionGetObjectTagging] = Action{
Name: "GetObjectTagging",
Service: "s3",
}
ActionMap[ActionHeadBucket] = Action{
Name: "HeadBucket",
Service: "s3",
}
ActionMap[ActionHeadObject] = Action{
Name: "HeadObject",
Service: "s3",
}
ActionMap[ActionListAllMyBuckets] = Action{
Name: "ListAllMyBuckets",
Service: "s3",
}
ActionMap[ActionListMultipartUploads] = Action{
Name: "ListMultipartUploads",
Service: "s3",
}
ActionMap[ActionListObjectVersions] = Action{
Name: "ListObjectVersions",
Service: "s3",
}
ActionMap[ActionListObjects] = Action{
Name: "ListObjects",
Service: "s3",
}
ActionMap[ActionListObjectsV2] = Action{
Name: "ListObjectsV2",
Service: "s3",
}
ActionMap[ActionListParts] = Action{
Name: "ListParts",
Service: "s3",
}
ActionMap[ActionPutBucketAcl] = Action{
Name: "PutBucketAcl",
Service: "s3",
}
ActionMap[ActionPutBucketPolicy] = Action{
Name: "PutBucketPolicy",
Service: "s3",
}
ActionMap[ActionPutBucketTagging] = Action{
Name: "PutBucketTagging",
Service: "s3",
}
ActionMap[ActionPutBucketVersioning] = Action{
Name: "PutBucketVersioning",
Service: "s3",
}
ActionMap[ActionPutObject] = Action{
Name: "PutObject",
Service: "s3",
}
ActionMap[ActionPutObjectAcl] = Action{
Name: "PutObjectAcl",
Service: "s3",
}
ActionMap[ActionPutObjectLegalHold] = Action{
Name: "PutObjectLegalHold",
Service: "s3",
}
ActionMap[ActionPutObjectLockConfiguration] = Action{
Name: "PutObjectLockConfiguration",
Service: "s3",
}
ActionMap[ActionPutObjectRetention] = Action{
Name: "PutObjectRetention",
Service: "s3",
}
ActionMap[ActionPutObjectTagging] = Action{
Name: "PutObjectTagging",
Service: "s3",
}
ActionMap[ActionRestoreObject] = Action{
Name: "RestoreObject",
Service: "s3",
}
ActionMap[ActionSelectObjectContent] = Action{
Name: "SelectObjectContent",
Service: "s3",
}
ActionMap[ActionUploadPart] = Action{
Name: "UploadPart",
Service: "s3",
}
ActionMap[ActionUploadPartCopy] = Action{
Name: "UploadPartCopy",
Service: "s3",
}
}

View File

@@ -1,65 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package metrics
import (
"fmt"
dogstats "github.com/DataDog/datadog-go/v5/statsd"
)
// vgwDogStatsd metrics type
type vgwDogStatsd struct {
c *dogstats.Client
}
var (
rateSampleAlways = 1.0
)
// newDogStatsd takes a server address and returns a statsd merics
func newDogStatsd(server string, service string) (*vgwDogStatsd, error) {
c, err := dogstats.New(server,
dogstats.WithMaxMessagesPerPayload(1000),
dogstats.WithNamespace("versitygw"),
dogstats.WithTags([]string{
"service:" + service,
}))
if err != nil {
return nil, err
}
return &vgwDogStatsd{c: c}, nil
}
// Close closes statsd connections
func (s *vgwDogStatsd) Close() {
s.c.Close()
}
func (t Tag) ddString() string {
if t.Value == "" {
return t.Key
}
return fmt.Sprintf("%v:%v", t.Key, t.Value)
}
// Add adds value to key
func (s *vgwDogStatsd) Add(key string, value int64, tags ...Tag) {
stags := make([]string, len(tags))
for i, t := range tags {
stags[i] = t.ddString()
}
s.c.Count(key, value, stags, rateSampleAlways)
}

View File

@@ -1,225 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package metrics
import (
"context"
"errors"
"fmt"
"net/http"
"os"
"strings"
"sync"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3err"
)
var (
// max size of data items to buffer before dropping
// new incoming data items
dataItemCount = 100000
)
// Tag is added metadata for metrics
type Tag struct {
// Key is tag name
Key string
// Value is tag data
Value string
}
// Manager is a manager of metrics plugins
type Manager struct {
wg sync.WaitGroup
ctx context.Context
config Config
publishers []publisher
addDataChan chan datapoint
}
type Config struct {
ServiceName string
StatsdServers string
DogStatsdServers string
}
// NewManager initializes metrics plugins and returns a new metrics manager
func NewManager(ctx context.Context, conf Config) (*Manager, error) {
if len(conf.StatsdServers) == 0 && len(conf.DogStatsdServers) == 0 {
return nil, nil
}
if conf.ServiceName == "" {
hostname, err := os.Hostname()
if err != nil {
return nil, fmt.Errorf("failed to get hostname: %w", err)
}
conf.ServiceName = hostname
}
addDataChan := make(chan datapoint, dataItemCount)
mgr := &Manager{
addDataChan: addDataChan,
ctx: ctx,
config: conf,
}
// setup statsd endpoints
if len(conf.StatsdServers) > 0 {
statsdServers := strings.Split(conf.StatsdServers, ",")
for _, server := range statsdServers {
statsd, err := newStatsd(server, conf.ServiceName)
if err != nil {
return nil, err
}
mgr.publishers = append(mgr.publishers, statsd)
}
}
// setup dogstatsd endpoints
if len(conf.DogStatsdServers) > 0 {
dogStatsdServers := strings.Split(conf.DogStatsdServers, ",")
for _, server := range dogStatsdServers {
dogStatsd, err := newDogStatsd(server, conf.ServiceName)
if err != nil {
return nil, err
}
mgr.publishers = append(mgr.publishers, dogStatsd)
}
}
mgr.wg.Add(1)
go mgr.addForwarder(addDataChan)
return mgr, nil
}
func (m *Manager) Send(ctx *fiber.Ctx, err error, action string, count int64, status int) {
// In case of Authentication failures, url parsing ...
if action == "" {
action = ActionUndetected
}
a := ActionMap[action]
reqTags := []Tag{
{Key: "method", Value: ctx.Method()},
{Key: "api", Value: a.Service},
{Key: "action", Value: a.Name},
}
reqStatus := status
if err != nil {
var apierr s3err.APIError
if errors.As(err, &apierr) {
reqStatus = apierr.HTTPStatusCode
} else {
reqStatus = http.StatusInternalServerError
}
}
if reqStatus == 0 {
reqStatus = http.StatusOK
}
reqTags = append(reqTags, Tag{
Key: "status",
Value: fmt.Sprintf("%v", reqStatus),
})
if err != nil {
m.increment("failed_count", reqTags...)
} else {
m.increment("success_count", reqTags...)
}
switch action {
case ActionPutObject:
m.add("bytes_written", count, reqTags...)
m.increment("object_created_count", reqTags...)
case ActionCompleteMultipartUpload:
m.increment("object_created_count", reqTags...)
case ActionUploadPart:
m.add("bytes_written", count, reqTags...)
case ActionGetObject:
m.add("bytes_read", count, reqTags...)
case ActionDeleteObject:
m.increment("object_removed_count", reqTags...)
case ActionDeleteObjects:
m.add("object_removed_count", count, reqTags...)
}
}
// increment increments the key by one
func (m *Manager) increment(key string, tags ...Tag) {
m.add(key, 1, tags...)
}
// add adds value to key
func (m *Manager) add(key string, value int64, tags ...Tag) {
if m.ctx.Err() != nil {
return
}
d := datapoint{
key: key,
value: value,
tags: tags,
}
select {
case m.addDataChan <- d:
default:
// channel full, drop the updates
}
}
// Close closes metrics channels, waits for data to complete, closes all plugins
func (m *Manager) Close() {
// drain the datapoint channels
close(m.addDataChan)
m.wg.Wait()
// close all publishers
for _, p := range m.publishers {
p.Close()
}
}
// publisher is the interface for interacting with the metrics plugins
type publisher interface {
Add(key string, value int64, tags ...Tag)
Close()
}
func (m *Manager) addForwarder(addChan <-chan datapoint) {
for data := range addChan {
for _, s := range m.publishers {
s.Add(data.key, data.value, data.tags...)
}
}
m.wg.Done()
}
type datapoint struct {
key string
value int64
tags []Tag
}

View File

@@ -1,51 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package metrics
import (
"github.com/smira/go-statsd"
)
// vgwStatsd metrics type
type vgwStatsd struct {
c *statsd.Client
}
// newStatsd takes a server address and returns a statsd merics
// Supply service name to be used as a tag to identify the spcific
// gateway instance, this may typically be the gateway hostname
func newStatsd(server string, service string) (*vgwStatsd, error) {
c := statsd.NewClient(
server,
statsd.MetricPrefix("versitygw."),
statsd.TagStyle(statsd.TagFormatInfluxDB),
statsd.DefaultTags(statsd.StringTag("service", service)),
)
return &vgwStatsd{c: c}, nil
}
// Close closes statsd connections
func (s *vgwStatsd) Close() {
s.c.Close()
}
// Add adds value to key
func (s *vgwStatsd) Add(key string, value int64, tags ...Tag) {
stags := make([]statsd.Tag, len(tags))
for i, t := range tags {
stags[i] = statsd.StringTag(t.Key, t.Value)
}
s.c.Incr(key, value, stags...)
}

View File

@@ -19,13 +19,12 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3log"
)
type S3AdminRouter struct{}
func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger) {
controller := controllers.NewAdminController(iam, be, logger)
func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
controller := controllers.NewAdminController(iam, be)
// CreateUser admin api
app.Patch("/create-user", controller.CreateUser)
@@ -33,9 +32,6 @@ func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMSe
// DeleteUsers admin api
app.Patch("/delete-user", controller.DeleteUser)
// UpdateUser admin api
app.Patch("/update-user", controller.UpdateUser)
// ListUsers admin api
app.Patch("/list-users", controller.ListUsers)

View File

@@ -22,7 +22,6 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3log"
)
type S3AdminServer struct {
@@ -33,7 +32,7 @@ type S3AdminServer struct {
cert *tls.Certificate
}
func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, opts ...AdminOpt) *S3AdminServer {
func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
server := &S3AdminServer{
app: app,
backend: be,
@@ -47,13 +46,14 @@ func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUse
// Logging middlewares
app.Use(logger.New())
app.Use(middlewares.DecodeURL(l, nil))
app.Use(middlewares.DecodeURL(nil))
// Authentication middlewares
app.Use(middlewares.VerifyV4Signature(root, iam, l, nil, region, false))
app.Use(middlewares.VerifyMD5Body(l))
app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
app.Use(middlewares.VerifyMD5Body(nil))
app.Use(middlewares.AclParser(be, nil))
server.router.Init(app, be, iam, l)
server.router.Init(app, be, iam)
return server
}

View File

@@ -16,310 +16,107 @@ package controllers
import (
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3log"
)
type AdminController struct {
iam auth.IAMService
be backend.Backend
l s3log.AuditLogger
}
func NewAdminController(iam auth.IAMService, be backend.Backend, l s3log.AuditLogger) AdminController {
return AdminController{iam: iam, be: be, l: l}
func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
return AdminController{iam: iam, be: be}
}
func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusForbidden,
action: "admin:CreateUser",
})
return fmt.Errorf("access denied: only admin users have access to this resource")
}
var usr auth.Account
err := json.Unmarshal(ctx.Body(), &usr)
if err != nil {
return sendResponse(ctx, fmt.Errorf("failed to parse request body: %w", err), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusBadRequest,
action: "admin:CreateUser",
})
return fmt.Errorf("failed to parse request body: %w", err)
}
if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
return sendResponse(ctx, errors.New("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusBadRequest,
action: "admin:CreateUser",
})
return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
}
err = c.iam.CreateAccount(usr)
if err != nil {
status := fiber.StatusInternalServerError
err = fmt.Errorf("failed to create user: %w", err)
if strings.Contains(err.Error(), "user already exists") {
status = fiber.StatusConflict
}
return sendResponse(ctx, err, nil,
&metaOptions{
status: status,
logger: c.l,
action: "admin:CreateUser",
})
return fmt.Errorf("failed to create a user: %w", err)
}
return sendResponse(ctx, nil, "The user has been created successfully", &metaOptions{
status: fiber.StatusCreated,
logger: c.l,
action: "admin:CreateUser",
})
}
func (c AdminController) UpdateUser(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusForbidden,
action: "admin:UpdateUser",
})
}
access := ctx.Query("access")
if access == "" {
return sendResponse(ctx, errors.New("missing user access parameter"), nil,
&metaOptions{
status: fiber.StatusBadRequest,
logger: c.l,
action: "admin:UpdateUser",
})
}
var props auth.MutableProps
if err := json.Unmarshal(ctx.Body(), &props); err != nil {
return sendResponse(ctx, fmt.Errorf("invalid request body %w", err), nil,
&metaOptions{
status: fiber.StatusBadRequest,
logger: c.l,
action: "admin:UpdateUser",
})
}
err := c.iam.UpdateUserAccount(access, props)
if err != nil {
status := fiber.StatusInternalServerError
err = fmt.Errorf("failed to update user account: %w", err)
if strings.Contains(err.Error(), "user not found") {
status = fiber.StatusNotFound
}
return sendResponse(ctx, err, nil,
&metaOptions{
status: status,
logger: c.l,
action: "admin:UpdateUser",
})
}
return sendResponse(ctx, nil, "the user has been updated successfully",
&metaOptions{
logger: c.l,
action: "admin:UpdateUser",
})
return ctx.SendString("The user has been created successfully")
}
func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
access := ctx.Query("access")
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusForbidden,
action: "admin:DeleteUser",
})
return fmt.Errorf("access denied: only admin users have access to this resource")
}
err := c.iam.DeleteUserAccount(access)
if err != nil {
return sendResponse(ctx, err, nil,
&metaOptions{
logger: c.l,
action: "admin:DeleteUser",
})
return err
}
return sendResponse(ctx, nil, "The user has been deleted successfully",
&metaOptions{
logger: c.l,
action: "admin:DeleteUser",
})
return ctx.SendString("The user has been deleted successfully")
}
func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusForbidden,
action: "admin:ListUsers",
})
return fmt.Errorf("access denied: only admin users have access to this resource")
}
accs, err := c.iam.ListUserAccounts()
return sendResponse(ctx, err, accs,
&metaOptions{
logger: c.l,
action: "admin:ListUsers",
})
if err != nil {
return err
}
return ctx.JSON(accs)
}
func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusForbidden,
action: "admin:ChangeBucketOwner",
})
return fmt.Errorf("access denied: only admin users have access to this resource")
}
owner := ctx.Query("owner")
bucket := ctx.Query("bucket")
accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
if err != nil {
return sendResponse(ctx, err, nil,
&metaOptions{
logger: c.l,
action: "admin:ChangeBucketOwner",
})
return err
}
if len(accs) > 0 {
return sendResponse(ctx, errors.New("user specified as the new bucket owner does not exist"), nil,
&metaOptions{
logger: c.l,
action: "admin:ChangeBucketOwner",
status: fiber.StatusNotFound,
})
return fmt.Errorf("user specified as the new bucket owner does not exist")
}
acl := auth.ACL{
Owner: owner,
Grantees: []auth.Grantee{
{
Permission: types.PermissionFullControl,
Access: owner,
Type: types.TypeCanonicalUser,
},
},
}
aclParsed, err := json.Marshal(acl)
err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
if err != nil {
return sendResponse(ctx, fmt.Errorf("failed to marshal the bucket acl: %w", err), nil,
&metaOptions{
logger: c.l,
action: "admin:ChangeBucketOwner",
})
return err
}
err = c.be.ChangeBucketOwner(ctx.Context(), bucket, aclParsed)
return sendResponse(ctx, err, "Bucket owner has been updated successfully",
&metaOptions{
logger: c.l,
action: "admin:ChangeBucketOwner",
})
return ctx.Status(201).SendString("Bucket owner has been updated successfully")
}
func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
acct := ctx.Locals("account").(auth.Account)
if acct.Role != "admin" {
return sendResponse(ctx, errors.New("access denied: only admin users have access to this resource"), nil,
&metaOptions{
logger: c.l,
status: fiber.StatusForbidden,
action: "admin:ListBuckets",
})
return fmt.Errorf("access denied: only admin users have access to this resource")
}
buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
return sendResponse(ctx, err, buckets,
&metaOptions{
logger: c.l,
action: "admin:ListBuckets",
})
}
type metaOptions struct {
action string
status int
logger s3log.AuditLogger
}
func sendResponse(ctx *fiber.Ctx, err error, data any, m *metaOptions) error {
status := m.status
if err != nil {
if status == 0 {
status = fiber.StatusInternalServerError
}
if m.logger != nil {
m.logger.Log(ctx, err, []byte(err.Error()), s3log.LogMeta{
Action: m.action,
HttpStatus: status,
})
}
return ctx.Status(status).SendString(err.Error())
}
if status == 0 {
status = fiber.StatusOK
}
msg, ok := data.(string)
if ok {
if m.logger != nil {
m.logger.Log(ctx, nil, []byte(msg), s3log.LogMeta{
Action: m.action,
HttpStatus: status,
})
}
return ctx.Status(status).SendString(msg)
}
dataJSON, err := json.Marshal(data)
if err != nil {
return err
}
if m.logger != nil {
m.logger.Log(ctx, nil, dataJSON, s3log.LogMeta{
HttpStatus: status,
Action: m.action,
})
}
ctx.Set(fiber.HeaderContentType, fiber.MIMEApplicationJSON)
return ctx.Status(status).Send(dataJSON)
return ctx.JSON(buckets)
}

View File

@@ -85,7 +85,7 @@ func TestAdminController_CreateUser(t *testing.T) {
req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(succUsr)),
},
wantErr: false,
statusCode: 201,
statusCode: 200,
},
{
name: "Admin-create-user-invalid-user-role",
@@ -94,7 +94,7 @@ func TestAdminController_CreateUser(t *testing.T) {
req: httptest.NewRequest(http.MethodPatch, "/create-user", bytes.NewBuffer(user)),
},
wantErr: false,
statusCode: 400,
statusCode: 500,
},
{
name: "Admin-create-user-invalid-requester-role",
@@ -103,7 +103,7 @@ func TestAdminController_CreateUser(t *testing.T) {
req: httptest.NewRequest(http.MethodPatch, "/create-user", nil),
},
wantErr: false,
statusCode: 403,
statusCode: 500,
},
}
for _, tt := range tests {
@@ -119,122 +119,6 @@ func TestAdminController_CreateUser(t *testing.T) {
}
}
func TestAdminController_UpdateUser(t *testing.T) {
type args struct {
req *http.Request
}
adminController := AdminController{
iam: &IAMServiceMock{
UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
return nil
},
},
}
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
app.Patch("/update-user", adminController.UpdateUser)
appErr := fiber.New()
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "user1", Secret: "secret", Role: "user"})
return ctx.Next()
})
appErr.Patch("/update-user", adminController.UpdateUser)
successBody, _ := json.Marshal(auth.MutableProps{Secret: getPtr("hello")})
adminControllerErr := AdminController{
iam: &IAMServiceMock{
UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
return auth.ErrNoSuchUser
},
},
}
appNotFound := fiber.New()
appNotFound.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("account", auth.Account{Access: "admin1", Secret: "secret", Role: "admin"})
return ctx.Next()
})
appNotFound.Patch("/update-user", adminControllerErr.UpdateUser)
tests := []struct {
name string
app *fiber.App
args args
wantErr bool
statusCode int
}{
{
name: "Admin-update-user-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Admin-update-user-missing-access",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/update-user", bytes.NewBuffer(successBody)),
},
wantErr: false,
statusCode: 400,
},
{
name: "Admin-update-user-invalid-request-body",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Admin-update-user-invalid-requester-role",
app: appErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", nil),
},
wantErr: false,
statusCode: 403,
},
{
name: "Admin-update-user-not-found",
app: appNotFound,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/update-user?access=access", bytes.NewBuffer(successBody)),
},
wantErr: false,
statusCode: 404,
},
}
for _, tt := range tests {
resp, err := tt.app.Test(tt.args.req)
if (err != nil) != tt.wantErr {
t.Errorf("AdminController.UpdateUser() error = %v, wantErr %v", err, tt.wantErr)
}
if resp.StatusCode != tt.statusCode {
t.Errorf("AdminController.UpdateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
}
}
}
func TestAdminController_DeleteUser(t *testing.T) {
type args struct {
req *http.Request
@@ -289,7 +173,7 @@ func TestAdminController_DeleteUser(t *testing.T) {
req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
},
wantErr: false,
statusCode: 403,
statusCode: 500,
},
}
for _, tt := range tests {
@@ -367,7 +251,7 @@ func TestAdminController_ListUsers(t *testing.T) {
req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
},
wantErr: false,
statusCode: 403,
statusCode: 500,
},
{
name: "Admin-list-users-iam-error",
@@ -407,7 +291,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
}
adminController := AdminController{
be: &BackendMock{
ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error {
ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, newOwner string) error {
return nil
},
},
@@ -484,7 +368,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
},
wantErr: false,
statusCode: 403,
statusCode: 500,
},
{
name: "Change-bucket-owner-check-account-server-error",
@@ -502,7 +386,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
},
wantErr: false,
statusCode: 404,
statusCode: 500,
},
{
name: "Change-bucket-owner-success",
@@ -511,7 +395,7 @@ func TestAdminController_ChangeBucketOwner(t *testing.T) {
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
},
wantErr: false,
statusCode: 200,
statusCode: 201,
},
}
for _, tt := range tests {
@@ -571,7 +455,7 @@ func TestAdminController_ListBuckets(t *testing.T) {
req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
},
wantErr: false,
statusCode: 403,
statusCode: 500,
},
{
name: "List-buckets-success",

View File

@@ -7,9 +7,9 @@ import (
"bufio"
"context"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3response"
"io"
"sync"
)
@@ -26,7 +26,7 @@ var _ backend.Backend = &BackendMock{}
// AbortMultipartUploadFunc: func(contextMoqParam context.Context, abortMultipartUploadInput *s3.AbortMultipartUploadInput) error {
// panic("mock out the AbortMultipartUpload method")
// },
// ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, acl []byte) error {
// ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket string, newOwner string) error {
// panic("mock out the ChangeBucketOwner method")
// },
// CompleteMultipartUploadFunc: func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
@@ -38,15 +38,12 @@ var _ backend.Backend = &BackendMock{}
// CreateBucketFunc: func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error {
// panic("mock out the CreateBucket method")
// },
// CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
// CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
// panic("mock out the CreateMultipartUpload method")
// },
// DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
// panic("mock out the DeleteBucket method")
// },
// DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
// panic("mock out the DeleteBucketOwnershipControls method")
// },
// DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
// panic("mock out the DeleteBucketPolicy method")
// },
@@ -65,9 +62,6 @@ var _ backend.Backend = &BackendMock{}
// GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) {
// panic("mock out the GetBucketAcl method")
// },
// GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
// panic("mock out the GetBucketOwnershipControls method")
// },
// GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
// panic("mock out the GetBucketPolicy method")
// },
@@ -77,24 +71,15 @@ var _ backend.Backend = &BackendMock{}
// GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
// panic("mock out the GetBucketVersioning method")
// },
// GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
// GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
// panic("mock out the GetObject method")
// },
// GetObjectAclFunc: func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
// panic("mock out the GetObjectAcl method")
// },
// GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
// GetObjectAttributesFunc: func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
// panic("mock out the GetObjectAttributes method")
// },
// GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket string, object string, versionId string) (*bool, error) {
// panic("mock out the GetObjectLegalHold method")
// },
// GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
// panic("mock out the GetObjectLockConfiguration method")
// },
// GetObjectRetentionFunc: func(contextMoqParam context.Context, bucket string, object string, versionId string) ([]byte, error) {
// panic("mock out the GetObjectRetention method")
// },
// GetObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string) (map[string]string, error) {
// panic("mock out the GetObjectTagging method")
// },
@@ -116,10 +101,10 @@ var _ backend.Backend = &BackendMock{}
// ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
// panic("mock out the ListObjectVersions method")
// },
// ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
// ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
// panic("mock out the ListObjects method")
// },
// ListObjectsV2Func: func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
// ListObjectsV2Func: func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
// panic("mock out the ListObjectsV2 method")
// },
// ListPartsFunc: func(contextMoqParam context.Context, listPartsInput *s3.ListPartsInput) (s3response.ListPartsResult, error) {
@@ -128,9 +113,6 @@ var _ backend.Backend = &BackendMock{}
// PutBucketAclFunc: func(contextMoqParam context.Context, bucket string, data []byte) error {
// panic("mock out the PutBucketAcl method")
// },
// PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
// panic("mock out the PutBucketOwnershipControls method")
// },
// PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
// panic("mock out the PutBucketPolicy method")
// },
@@ -146,15 +128,6 @@ var _ backend.Backend = &BackendMock{}
// PutObjectAclFunc: func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error {
// panic("mock out the PutObjectAcl method")
// },
// PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket string, object string, versionId string, status bool) error {
// panic("mock out the PutObjectLegalHold method")
// },
// PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error {
// panic("mock out the PutObjectLockConfiguration method")
// },
// PutObjectRetentionFunc: func(contextMoqParam context.Context, bucket string, object string, versionId string, bypass bool, retention []byte) error {
// panic("mock out the PutObjectRetention method")
// },
// PutObjectTaggingFunc: func(contextMoqParam context.Context, bucket string, object string, tags map[string]string) error {
// panic("mock out the PutObjectTagging method")
// },
@@ -187,7 +160,7 @@ type BackendMock struct {
AbortMultipartUploadFunc func(contextMoqParam context.Context, abortMultipartUploadInput *s3.AbortMultipartUploadInput) error
// ChangeBucketOwnerFunc mocks the ChangeBucketOwner method.
ChangeBucketOwnerFunc func(contextMoqParam context.Context, bucket string, acl []byte) error
ChangeBucketOwnerFunc func(contextMoqParam context.Context, bucket string, newOwner string) error
// CompleteMultipartUploadFunc mocks the CompleteMultipartUpload method.
CompleteMultipartUploadFunc func(contextMoqParam context.Context, completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
@@ -199,14 +172,11 @@ type BackendMock struct {
CreateBucketFunc func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error
// CreateMultipartUploadFunc mocks the CreateMultipartUpload method.
CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
// DeleteBucketFunc mocks the DeleteBucket method.
DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
// DeleteBucketOwnershipControlsFunc mocks the DeleteBucketOwnershipControls method.
DeleteBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) error
// DeleteBucketPolicyFunc mocks the DeleteBucketPolicy method.
DeleteBucketPolicyFunc func(contextMoqParam context.Context, bucket string) error
@@ -225,9 +195,6 @@ type BackendMock struct {
// GetBucketAclFunc mocks the GetBucketAcl method.
GetBucketAclFunc func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error)
// GetBucketOwnershipControlsFunc mocks the GetBucketOwnershipControls method.
GetBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error)
// GetBucketPolicyFunc mocks the GetBucketPolicy method.
GetBucketPolicyFunc func(contextMoqParam context.Context, bucket string) ([]byte, error)
@@ -238,22 +205,13 @@ type BackendMock struct {
GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
// GetObjectFunc mocks the GetObject method.
GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error)
GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error)
// GetObjectAclFunc mocks the GetObjectAcl method.
GetObjectAclFunc func(contextMoqParam context.Context, getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
// GetObjectAttributesFunc mocks the GetObjectAttributes method.
GetObjectAttributesFunc func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
// GetObjectLegalHoldFunc mocks the GetObjectLegalHold method.
GetObjectLegalHoldFunc func(contextMoqParam context.Context, bucket string, object string, versionId string) (*bool, error)
// GetObjectLockConfigurationFunc mocks the GetObjectLockConfiguration method.
GetObjectLockConfigurationFunc func(contextMoqParam context.Context, bucket string) ([]byte, error)
// GetObjectRetentionFunc mocks the GetObjectRetention method.
GetObjectRetentionFunc func(contextMoqParam context.Context, bucket string, object string, versionId string) ([]byte, error)
GetObjectAttributesFunc func(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
// GetObjectTaggingFunc mocks the GetObjectTagging method.
GetObjectTaggingFunc func(contextMoqParam context.Context, bucket string, object string) (map[string]string, error)
@@ -277,10 +235,10 @@ type BackendMock struct {
ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
// ListObjectsFunc mocks the ListObjects method.
ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
// ListObjectsV2Func mocks the ListObjectsV2 method.
ListObjectsV2Func func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
ListObjectsV2Func func(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
// ListPartsFunc mocks the ListParts method.
ListPartsFunc func(contextMoqParam context.Context, listPartsInput *s3.ListPartsInput) (s3response.ListPartsResult, error)
@@ -288,9 +246,6 @@ type BackendMock struct {
// PutBucketAclFunc mocks the PutBucketAcl method.
PutBucketAclFunc func(contextMoqParam context.Context, bucket string, data []byte) error
// PutBucketOwnershipControlsFunc mocks the PutBucketOwnershipControls method.
PutBucketOwnershipControlsFunc func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error
// PutBucketPolicyFunc mocks the PutBucketPolicy method.
PutBucketPolicyFunc func(contextMoqParam context.Context, bucket string, policy []byte) error
@@ -306,15 +261,6 @@ type BackendMock struct {
// PutObjectAclFunc mocks the PutObjectAcl method.
PutObjectAclFunc func(contextMoqParam context.Context, putObjectAclInput *s3.PutObjectAclInput) error
// PutObjectLegalHoldFunc mocks the PutObjectLegalHold method.
PutObjectLegalHoldFunc func(contextMoqParam context.Context, bucket string, object string, versionId string, status bool) error
// PutObjectLockConfigurationFunc mocks the PutObjectLockConfiguration method.
PutObjectLockConfigurationFunc func(contextMoqParam context.Context, bucket string, config []byte) error
// PutObjectRetentionFunc mocks the PutObjectRetention method.
PutObjectRetentionFunc func(contextMoqParam context.Context, bucket string, object string, versionId string, bypass bool, retention []byte) error
// PutObjectTaggingFunc mocks the PutObjectTagging method.
PutObjectTaggingFunc func(contextMoqParam context.Context, bucket string, object string, tags map[string]string) error
@@ -351,8 +297,8 @@ type BackendMock struct {
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
// ACL is the acl argument value.
ACL []byte
// NewOwner is the newOwner argument value.
NewOwner string
}
// CompleteMultipartUpload holds details about calls to the CompleteMultipartUpload method.
CompleteMultipartUpload []struct {
@@ -391,13 +337,6 @@ type BackendMock struct {
// DeleteBucketInput is the deleteBucketInput argument value.
DeleteBucketInput *s3.DeleteBucketInput
}
// DeleteBucketOwnershipControls holds details about calls to the DeleteBucketOwnershipControls method.
DeleteBucketOwnershipControls []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
}
// DeleteBucketPolicy holds details about calls to the DeleteBucketPolicy method.
DeleteBucketPolicy []struct {
// ContextMoqParam is the contextMoqParam argument value.
@@ -442,13 +381,6 @@ type BackendMock struct {
// GetBucketAclInput is the getBucketAclInput argument value.
GetBucketAclInput *s3.GetBucketAclInput
}
// GetBucketOwnershipControls holds details about calls to the GetBucketOwnershipControls method.
GetBucketOwnershipControls []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
}
// GetBucketPolicy holds details about calls to the GetBucketPolicy method.
GetBucketPolicy []struct {
// ContextMoqParam is the contextMoqParam argument value.
@@ -476,6 +408,8 @@ type BackendMock struct {
ContextMoqParam context.Context
// GetObjectInput is the getObjectInput argument value.
GetObjectInput *s3.GetObjectInput
// Writer is the writer argument value.
Writer io.Writer
}
// GetObjectAcl holds details about calls to the GetObjectAcl method.
GetObjectAcl []struct {
@@ -491,35 +425,6 @@ type BackendMock struct {
// GetObjectAttributesInput is the getObjectAttributesInput argument value.
GetObjectAttributesInput *s3.GetObjectAttributesInput
}
// GetObjectLegalHold holds details about calls to the GetObjectLegalHold method.
GetObjectLegalHold []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
// Object is the object argument value.
Object string
// VersionId is the versionId argument value.
VersionId string
}
// GetObjectLockConfiguration holds details about calls to the GetObjectLockConfiguration method.
GetObjectLockConfiguration []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
}
// GetObjectRetention holds details about calls to the GetObjectRetention method.
GetObjectRetention []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
// Object is the object argument value.
Object string
// VersionId is the versionId argument value.
VersionId string
}
// GetObjectTagging holds details about calls to the GetObjectTagging method.
GetObjectTagging []struct {
// ContextMoqParam is the contextMoqParam argument value.
@@ -601,15 +506,6 @@ type BackendMock struct {
// Data is the data argument value.
Data []byte
}
// PutBucketOwnershipControls holds details about calls to the PutBucketOwnershipControls method.
PutBucketOwnershipControls []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
// Ownership is the ownership argument value.
Ownership types.ObjectOwnership
}
// PutBucketPolicy holds details about calls to the PutBucketPolicy method.
PutBucketPolicy []struct {
// ContextMoqParam is the contextMoqParam argument value.
@@ -649,43 +545,6 @@ type BackendMock struct {
// PutObjectAclInput is the putObjectAclInput argument value.
PutObjectAclInput *s3.PutObjectAclInput
}
// PutObjectLegalHold holds details about calls to the PutObjectLegalHold method.
PutObjectLegalHold []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
// Object is the object argument value.
Object string
// VersionId is the versionId argument value.
VersionId string
// Status is the status argument value.
Status bool
}
// PutObjectLockConfiguration holds details about calls to the PutObjectLockConfiguration method.
PutObjectLockConfiguration []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
// Config is the config argument value.
Config []byte
}
// PutObjectRetention holds details about calls to the PutObjectRetention method.
PutObjectRetention []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
// Object is the object argument value.
Object string
// VersionId is the versionId argument value.
VersionId string
// Bypass is the bypass argument value.
Bypass bool
// Retention is the retention argument value.
Retention []byte
}
// PutObjectTagging holds details about calls to the PutObjectTagging method.
PutObjectTagging []struct {
// ContextMoqParam is the contextMoqParam argument value.
@@ -732,57 +591,48 @@ type BackendMock struct {
UploadPartCopyInput *s3.UploadPartCopyInput
}
}
lockAbortMultipartUpload sync.RWMutex
lockChangeBucketOwner sync.RWMutex
lockCompleteMultipartUpload sync.RWMutex
lockCopyObject sync.RWMutex
lockCreateBucket sync.RWMutex
lockCreateMultipartUpload sync.RWMutex
lockDeleteBucket sync.RWMutex
lockDeleteBucketOwnershipControls sync.RWMutex
lockDeleteBucketPolicy sync.RWMutex
lockDeleteBucketTagging sync.RWMutex
lockDeleteObject sync.RWMutex
lockDeleteObjectTagging sync.RWMutex
lockDeleteObjects sync.RWMutex
lockGetBucketAcl sync.RWMutex
lockGetBucketOwnershipControls sync.RWMutex
lockGetBucketPolicy sync.RWMutex
lockGetBucketTagging sync.RWMutex
lockGetBucketVersioning sync.RWMutex
lockGetObject sync.RWMutex
lockGetObjectAcl sync.RWMutex
lockGetObjectAttributes sync.RWMutex
lockGetObjectLegalHold sync.RWMutex
lockGetObjectLockConfiguration sync.RWMutex
lockGetObjectRetention sync.RWMutex
lockGetObjectTagging sync.RWMutex
lockHeadBucket sync.RWMutex
lockHeadObject sync.RWMutex
lockListBuckets sync.RWMutex
lockListBucketsAndOwners sync.RWMutex
lockListMultipartUploads sync.RWMutex
lockListObjectVersions sync.RWMutex
lockListObjects sync.RWMutex
lockListObjectsV2 sync.RWMutex
lockListParts sync.RWMutex
lockPutBucketAcl sync.RWMutex
lockPutBucketOwnershipControls sync.RWMutex
lockPutBucketPolicy sync.RWMutex
lockPutBucketTagging sync.RWMutex
lockPutBucketVersioning sync.RWMutex
lockPutObject sync.RWMutex
lockPutObjectAcl sync.RWMutex
lockPutObjectLegalHold sync.RWMutex
lockPutObjectLockConfiguration sync.RWMutex
lockPutObjectRetention sync.RWMutex
lockPutObjectTagging sync.RWMutex
lockRestoreObject sync.RWMutex
lockSelectObjectContent sync.RWMutex
lockShutdown sync.RWMutex
lockString sync.RWMutex
lockUploadPart sync.RWMutex
lockUploadPartCopy sync.RWMutex
lockAbortMultipartUpload sync.RWMutex
lockChangeBucketOwner sync.RWMutex
lockCompleteMultipartUpload sync.RWMutex
lockCopyObject sync.RWMutex
lockCreateBucket sync.RWMutex
lockCreateMultipartUpload sync.RWMutex
lockDeleteBucket sync.RWMutex
lockDeleteBucketPolicy sync.RWMutex
lockDeleteBucketTagging sync.RWMutex
lockDeleteObject sync.RWMutex
lockDeleteObjectTagging sync.RWMutex
lockDeleteObjects sync.RWMutex
lockGetBucketAcl sync.RWMutex
lockGetBucketPolicy sync.RWMutex
lockGetBucketTagging sync.RWMutex
lockGetBucketVersioning sync.RWMutex
lockGetObject sync.RWMutex
lockGetObjectAcl sync.RWMutex
lockGetObjectAttributes sync.RWMutex
lockGetObjectTagging sync.RWMutex
lockHeadBucket sync.RWMutex
lockHeadObject sync.RWMutex
lockListBuckets sync.RWMutex
lockListBucketsAndOwners sync.RWMutex
lockListMultipartUploads sync.RWMutex
lockListObjectVersions sync.RWMutex
lockListObjects sync.RWMutex
lockListObjectsV2 sync.RWMutex
lockListParts sync.RWMutex
lockPutBucketAcl sync.RWMutex
lockPutBucketPolicy sync.RWMutex
lockPutBucketTagging sync.RWMutex
lockPutBucketVersioning sync.RWMutex
lockPutObject sync.RWMutex
lockPutObjectAcl sync.RWMutex
lockPutObjectTagging sync.RWMutex
lockRestoreObject sync.RWMutex
lockSelectObjectContent sync.RWMutex
lockShutdown sync.RWMutex
lockString sync.RWMutex
lockUploadPart sync.RWMutex
lockUploadPartCopy sync.RWMutex
}
// AbortMultipartUpload calls AbortMultipartUploadFunc.
@@ -822,23 +672,23 @@ func (mock *BackendMock) AbortMultipartUploadCalls() []struct {
}
// ChangeBucketOwner calls ChangeBucketOwnerFunc.
func (mock *BackendMock) ChangeBucketOwner(contextMoqParam context.Context, bucket string, acl []byte) error {
func (mock *BackendMock) ChangeBucketOwner(contextMoqParam context.Context, bucket string, newOwner string) error {
if mock.ChangeBucketOwnerFunc == nil {
panic("BackendMock.ChangeBucketOwnerFunc: method is nil but Backend.ChangeBucketOwner was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
ACL []byte
NewOwner string
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
ACL: acl,
NewOwner: newOwner,
}
mock.lockChangeBucketOwner.Lock()
mock.calls.ChangeBucketOwner = append(mock.calls.ChangeBucketOwner, callInfo)
mock.lockChangeBucketOwner.Unlock()
return mock.ChangeBucketOwnerFunc(contextMoqParam, bucket, acl)
return mock.ChangeBucketOwnerFunc(contextMoqParam, bucket, newOwner)
}
// ChangeBucketOwnerCalls gets all the calls that were made to ChangeBucketOwner.
@@ -848,12 +698,12 @@ func (mock *BackendMock) ChangeBucketOwner(contextMoqParam context.Context, buck
func (mock *BackendMock) ChangeBucketOwnerCalls() []struct {
ContextMoqParam context.Context
Bucket string
ACL []byte
NewOwner string
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
ACL []byte
NewOwner string
}
mock.lockChangeBucketOwner.RLock()
calls = mock.calls.ChangeBucketOwner
@@ -974,7 +824,7 @@ func (mock *BackendMock) CreateBucketCalls() []struct {
}
// CreateMultipartUpload calls CreateMultipartUploadFunc.
func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
func (mock *BackendMock) CreateMultipartUpload(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
if mock.CreateMultipartUploadFunc == nil {
panic("BackendMock.CreateMultipartUploadFunc: method is nil but Backend.CreateMultipartUpload was just called")
}
@@ -1045,42 +895,6 @@ func (mock *BackendMock) DeleteBucketCalls() []struct {
return calls
}
// DeleteBucketOwnershipControls calls DeleteBucketOwnershipControlsFunc.
func (mock *BackendMock) DeleteBucketOwnershipControls(contextMoqParam context.Context, bucket string) error {
if mock.DeleteBucketOwnershipControlsFunc == nil {
panic("BackendMock.DeleteBucketOwnershipControlsFunc: method is nil but Backend.DeleteBucketOwnershipControls was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
}
mock.lockDeleteBucketOwnershipControls.Lock()
mock.calls.DeleteBucketOwnershipControls = append(mock.calls.DeleteBucketOwnershipControls, callInfo)
mock.lockDeleteBucketOwnershipControls.Unlock()
return mock.DeleteBucketOwnershipControlsFunc(contextMoqParam, bucket)
}
// DeleteBucketOwnershipControlsCalls gets all the calls that were made to DeleteBucketOwnershipControls.
// Check the length with:
//
// len(mockedBackend.DeleteBucketOwnershipControlsCalls())
func (mock *BackendMock) DeleteBucketOwnershipControlsCalls() []struct {
ContextMoqParam context.Context
Bucket string
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
}
mock.lockDeleteBucketOwnershipControls.RLock()
calls = mock.calls.DeleteBucketOwnershipControls
mock.lockDeleteBucketOwnershipControls.RUnlock()
return calls
}
// DeleteBucketPolicy calls DeleteBucketPolicyFunc.
func (mock *BackendMock) DeleteBucketPolicy(contextMoqParam context.Context, bucket string) error {
if mock.DeleteBucketPolicyFunc == nil {
@@ -1301,42 +1115,6 @@ func (mock *BackendMock) GetBucketAclCalls() []struct {
return calls
}
// GetBucketOwnershipControls calls GetBucketOwnershipControlsFunc.
func (mock *BackendMock) GetBucketOwnershipControls(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
if mock.GetBucketOwnershipControlsFunc == nil {
panic("BackendMock.GetBucketOwnershipControlsFunc: method is nil but Backend.GetBucketOwnershipControls was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
}
mock.lockGetBucketOwnershipControls.Lock()
mock.calls.GetBucketOwnershipControls = append(mock.calls.GetBucketOwnershipControls, callInfo)
mock.lockGetBucketOwnershipControls.Unlock()
return mock.GetBucketOwnershipControlsFunc(contextMoqParam, bucket)
}
// GetBucketOwnershipControlsCalls gets all the calls that were made to GetBucketOwnershipControls.
// Check the length with:
//
// len(mockedBackend.GetBucketOwnershipControlsCalls())
func (mock *BackendMock) GetBucketOwnershipControlsCalls() []struct {
ContextMoqParam context.Context
Bucket string
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
}
mock.lockGetBucketOwnershipControls.RLock()
calls = mock.calls.GetBucketOwnershipControls
mock.lockGetBucketOwnershipControls.RUnlock()
return calls
}
// GetBucketPolicy calls GetBucketPolicyFunc.
func (mock *BackendMock) GetBucketPolicy(contextMoqParam context.Context, bucket string) ([]byte, error) {
if mock.GetBucketPolicyFunc == nil {
@@ -1446,21 +1224,23 @@ func (mock *BackendMock) GetBucketVersioningCalls() []struct {
}
// GetObject calls GetObjectFunc.
func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
if mock.GetObjectFunc == nil {
panic("BackendMock.GetObjectFunc: method is nil but Backend.GetObject was just called")
}
callInfo := struct {
ContextMoqParam context.Context
GetObjectInput *s3.GetObjectInput
Writer io.Writer
}{
ContextMoqParam: contextMoqParam,
GetObjectInput: getObjectInput,
Writer: writer,
}
mock.lockGetObject.Lock()
mock.calls.GetObject = append(mock.calls.GetObject, callInfo)
mock.lockGetObject.Unlock()
return mock.GetObjectFunc(contextMoqParam, getObjectInput)
return mock.GetObjectFunc(contextMoqParam, getObjectInput, writer)
}
// GetObjectCalls gets all the calls that were made to GetObject.
@@ -1470,10 +1250,12 @@ func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInp
func (mock *BackendMock) GetObjectCalls() []struct {
ContextMoqParam context.Context
GetObjectInput *s3.GetObjectInput
Writer io.Writer
} {
var calls []struct {
ContextMoqParam context.Context
GetObjectInput *s3.GetObjectInput
Writer io.Writer
}
mock.lockGetObject.RLock()
calls = mock.calls.GetObject
@@ -1518,7 +1300,7 @@ func (mock *BackendMock) GetObjectAclCalls() []struct {
}
// GetObjectAttributes calls GetObjectAttributesFunc.
func (mock *BackendMock) GetObjectAttributes(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
func (mock *BackendMock) GetObjectAttributes(contextMoqParam context.Context, getObjectAttributesInput *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
if mock.GetObjectAttributesFunc == nil {
panic("BackendMock.GetObjectAttributesFunc: method is nil but Backend.GetObjectAttributes was just called")
}
@@ -1553,130 +1335,6 @@ func (mock *BackendMock) GetObjectAttributesCalls() []struct {
return calls
}
// GetObjectLegalHold calls GetObjectLegalHoldFunc.
func (mock *BackendMock) GetObjectLegalHold(contextMoqParam context.Context, bucket string, object string, versionId string) (*bool, error) {
if mock.GetObjectLegalHoldFunc == nil {
panic("BackendMock.GetObjectLegalHoldFunc: method is nil but Backend.GetObjectLegalHold was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
Object string
VersionId string
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
Object: object,
VersionId: versionId,
}
mock.lockGetObjectLegalHold.Lock()
mock.calls.GetObjectLegalHold = append(mock.calls.GetObjectLegalHold, callInfo)
mock.lockGetObjectLegalHold.Unlock()
return mock.GetObjectLegalHoldFunc(contextMoqParam, bucket, object, versionId)
}
// GetObjectLegalHoldCalls gets all the calls that were made to GetObjectLegalHold.
// Check the length with:
//
// len(mockedBackend.GetObjectLegalHoldCalls())
func (mock *BackendMock) GetObjectLegalHoldCalls() []struct {
ContextMoqParam context.Context
Bucket string
Object string
VersionId string
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
Object string
VersionId string
}
mock.lockGetObjectLegalHold.RLock()
calls = mock.calls.GetObjectLegalHold
mock.lockGetObjectLegalHold.RUnlock()
return calls
}
// GetObjectLockConfiguration calls GetObjectLockConfigurationFunc.
func (mock *BackendMock) GetObjectLockConfiguration(contextMoqParam context.Context, bucket string) ([]byte, error) {
if mock.GetObjectLockConfigurationFunc == nil {
panic("BackendMock.GetObjectLockConfigurationFunc: method is nil but Backend.GetObjectLockConfiguration was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
}
mock.lockGetObjectLockConfiguration.Lock()
mock.calls.GetObjectLockConfiguration = append(mock.calls.GetObjectLockConfiguration, callInfo)
mock.lockGetObjectLockConfiguration.Unlock()
return mock.GetObjectLockConfigurationFunc(contextMoqParam, bucket)
}
// GetObjectLockConfigurationCalls gets all the calls that were made to GetObjectLockConfiguration.
// Check the length with:
//
// len(mockedBackend.GetObjectLockConfigurationCalls())
func (mock *BackendMock) GetObjectLockConfigurationCalls() []struct {
ContextMoqParam context.Context
Bucket string
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
}
mock.lockGetObjectLockConfiguration.RLock()
calls = mock.calls.GetObjectLockConfiguration
mock.lockGetObjectLockConfiguration.RUnlock()
return calls
}
// GetObjectRetention calls GetObjectRetentionFunc.
func (mock *BackendMock) GetObjectRetention(contextMoqParam context.Context, bucket string, object string, versionId string) ([]byte, error) {
if mock.GetObjectRetentionFunc == nil {
panic("BackendMock.GetObjectRetentionFunc: method is nil but Backend.GetObjectRetention was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
Object string
VersionId string
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
Object: object,
VersionId: versionId,
}
mock.lockGetObjectRetention.Lock()
mock.calls.GetObjectRetention = append(mock.calls.GetObjectRetention, callInfo)
mock.lockGetObjectRetention.Unlock()
return mock.GetObjectRetentionFunc(contextMoqParam, bucket, object, versionId)
}
// GetObjectRetentionCalls gets all the calls that were made to GetObjectRetention.
// Check the length with:
//
// len(mockedBackend.GetObjectRetentionCalls())
func (mock *BackendMock) GetObjectRetentionCalls() []struct {
ContextMoqParam context.Context
Bucket string
Object string
VersionId string
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
Object string
VersionId string
}
mock.lockGetObjectRetention.RLock()
calls = mock.calls.GetObjectRetention
mock.lockGetObjectRetention.RUnlock()
return calls
}
// GetObjectTagging calls GetObjectTaggingFunc.
func (mock *BackendMock) GetObjectTagging(contextMoqParam context.Context, bucket string, object string) (map[string]string, error) {
if mock.GetObjectTaggingFunc == nil {
@@ -1934,7 +1592,7 @@ func (mock *BackendMock) ListObjectVersionsCalls() []struct {
}
// ListObjects calls ListObjectsFunc.
func (mock *BackendMock) ListObjects(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
func (mock *BackendMock) ListObjects(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
if mock.ListObjectsFunc == nil {
panic("BackendMock.ListObjectsFunc: method is nil but Backend.ListObjects was just called")
}
@@ -1970,7 +1628,7 @@ func (mock *BackendMock) ListObjectsCalls() []struct {
}
// ListObjectsV2 calls ListObjectsV2Func.
func (mock *BackendMock) ListObjectsV2(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
func (mock *BackendMock) ListObjectsV2(contextMoqParam context.Context, listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
if mock.ListObjectsV2Func == nil {
panic("BackendMock.ListObjectsV2Func: method is nil but Backend.ListObjectsV2 was just called")
}
@@ -2081,46 +1739,6 @@ func (mock *BackendMock) PutBucketAclCalls() []struct {
return calls
}
// PutBucketOwnershipControls calls PutBucketOwnershipControlsFunc.
func (mock *BackendMock) PutBucketOwnershipControls(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
if mock.PutBucketOwnershipControlsFunc == nil {
panic("BackendMock.PutBucketOwnershipControlsFunc: method is nil but Backend.PutBucketOwnershipControls was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
Ownership types.ObjectOwnership
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
Ownership: ownership,
}
mock.lockPutBucketOwnershipControls.Lock()
mock.calls.PutBucketOwnershipControls = append(mock.calls.PutBucketOwnershipControls, callInfo)
mock.lockPutBucketOwnershipControls.Unlock()
return mock.PutBucketOwnershipControlsFunc(contextMoqParam, bucket, ownership)
}
// PutBucketOwnershipControlsCalls gets all the calls that were made to PutBucketOwnershipControls.
// Check the length with:
//
// len(mockedBackend.PutBucketOwnershipControlsCalls())
func (mock *BackendMock) PutBucketOwnershipControlsCalls() []struct {
ContextMoqParam context.Context
Bucket string
Ownership types.ObjectOwnership
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
Ownership types.ObjectOwnership
}
mock.lockPutBucketOwnershipControls.RLock()
calls = mock.calls.PutBucketOwnershipControls
mock.lockPutBucketOwnershipControls.RUnlock()
return calls
}
// PutBucketPolicy calls PutBucketPolicyFunc.
func (mock *BackendMock) PutBucketPolicy(contextMoqParam context.Context, bucket string, policy []byte) error {
if mock.PutBucketPolicyFunc == nil {
@@ -2309,146 +1927,6 @@ func (mock *BackendMock) PutObjectAclCalls() []struct {
return calls
}
// PutObjectLegalHold calls PutObjectLegalHoldFunc.
func (mock *BackendMock) PutObjectLegalHold(contextMoqParam context.Context, bucket string, object string, versionId string, status bool) error {
if mock.PutObjectLegalHoldFunc == nil {
panic("BackendMock.PutObjectLegalHoldFunc: method is nil but Backend.PutObjectLegalHold was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
Object string
VersionId string
Status bool
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
Object: object,
VersionId: versionId,
Status: status,
}
mock.lockPutObjectLegalHold.Lock()
mock.calls.PutObjectLegalHold = append(mock.calls.PutObjectLegalHold, callInfo)
mock.lockPutObjectLegalHold.Unlock()
return mock.PutObjectLegalHoldFunc(contextMoqParam, bucket, object, versionId, status)
}
// PutObjectLegalHoldCalls gets all the calls that were made to PutObjectLegalHold.
// Check the length with:
//
// len(mockedBackend.PutObjectLegalHoldCalls())
func (mock *BackendMock) PutObjectLegalHoldCalls() []struct {
ContextMoqParam context.Context
Bucket string
Object string
VersionId string
Status bool
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
Object string
VersionId string
Status bool
}
mock.lockPutObjectLegalHold.RLock()
calls = mock.calls.PutObjectLegalHold
mock.lockPutObjectLegalHold.RUnlock()
return calls
}
// PutObjectLockConfiguration calls PutObjectLockConfigurationFunc.
func (mock *BackendMock) PutObjectLockConfiguration(contextMoqParam context.Context, bucket string, config []byte) error {
if mock.PutObjectLockConfigurationFunc == nil {
panic("BackendMock.PutObjectLockConfigurationFunc: method is nil but Backend.PutObjectLockConfiguration was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
Config []byte
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
Config: config,
}
mock.lockPutObjectLockConfiguration.Lock()
mock.calls.PutObjectLockConfiguration = append(mock.calls.PutObjectLockConfiguration, callInfo)
mock.lockPutObjectLockConfiguration.Unlock()
return mock.PutObjectLockConfigurationFunc(contextMoqParam, bucket, config)
}
// PutObjectLockConfigurationCalls gets all the calls that were made to PutObjectLockConfiguration.
// Check the length with:
//
// len(mockedBackend.PutObjectLockConfigurationCalls())
func (mock *BackendMock) PutObjectLockConfigurationCalls() []struct {
ContextMoqParam context.Context
Bucket string
Config []byte
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
Config []byte
}
mock.lockPutObjectLockConfiguration.RLock()
calls = mock.calls.PutObjectLockConfiguration
mock.lockPutObjectLockConfiguration.RUnlock()
return calls
}
// PutObjectRetention calls PutObjectRetentionFunc.
func (mock *BackendMock) PutObjectRetention(contextMoqParam context.Context, bucket string, object string, versionId string, bypass bool, retention []byte) error {
if mock.PutObjectRetentionFunc == nil {
panic("BackendMock.PutObjectRetentionFunc: method is nil but Backend.PutObjectRetention was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
Object string
VersionId string
Bypass bool
Retention []byte
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
Object: object,
VersionId: versionId,
Bypass: bypass,
Retention: retention,
}
mock.lockPutObjectRetention.Lock()
mock.calls.PutObjectRetention = append(mock.calls.PutObjectRetention, callInfo)
mock.lockPutObjectRetention.Unlock()
return mock.PutObjectRetentionFunc(contextMoqParam, bucket, object, versionId, bypass, retention)
}
// PutObjectRetentionCalls gets all the calls that were made to PutObjectRetention.
// Check the length with:
//
// len(mockedBackend.PutObjectRetentionCalls())
func (mock *BackendMock) PutObjectRetentionCalls() []struct {
ContextMoqParam context.Context
Bucket string
Object string
VersionId string
Bypass bool
Retention []byte
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
Object string
VersionId string
Bypass bool
Retention []byte
}
mock.lockPutObjectRetention.RLock()
calls = mock.calls.PutObjectRetention
mock.lockPutObjectRetention.RUnlock()
return calls
}
// PutObjectTagging calls PutObjectTaggingFunc.
func (mock *BackendMock) PutObjectTagging(contextMoqParam context.Context, bucket string, object string, tags map[string]string) error {
if mock.PutObjectTaggingFunc == nil {

File diff suppressed because it is too large Load Diff

View File

@@ -19,6 +19,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"reflect"
@@ -76,7 +77,7 @@ func TestNew(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := New(tt.args.be, tt.args.iam, nil, nil, nil, false, false)
got := New(tt.args.be, tt.args.iam, nil, nil, false)
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("New() = %v, want %v", got, tt.want)
}
@@ -187,10 +188,10 @@ func TestS3ApiController_GetActions(t *testing.T) {
GetObjectAclFunc: func(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
return &s3.GetObjectAclOutput{}, nil
},
GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
return s3response.GetObjectAttributesResult{}, nil
GetObjectAttributesFunc: func(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
return &s3.GetObjectAttributesOutput{}, nil
},
GetObjectFunc: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
GetObjectFunc: func(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
return &s3.GetObjectOutput{
Metadata: map[string]string{"hello": "world"},
ContentType: getPtr("application/xml"),
@@ -204,19 +205,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
GetObjectTaggingFunc: func(_ context.Context, bucket, object string) (map[string]string, error) {
return map[string]string{"hello": "world"}, nil
},
GetObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string) ([]byte, error) {
result, err := json.Marshal(types.ObjectLockRetention{
Mode: types.ObjectLockRetentionModeCompliance,
})
if err != nil {
return nil, err
}
return result, nil
},
GetObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string) (*bool, error) {
result := true
return &result, nil
},
},
}
app.Use(func(ctx *fiber.Ctx) error {
@@ -248,24 +236,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-get-object-retention-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?retention", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-get-object-legal-hold-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket/my-obj?legal-hold", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-actions-invalid-max-parts-string",
app: app,
@@ -359,11 +329,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
req *http.Request
}
objectLockResult, err := json.Marshal(auth.BucketLockConfig{})
if err != nil {
t.Errorf("failed to parse object lock result %v", err)
}
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
@@ -373,11 +338,11 @@ func TestS3ApiController_ListActions(t *testing.T) {
ListMultipartUploadsFunc: func(_ context.Context, output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
return s3response.ListMultipartUploadsResult{}, nil
},
ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
return s3response.ListObjectsV2Result{}, nil
ListObjectsV2Func: func(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
return &s3.ListObjectsV2Output{}, nil
},
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
return s3response.ListObjectsResult{}, nil
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return &s3.ListObjectsOutput{}, nil
},
GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
return map[string]string{}, nil
@@ -391,12 +356,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
GetBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return []byte{}, nil
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return objectLockResult, nil
},
GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
return types.ObjectOwnershipBucketOwnerEnforced, nil
},
},
}
@@ -410,14 +369,14 @@ func TestS3ApiController_ListActions(t *testing.T) {
app.Get("/:bucket", s3ApiController.ListActions)
// Error case
//Error case
s3ApiControllerError := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
},
GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -450,15 +409,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
wantErr: false,
statusCode: 404,
},
{
name: "Get-bucket-ownership-control-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket?ownershipControls", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-bucket-tagging-success",
app: app,
@@ -468,15 +418,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Get-object-lock-configuration-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket?object-lock", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-bucket-acl-success",
app: app,
@@ -573,7 +514,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
app := fiber.New()
// Mock valid acl
acl := auth.ACL{Owner: "valid access"}
acl := auth.ACL{Owner: "valid access", ACL: "public-read-write"}
acldata, err := json.Marshal(acl)
if err != nil {
t.Errorf("Failed to parse the params: %v", err.Error())
@@ -604,6 +545,14 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
</AccessControlPolicy>
`
succBody := `
<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Owner>
<ID>valid access</ID>
</Owner>
</AccessControlPolicy>
`
tagBody := `
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
@@ -635,34 +584,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
}
`
objectLockBody := `
<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<ObjectLockEnabled>Enabled</ObjectLockEnabled>
<Rule>
<DefaultRetention>
<Mode>GOVERNANCE</Mode>
<Years>2</Years>
</DefaultRetention>
</Rule>
</ObjectLockConfiguration>
`
ownershipBody := `
<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Rule>
<ObjectOwnership>BucketOwnerEnforced</ObjectOwnership>
</Rule>
</OwnershipControls>
`
invalidOwnershipBody := `
<OwnershipControls xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Rule>
<ObjectOwnership>invalid_value</ObjectOwnership>
</Rule>
</OwnershipControls>
`
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -683,15 +604,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
PutBucketPolicyFunc: func(contextMoqParam context.Context, bucket string, policy []byte) error {
return nil
},
PutObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string, config []byte) error {
return nil
},
PutBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string, ownership types.ObjectOwnership) error {
return nil
},
GetBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) (types.ObjectOwnership, error) {
return types.ObjectOwnershipBucketOwnerPreferred, nil
},
},
}
// Mock ctx.Locals
@@ -715,18 +627,16 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
// PutBucketAcl incorrect bucket owner case
incorrectBucketOwner := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(invOwnerBody))
incorrectBucketOwner.Header.Set("X-Amz-Acl", "private")
// PutBucketAcl acl success
aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", nil)
aclSuccReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(succBody))
aclSuccReq.Header.Set("X-Amz-Acl", "private")
// Invalid acl body case
errAclBodyReq := httptest.NewRequest(http.MethodPut, "/my-bucket?acl", strings.NewReader(body))
errAclBodyReq.Header.Set("X-Amz-Grant-Read", "hello")
invAclOwnershipReq := httptest.NewRequest(http.MethodPut, "/my-bucket", nil)
invAclOwnershipReq.Header.Set("X-Amz-Grant-Read", "hello")
tests := []struct {
name string
app *fiber.App
@@ -752,42 +662,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "Put-bucket-ownership-controls-invalid-ownership",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(invalidOwnershipBody)),
},
wantErr: false,
statusCode: 400,
},
{
name: "Put-bucket-ownership-controls-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket?ownershipControls", strings.NewReader(ownershipBody)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Put-object-lock-configuration-invalid-body",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Put-object-lock-configuration-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket?object-lock", strings.NewReader(objectLockBody)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Put-bucket-versioning-invalid-body",
app: app,
@@ -858,7 +732,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
req: incorrectBucketOwner,
},
wantErr: false,
statusCode: 400,
statusCode: 403,
},
{
name: "Put-bucket-acl-success",
@@ -870,16 +744,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
statusCode: 200,
},
{
name: "Create-bucket-invalid-acl-ownership-combination",
app: app,
args: args{
req: invAclOwnershipReq,
},
wantErr: false,
statusCode: 400,
},
{
name: "Create-bucket-invalid-bucket-name",
name: "Put-bucket-invalid-bucket-name",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/aa", nil),
@@ -888,7 +753,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
statusCode: 400,
},
{
name: "Create-bucket-success",
name: "Put-bucket-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket", nil),
@@ -941,19 +806,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
</Tagging>
`
retentionBody := `
<Retention xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Mode>GOVERNANCE</Mode>
<RetainUntilDate>2025-01-01T00:00:00Z</RetainUntilDate>
</Retention>
`
legalHoldBody := `
<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Status>ON</Status>
</LegalHold>
`
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
@@ -980,15 +832,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
UploadPartCopyFunc: func(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
return s3response.CopyObjectResult{}, nil
},
PutObjectLegalHoldFunc: func(contextMoqParam context.Context, bucket, object, versionId string, status bool) error {
return nil
},
PutObjectRetentionFunc: func(contextMoqParam context.Context, bucket, object, versionId string, bypass bool, retention []byte) error {
return nil
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
},
},
}
app.Use(func(ctx *fiber.Ctx) error {
@@ -1067,42 +910,6 @@ func TestS3ApiController_PutActions(t *testing.T) {
wantErr: false,
statusCode: 200,
},
{
name: "put-object-retention-invalid-request",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "put-object-retention-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?retention", strings.NewReader(retentionBody)),
},
wantErr: false,
statusCode: 200,
},
{
name: "put-legal-hold-invalid-request",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "put-legal-hold-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket/my-key?legal-hold", strings.NewReader(legalHoldBody)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Put-object-acl-invalid-acl",
app: app,
@@ -1223,12 +1030,6 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
return nil
},
DeleteBucketPolicyFunc: func(contextMoqParam context.Context, bucket string) error {
return nil
},
DeleteBucketOwnershipControlsFunc: func(contextMoqParam context.Context, bucket string) error {
return nil
},
},
}
@@ -1267,23 +1068,6 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
wantErr: false,
statusCode: 204,
},
{
name: "Delete-bucket-ownership-controls-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodDelete, "/my-bucket?ownershipControls", nil),
},
wantErr: false,
statusCode: 204,
}, {
name: "Delete-bucket-policy-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodDelete, "/my-bucket?policy", nil),
},
wantErr: false,
statusCode: 204,
},
}
for _, tt := range tests {
resp, err := tt.app.Test(tt.args.req)
@@ -1312,9 +1096,6 @@ func TestS3ApiController_DeleteObjects(t *testing.T) {
DeleteObjectsFunc: func(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
return s3response.DeleteResult{}, nil
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
},
},
}
@@ -1392,9 +1173,6 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
DeleteObjectTaggingFunc: func(_ context.Context, bucket, object string) error {
return nil
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
},
},
}
@@ -1417,9 +1195,6 @@ func TestS3ApiController_DeleteActions(t *testing.T) {
DeleteObjectFunc: func(context.Context, *s3.DeleteObjectInput) error {
return s3err.GetAPIError(7)
},
GetObjectLockConfigurationFunc: func(contextMoqParam context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)
},
}}
appErr.Use(func(ctx *fiber.Ctx) error {
@@ -1510,7 +1285,6 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
ctx.Locals("region", "us-east-1")
return ctx.Next()
})
@@ -1534,7 +1308,6 @@ func TestS3ApiController_HeadBucket(t *testing.T) {
ctx.Locals("isRoot", true)
ctx.Locals("isDebug", false)
ctx.Locals("parsedAcl", auth.ACL{})
ctx.Locals("region", "us-east-1")
return ctx.Next()
})
@@ -1697,8 +1470,8 @@ func TestS3ApiController_CreateActions(t *testing.T) {
CompleteMultipartUploadFunc: func(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
return &s3.CompleteMultipartUploadOutput{}, nil
},
CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
return s3response.InitiateMultipartUploadResult{}, nil
CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
return &s3.CreateMultipartUploadOutput{}, nil
},
SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) {
return func(w *bufio.Writer) {}
@@ -1733,11 +1506,20 @@ func TestS3ApiController_CreateActions(t *testing.T) {
name: "Restore-object-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", nil),
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", strings.NewReader(`<root><key>body</key></root>`)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Restore-object-error",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPost, "/my-bucket/my-key?restore", nil),
},
wantErr: false,
statusCode: 500,
},
{
name: "Select-object-content-invalid-body",
app: app,

View File

@@ -33,9 +33,6 @@ var _ auth.IAMService = &IAMServiceMock{}
// ShutdownFunc: func() error {
// panic("mock out the Shutdown method")
// },
// UpdateUserAccountFunc: func(access string, props auth.MutableProps) error {
// panic("mock out the UpdateUserAccount method")
// },
// }
//
// // use mockedIAMService in code that requires auth.IAMService
@@ -58,9 +55,6 @@ type IAMServiceMock struct {
// ShutdownFunc mocks the Shutdown method.
ShutdownFunc func() error
// UpdateUserAccountFunc mocks the UpdateUserAccount method.
UpdateUserAccountFunc func(access string, props auth.MutableProps) error
// calls tracks calls to the methods.
calls struct {
// CreateAccount holds details about calls to the CreateAccount method.
@@ -84,20 +78,12 @@ type IAMServiceMock struct {
// Shutdown holds details about calls to the Shutdown method.
Shutdown []struct {
}
// UpdateUserAccount holds details about calls to the UpdateUserAccount method.
UpdateUserAccount []struct {
// Access is the access argument value.
Access string
// Props is the props argument value.
Props auth.MutableProps
}
}
lockCreateAccount sync.RWMutex
lockDeleteUserAccount sync.RWMutex
lockGetUserAccount sync.RWMutex
lockListUserAccounts sync.RWMutex
lockShutdown sync.RWMutex
lockUpdateUserAccount sync.RWMutex
}
// CreateAccount calls CreateAccountFunc.
@@ -249,39 +235,3 @@ func (mock *IAMServiceMock) ShutdownCalls() []struct {
mock.lockShutdown.RUnlock()
return calls
}
// UpdateUserAccount calls UpdateUserAccountFunc.
func (mock *IAMServiceMock) UpdateUserAccount(access string, props auth.MutableProps) error {
if mock.UpdateUserAccountFunc == nil {
panic("IAMServiceMock.UpdateUserAccountFunc: method is nil but IAMService.UpdateUserAccount was just called")
}
callInfo := struct {
Access string
Props auth.MutableProps
}{
Access: access,
Props: props,
}
mock.lockUpdateUserAccount.Lock()
mock.calls.UpdateUserAccount = append(mock.calls.UpdateUserAccount, callInfo)
mock.lockUpdateUserAccount.Unlock()
return mock.UpdateUserAccountFunc(access, props)
}
// UpdateUserAccountCalls gets all the calls that were made to UpdateUserAccount.
// Check the length with:
//
// len(mockedIAMService.UpdateUserAccountCalls())
func (mock *IAMServiceMock) UpdateUserAccountCalls() []struct {
Access string
Props auth.MutableProps
} {
var calls []struct {
Access string
Props auth.MutableProps
}
mock.lockUpdateUserAccount.RLock()
calls = mock.calls.UpdateUserAccount
mock.lockUpdateUserAccount.RUnlock()
return calls
}

View File

@@ -24,7 +24,6 @@ import (
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
@@ -32,7 +31,7 @@ var (
singlePath = regexp.MustCompile(`^/[^/]+/?$`)
)
func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fiber.Handler {
func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
path := ctx.Path()
@@ -49,19 +48,10 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger, readonly bool) fibe
!ctx.Request().URI().QueryArgs().Has("acl") &&
!ctx.Request().URI().QueryArgs().Has("tagging") &&
!ctx.Request().URI().QueryArgs().Has("versioning") &&
!ctx.Request().URI().QueryArgs().Has("policy") &&
!ctx.Request().URI().QueryArgs().Has("object-lock") &&
!ctx.Request().URI().QueryArgs().Has("ownershipControls") {
!ctx.Request().URI().QueryArgs().Has("policy") {
if err := auth.MayCreateBucket(acct, isRoot); err != nil {
return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
}
if readonly {
return controllers.SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrAccessDenied),
&controllers.MetaOpts{
Logger: logger,
Action: "CreateBucket",
})
}
return ctx.Next()
}
data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})

View File

@@ -25,7 +25,6 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
@@ -41,7 +40,7 @@ type RootUserConfig struct {
Secret string
}
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
@@ -55,16 +54,16 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
ctx.Locals("startTime", time.Now())
authorization := ctx.Get("Authorization")
if authorization == "" {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger)
}
authData, err := utils.ParseAuthorization(authorization)
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
if authData.Algorithm != "AWS4-HMAC-SHA256" {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger)
}
if authData.Region != region {
@@ -72,40 +71,40 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region),
HTTPStatusCode: http.StatusForbidden,
}, logger, mm)
}, logger)
}
ctx.Locals("isRoot", authData.Access == root.Access)
account, err := acct.getAccount(authData.Access)
if err == auth.ErrNoSuchUser {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
}
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
ctx.Locals("account", account)
// Check X-Amz-Date header
date := ctx.Get("X-Amz-Date")
if date == "" {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger)
}
// Parse the date and check the date validity
tdate, err := time.Parse(iso8601Format, date)
if err != nil {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger)
}
if date[:8] != authData.Date {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger)
}
// Validate the dates difference
err = utils.ValidateDate(tdate)
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
if utils.IsBigDataAction(ctx) {
@@ -126,7 +125,7 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
// Compare the calculated hash with the hash provided
if hashPayload != hexPayload {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger)
}
}
@@ -135,13 +134,13 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
if contentLengthStr != "" {
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger)
}
}
err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
return ctx.Next()
@@ -165,6 +164,6 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
return a.iam.GetUserAccount(access)
}
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger, mm *metrics.Manager) error {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}

View File

@@ -20,14 +20,13 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3log"
)
// ProcessChunkedBody initializes the chunked upload stream if the
// request appears to be a chunked upload
func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string) fiber.Handler {
func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string) fiber.Handler {
return func(ctx *fiber.Ctx) error {
decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
if decodedLength == "" {
@@ -37,7 +36,7 @@ func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.A
authData, err := utils.ParseAuthorization(ctx.Get("Authorization"))
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
acct := ctx.Locals("account").(auth.Account)
@@ -52,7 +51,7 @@ func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.A
return cr
})
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
return ctx.Next()
}

View File

@@ -20,13 +20,12 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, mm *metrics.Manager, region string, debug bool) fiber.Handler {
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
@@ -39,16 +38,16 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
authData, err := utils.ParsePresignedURIParts(ctx)
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
ctx.Locals("isRoot", authData.Access == root.Access)
account, err := acct.getAccount(authData.Access)
if err == auth.ErrNoSuchUser {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger, mm)
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
}
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
ctx.Locals("account", account)
@@ -62,7 +61,7 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger
err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug)
if err != nil {
return sendResponse(ctx, err, logger, mm)
return sendResponse(ctx, err, logger)
}
return ctx.Next()

View File

@@ -18,18 +18,17 @@ import (
"net/url"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func DecodeURL(logger s3log.AuditLogger, mm *metrics.Manager) fiber.Handler {
func DecodeURL(logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
reqURL := ctx.Request().URI().String()
decoded, err := url.Parse(reqURL)
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger, MetricsMng: mm})
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
}
ctx.Path(decoded.Path)
return ctx.Next()

View File

@@ -18,7 +18,6 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3log"
@@ -28,11 +27,11 @@ type S3ApiRouter struct {
WithAdmSrv bool
}
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, aLogger s3log.AuditLogger, evs s3event.S3EventSender, mm *metrics.Manager, debug bool, readonly bool) {
s3ApiController := controllers.New(be, iam, logger, evs, mm, debug, readonly)
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender, debug bool) {
s3ApiController := controllers.New(be, iam, logger, evs, debug)
if sa.WithAdmSrv {
adminController := controllers.NewAdminController(iam, be, aLogger)
adminController := controllers.NewAdminController(iam, be)
// CreateUser admin api
app.Patch("/create-user", adminController.CreateUser)
@@ -40,9 +39,6 @@ func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMServ
// DeleteUsers admin api
app.Patch("/delete-user", adminController.DeleteUser)
// UpdateUser admin api
app.Patch("update-user", adminController.UpdateUser)
// ListUsers admin api
app.Patch("/list-users", adminController.ListUsers)

View File

@@ -45,7 +45,7 @@ func TestS3ApiRouter_Init(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, nil, nil, false, false)
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil, false)
})
}
}

View File

@@ -22,36 +22,23 @@ import (
"github.com/gofiber/fiber/v2/middleware/logger"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/metrics"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3log"
)
type S3ApiServer struct {
app *fiber.App
backend backend.Backend
router *S3ApiRouter
port string
cert *tls.Certificate
quiet bool
debug bool
readonly bool
health string
app *fiber.App
backend backend.Backend
router *S3ApiRouter
port string
cert *tls.Certificate
quiet bool
debug bool
health string
}
func New(
app *fiber.App,
be backend.Backend,
root middlewares.RootUserConfig,
port, region string,
iam auth.IAMService,
l s3log.AuditLogger,
adminLogger s3log.AuditLogger,
evs s3event.S3EventSender,
mm *metrics.Manager,
opts ...Option,
) (*S3ApiServer, error) {
func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, evs s3event.S3EventSender, opts ...Option) (*S3ApiServer, error) {
server := &S3ApiServer{
app: app,
backend: be,
@@ -65,9 +52,7 @@ func New(
// Logging middlewares
if !server.quiet {
app.Use(logger.New(logger.Config{
Format: "${time} | ${status} | ${latency} | ${ip} | ${method} | ${path} | ${error} | ${queryParams}\n",
}))
app.Use(logger.New())
}
// Set up health endpoint if specified
if server.health != "" {
@@ -75,17 +60,17 @@ func New(
return ctx.SendStatus(http.StatusOK)
})
}
app.Use(middlewares.DecodeURL(l, mm))
app.Use(middlewares.DecodeURL(l))
app.Use(middlewares.RequestLogger(server.debug))
// Authentication middlewares
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, mm, region, server.debug))
app.Use(middlewares.VerifyV4Signature(root, iam, l, mm, region, server.debug))
app.Use(middlewares.ProcessChunkedBody(root, iam, l, mm, region))
app.Use(middlewares.VerifyPresignedV4Signature(root, iam, l, region, server.debug))
app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
app.Use(middlewares.ProcessChunkedBody(root, iam, l, region))
app.Use(middlewares.VerifyMD5Body(l))
app.Use(middlewares.AclParser(be, l, server.readonly))
app.Use(middlewares.AclParser(be, l))
server.router.Init(app, be, iam, l, adminLogger, evs, mm, server.debug, server.readonly)
server.router.Init(app, be, iam, l, evs, server.debug)
return server, nil
}
@@ -118,10 +103,6 @@ func WithHealth(health string) Option {
return func(s *S3ApiServer) { s.health = health }
}
func WithReadOnly() Option {
return func(s *S3ApiServer) { s.readonly = true }
}
func (sa *S3ApiServer) Serve() (err error) {
if sa.cert != nil {
return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)

View File

@@ -64,7 +64,7 @@ func TestNew(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil, nil, nil)
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil)
if (err != nil) != tt.wantErr {
t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
return

View File

@@ -1,17 +1,3 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (

View File

@@ -23,7 +23,6 @@ import (
"fmt"
"hash"
"io"
"math"
"strconv"
"time"
@@ -193,15 +192,12 @@ func (cr *ChunkReader) parseAndRemoveChunkInfo(p []byte) (int, error) {
cr.chunkDataLeft = 0
cr.chunkHash.Write(p[:chunkSize])
n, err := cr.parseAndRemoveChunkInfo(p[chunkSize:n])
if (chunkSize + int64(n)) > math.MaxInt {
return 0, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
}
return n + int(chunkSize), err
} else {
cr.chunkDataLeft = chunkSize - int64(n)
cr.chunkHash.Write(p[:n])
}
cr.chunkDataLeft = chunkSize - int64(n)
cr.chunkHash.Write(p[:n])
return n, nil
}
@@ -235,7 +231,6 @@ const (
// error if any. See the AWS documentation for the chunk header format. The
// header[0] byte is expected to be the first byte of the chunk size here.
func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int, error) {
stashLen := len(cr.stash)
if cr.stash != nil {
tmp := make([]byte, maxHeaderSize)
copy(tmp, cr.stash)
@@ -270,5 +265,5 @@ func (cr *ChunkReader) parseChunkHeaderBytes(header []byte) (int64, string, int,
signature := string(header[sigIndex:(sigIndex + sigEndIndex)])
dataStartOffset := sigIndex + sigEndIndex + len(chunkHdrDelim)
return chunkSize, signature, dataStartOffset - stashLen, nil
return chunkSize, signature, dataStartOffset, nil
}

View File

@@ -26,12 +26,10 @@ import (
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go/encoding/httpbinding"
"github.com/gofiber/fiber/v2"
"github.com/valyala/fasthttp"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
var (
@@ -224,118 +222,25 @@ func IsBigDataAction(ctx *fiber.Ctx) bool {
return false
}
// expiration time window
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html#RESTAuthenticationTimeStamp
const timeExpirationSec = 15 * 60
func ValidateDate(date time.Time) error {
now := time.Now().UTC()
diff := date.Unix() - now.Unix()
// Checks the dates difference to be within allotted window
if diff > timeExpirationSec || diff < -timeExpirationSec {
return s3err.GetAPIError(s3err.ErrRequestTimeTooSkewed)
// Checks the dates difference to be less than a minute
if diff > 60 {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
}
}
if diff < -60 {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
}
}
return nil
}
func ParseDeleteObjects(objs []types.ObjectIdentifier) (result []string) {
for _, obj := range objs {
result = append(result, *obj.Key)
}
return
}
func FilterObjectAttributes(attrs map[types.ObjectAttributes]struct{}, output s3response.GetObjectAttributesResult) s3response.GetObjectAttributesResult {
if _, ok := attrs[types.ObjectAttributesEtag]; !ok {
output.ETag = nil
}
if _, ok := attrs[types.ObjectAttributesObjectParts]; !ok {
output.ObjectParts = nil
}
if _, ok := attrs[types.ObjectAttributesObjectSize]; !ok {
output.ObjectSize = nil
}
if _, ok := attrs[types.ObjectAttributesStorageClass]; !ok {
output.StorageClass = ""
}
return output
}
func ParseObjectAttributes(ctx *fiber.Ctx) map[types.ObjectAttributes]struct{} {
attrs := map[types.ObjectAttributes]struct{}{}
ctx.Request().Header.VisitAll(func(key, value []byte) {
if string(key) == "X-Amz-Object-Attributes" {
oattrs := strings.Split(string(value), ",")
for _, a := range oattrs {
attrs[types.ObjectAttributes(a)] = struct{}{}
}
}
})
return attrs
}
type objLockCfg struct {
RetainUntilDate time.Time
ObjectLockMode types.ObjectLockMode
LegalHoldStatus types.ObjectLockLegalHoldStatus
}
func ParsObjectLockHdrs(ctx *fiber.Ctx) (*objLockCfg, error) {
legalHoldHdr := ctx.Get("X-Amz-Object-Lock-Legal-Hold")
objLockModeHdr := ctx.Get("X-Amz-Object-Lock-Mode")
objLockDate := ctx.Get("X-Amz-Object-Lock-Retain-Until-Date")
if (objLockDate != "" && objLockModeHdr == "") || (objLockDate == "" && objLockModeHdr != "") {
return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders)
}
var retainUntilDate time.Time
if objLockDate != "" {
rDate, err := time.Parse(time.RFC3339, objLockDate)
if err != nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
if rDate.Before(time.Now()) {
return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
}
retainUntilDate = rDate
}
objLockMode := types.ObjectLockMode(objLockModeHdr)
if objLockMode != "" &&
objLockMode != types.ObjectLockModeCompliance &&
objLockMode != types.ObjectLockModeGovernance {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
legalHold := types.ObjectLockLegalHoldStatus(legalHoldHdr)
if legalHold != "" && legalHold != types.ObjectLockLegalHoldStatusOff && legalHold != types.ObjectLockLegalHoldStatusOn {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
return &objLockCfg{
RetainUntilDate: retainUntilDate,
ObjectLockMode: objLockMode,
LegalHoldStatus: legalHold,
}, nil
}
func IsValidOwnership(val types.ObjectOwnership) bool {
switch val {
case types.ObjectOwnershipBucketOwnerEnforced:
return true
case types.ObjectOwnershipBucketOwnerPreferred:
return true
case types.ObjectOwnershipObjectWriter:
return true
default:
return false
}
}

View File

@@ -1,17 +1,3 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
@@ -20,10 +6,8 @@ import (
"reflect"
"testing"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gofiber/fiber/v2"
"github.com/valyala/fasthttp"
"github.com/versity/versitygw/s3response"
)
func TestCreateHttpRequestFromCtx(t *testing.T) {
@@ -280,105 +264,3 @@ func TestParseUint(t *testing.T) {
})
}
}
func TestFilterObjectAttributes(t *testing.T) {
type args struct {
attrs map[types.ObjectAttributes]struct{}
output s3response.GetObjectAttributesResult
}
etag, objSize := "etag", int64(3222)
tests := []struct {
name string
args args
want s3response.GetObjectAttributesResult
}{
{
name: "keep only ETag",
args: args{
attrs: map[types.ObjectAttributes]struct{}{
types.ObjectAttributesEtag: {},
},
output: s3response.GetObjectAttributesResult{
ObjectSize: &objSize,
ETag: &etag,
},
},
want: s3response.GetObjectAttributesResult{ETag: &etag},
},
{
name: "keep multiple props",
args: args{
attrs: map[types.ObjectAttributes]struct{}{
types.ObjectAttributesEtag: {},
types.ObjectAttributesObjectSize: {},
types.ObjectAttributesStorageClass: {},
},
output: s3response.GetObjectAttributesResult{
ObjectSize: &objSize,
ETag: &etag,
ObjectParts: &s3response.ObjectParts{},
VersionId: &etag,
},
},
want: s3response.GetObjectAttributesResult{
ETag: &etag,
ObjectSize: &objSize,
VersionId: &etag,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := FilterObjectAttributes(tt.args.attrs, tt.args.output); !reflect.DeepEqual(got, tt.want) {
t.Errorf("FilterObjectAttributes() = %v, want %v", got, tt.want)
}
})
}
}
func TestIsValidOwnership(t *testing.T) {
type args struct {
val types.ObjectOwnership
}
tests := []struct {
name string
args args
want bool
}{
{
name: "valid-BucketOwnerEnforced",
args: args{
val: types.ObjectOwnershipBucketOwnerEnforced,
},
want: true,
},
{
name: "valid-BucketOwnerPreferred",
args: args{
val: types.ObjectOwnershipBucketOwnerPreferred,
},
want: true,
},
{
name: "valid-ObjectWriter",
args: args{
val: types.ObjectOwnershipObjectWriter,
},
want: true,
},
{
name: "invalid_value",
args: args{
val: types.ObjectOwnership("invalid_value"),
},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := IsValidOwnership(tt.args.val); got != tt.want {
t.Errorf("IsValidOwnership() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -69,7 +69,6 @@ const (
ErrInvalidMaxParts
ErrInvalidPartNumberMarker
ErrInvalidPart
ErrInvalidPartNumber
ErrInternalError
ErrInvalidCopyDest
ErrInvalidCopySource
@@ -112,26 +111,6 @@ const (
ErrInvalidObjectState
ErrInvalidRange
ErrInvalidURI
ErrObjectLockConfigurationNotFound
ErrNoSuchObjectLockConfiguration
ErrInvalidBucketObjectLockConfiguration
ErrObjectLockConfigurationNotAllowed
ErrObjectLocked
ErrPastObjectLockRetainDate
ErrObjectLockInvalidRetentionPeriod
ErrNoSuchBucketPolicy
ErrBucketTaggingNotFound
ErrObjectLockInvalidHeaders
ErrRequestTimeTooSkewed
ErrInvalidBucketAclWithObjectOwnership
ErrBothCannedAndHeaderGrants
ErrOwnershipControlsNotFound
ErrAclNotSupported
ErrMalformedACL
ErrUnexpectedContent
ErrMissingSecurityHeader
ErrInvalidMetadataDirective
ErrKeyTooLong
// Non-AWS errors
ErrExistingObjectIsDirectory
@@ -153,7 +132,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrBucketNotEmpty: {
Code: "BucketNotEmpty",
Description: "The bucket you tried to delete is not empty.",
Description: "The bucket you tried to delete is not empty",
HTTPStatusCode: http.StatusConflict,
},
ErrBucketAlreadyExists: {
@@ -178,17 +157,17 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrInvalidMaxUploads: {
Code: "InvalidArgument",
Description: "Argument max-uploads must be an integer between 0 and 2147483647.",
Description: "Argument max-uploads must be an integer between 0 and 2147483647",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMaxKeys: {
Code: "InvalidArgument",
Description: "Argument maxKeys must be an integer between 0 and 2147483647.",
Description: "Argument maxKeys must be an integer between 0 and 2147483647",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMaxParts: {
Code: "InvalidArgument",
Description: "Argument max-parts must be an integer between 0 and 2147483647.",
Description: "Argument max-parts must be an integer between 0 and 2147483647",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidPartNumberMarker: {
@@ -198,7 +177,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrNoSuchBucket: {
Code: "NoSuchBucket",
Description: "The specified bucket does not exist.",
Description: "The specified bucket does not exist",
HTTPStatusCode: http.StatusNotFound,
},
ErrNoSuchKey: {
@@ -221,11 +200,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidPartNumber: {
Code: "InvalidArgument",
Description: "Part number must be an integer between 1 and 10000, inclusive.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidCopyDest: {
Code: "InvalidRequest",
Description: "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
@@ -268,7 +242,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrPostPolicyConditionInvalidFormat: {
Code: "PostPolicyInvalidKeyName",
Description: "Invalid according to Policy: Policy Condition failed.",
Description: "Invalid according to Policy: Policy Condition failed",
HTTPStatusCode: http.StatusForbidden,
},
ErrEntityTooSmall: {
@@ -303,7 +277,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrMalformedPresignedDate: {
Code: "AuthorizationQueryParametersError",
Description: "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\".",
Description: "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\"",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMissingSignHeadersTag: {
@@ -318,7 +292,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrUnsignedHeaders: {
Code: "AccessDenied",
Description: "There were headers present in the request which were not signed.",
Description: "There were headers present in the request which were not signed",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidQueryParams: {
@@ -333,22 +307,22 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrExpiredPresignRequest: {
Code: "AccessDenied",
Description: "Request has expired.",
Description: "Request has expired",
HTTPStatusCode: http.StatusForbidden,
},
ErrMalformedExpires: {
Code: "AuthorizationQueryParametersError",
Description: "X-Amz-Expires should be a number.",
Description: "X-Amz-Expires should be a number",
HTTPStatusCode: http.StatusBadRequest,
},
ErrNegativeExpires: {
Code: "AuthorizationQueryParametersError",
Description: "X-Amz-Expires must be non-negative.",
Description: "X-Amz-Expires must be non-negative",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMaximumExpires: {
Code: "AuthorizationQueryParametersError",
Description: "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds.",
Description: "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidAccessKeyID: {
@@ -358,7 +332,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrRequestNotReadyYet: {
Code: "AccessDenied",
Description: "Request is not valid yet.",
Description: "Request is not valid yet",
HTTPStatusCode: http.StatusForbidden,
},
ErrSignatureDoesNotMatch: {
@@ -368,17 +342,17 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrSignatureDateDoesNotMatch: {
Code: "SignatureDoesNotMatch",
Description: "Date in Credential scope does not match YYYYMMDD from ISO-8601 version of date from HTTP.",
Description: "Date in Credential scope does not match YYYYMMDD from ISO-8601 version of date from HTTP",
HTTPStatusCode: http.StatusForbidden,
},
ErrSignatureTerminationStr: {
Code: "SignatureDoesNotMatch",
Description: "Credential should be scoped with a valid terminator: 'aws4_request'.",
Description: "Credential should be scoped with a valid terminator: 'aws4_request'",
HTTPStatusCode: http.StatusForbidden,
},
ErrSignatureIncorrService: {
Code: "SignatureDoesNotMatch",
Description: "Credential should be scoped to correct service: s3.",
Description: "Credential should be scoped to correct service: s3",
HTTPStatusCode: http.StatusForbidden,
},
ErrContentSHA256Mismatch: {
@@ -388,32 +362,32 @@ var errorCodeResponse = map[ErrorCode]APIError{
},
ErrMissingDateHeader: {
Code: "AccessDenied",
Description: "AWS authentication requires a valid Date or x-amz-date header.",
Description: "AWS authentication requires a valid Date or x-amz-date header",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidRequest: {
Code: "InvalidRequest",
Description: "Invalid Request.",
Description: "Invalid Request",
HTTPStatusCode: http.StatusBadRequest,
},
ErrAuthNotSetup: {
Code: "InvalidRequest",
Description: "Signed request requires setting up SeaweedFS S3 authentication.",
Description: "Signed request requires setting up SeaweedFS S3 authentication",
HTTPStatusCode: http.StatusBadRequest,
},
ErrNotImplemented: {
Code: "NotImplemented",
Description: "A header you provided implies functionality that is not implemented.",
Description: "A header you provided implies functionality that is not implemented",
HTTPStatusCode: http.StatusNotImplemented,
},
ErrPreconditionFailed: {
Code: "PreconditionFailed",
Description: "At least one of the pre-conditions you specified did not hold.",
Description: "At least one of the pre-conditions you specified did not hold",
HTTPStatusCode: http.StatusPreconditionFailed,
},
ErrInvalidObjectState: {
Code: "InvalidObjectState",
Description: "The operation is not valid for the current state of the object.",
Description: "The operation is not valid for the current state of the object",
HTTPStatusCode: http.StatusForbidden,
},
ErrInvalidRange: {
@@ -426,108 +400,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "The specified URI couldn't be parsed.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrObjectLockConfigurationNotFound: {
Code: "ObjectLockConfigurationNotFoundError",
Description: "Object Lock configuration does not exist for this bucket.",
HTTPStatusCode: http.StatusNotFound,
},
ErrNoSuchObjectLockConfiguration: {
Code: "NoSuchObjectLockConfiguration",
Description: "The specified object does not have an ObjectLock configuration.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidBucketObjectLockConfiguration: {
Code: "InvalidRequest",
Description: "Bucket is missing ObjectLockConfiguration.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrObjectLockConfigurationNotAllowed: {
Code: "InvalidBucketState",
Description: "Object Lock configuration cannot be enabled on existing buckets.",
HTTPStatusCode: http.StatusConflict,
},
ErrObjectLocked: {
Code: "InvalidRequest",
Description: "Object is WORM protected and cannot be overwritten.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrPastObjectLockRetainDate: {
Code: "InvalidRequest",
Description: "the retain until date must be in the future.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrObjectLockInvalidRetentionPeriod: {
Code: "InvalidRetentionPeriod",
Description: "the retention days/years must be positive integer.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrNoSuchBucketPolicy: {
Code: "NoSuchBucketPolicy",
Description: "The bucket policy does not exist.",
HTTPStatusCode: http.StatusNotFound,
},
ErrBucketTaggingNotFound: {
Code: "NoSuchTagSet",
Description: "The TagSet does not exist.",
HTTPStatusCode: http.StatusNotFound,
},
ErrObjectLockInvalidHeaders: {
Code: "InvalidRequest",
Description: "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrRequestTimeTooSkewed: {
Code: "RequestTimeTooSkewed",
Description: "The difference between the request time and the server's time is too large.",
HTTPStatusCode: http.StatusForbidden,
},
ErrInvalidBucketAclWithObjectOwnership: {
Code: "ErrInvalidBucketAclWithObjectOwnership",
Description: "Bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrBothCannedAndHeaderGrants: {
Code: "InvalidRequest",
Description: "Specifying both Canned ACLs and Header Grants is not allowed.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrOwnershipControlsNotFound: {
Code: "OwnershipControlsNotFoundError",
Description: "The bucket ownership controls were not found.",
HTTPStatusCode: http.StatusNotFound,
},
ErrAclNotSupported: {
Code: "AccessControlListNotSupported",
Description: "The bucket does not allow ACLs.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMalformedACL: {
Code: "MalformedACLError",
Description: "The XML you provided was not well-formed or did not validate against our published schema.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrUnexpectedContent: {
Code: "UnexpectedContent",
Description: "This request does not support content.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMissingSecurityHeader: {
Code: "MissingSecurityHeader",
Description: "Your request was missing a required header.",
HTTPStatusCode: http.StatusNotFound,
},
ErrInvalidMetadataDirective: {
Code: "InvalidArgument",
Description: "Unknown metadata directive.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrKeyTooLong: {
Code: "KeyTooLongError",
Description: "Your key is too long.",
HTTPStatusCode: http.StatusBadRequest,
},
// non aws errors
ErrExistingObjectIsDirectory: {
Code: "ExistingObjectIsDirectory",
Description: "Existing Object is a directory.",

View File

@@ -37,11 +37,11 @@ type EventMeta struct {
VersionId *string
}
type EventSchema struct {
Records []EventRecord
type EventFields struct {
Records []EventSchema
}
type EventRecord struct {
type EventSchema struct {
EventVersion string `json:"eventVersion"`
EventSource string `json:"eventSource"`
AwsRegion string `json:"awsRegion"`
@@ -117,7 +117,7 @@ type EventConfig struct {
}
func InitEventSender(cfg *EventConfig) (S3EventSender, error) {
filter, err := parseEventFiltersFile(cfg.FilterConfigFilePath)
filter, err := parseEventFilters(cfg.FilterConfigFilePath)
if err != nil {
return nil, fmt.Errorf("parse event filter config file %w", err)
}
@@ -139,54 +139,54 @@ func InitEventSender(cfg *EventConfig) (S3EventSender, error) {
return evSender, err
}
func createEventSchema(ctx *fiber.Ctx, meta EventMeta, configId ConfigurationId) EventSchema {
func createEventSchema(ctx *fiber.Ctx, meta EventMeta, configId ConfigurationId) ([]byte, error) {
path := strings.Split(ctx.Path(), "/")
bucket, object := path[1], strings.Join(path[2:], "/")
acc := ctx.Locals("account").(auth.Account)
return EventSchema{
Records: []EventRecord{
{
EventVersion: "2.2",
EventSource: "aws:s3",
AwsRegion: ctx.Locals("region").(string),
EventTime: time.Now().Format(time.RFC3339),
EventName: meta.EventName,
UserIdentity: EventUserIdentity{
PrincipalId: acc.Access,
},
RequestParameters: EventRequestParams{
SourceIPAddress: ctx.IP(),
},
ResponseElements: EventResponseElements{
RequestId: ctx.Get("X-Amz-Request-Id"),
HostId: ctx.Get("X-Amz-Id-2"),
},
S3: EventS3Data{
S3SchemaVersion: "1.0",
ConfigurationId: configId,
Bucket: EventS3BucketData{
Name: bucket,
OwnerIdentity: EventUserIdentity{
PrincipalId: meta.BucketOwner,
},
Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
},
Object: EventObjectData{
Key: object,
Size: meta.ObjectSize,
ETag: meta.ObjectETag,
VersionId: meta.VersionId,
Sequencer: genSequencer(),
event := []EventSchema{
{
EventVersion: "2.2",
EventSource: "aws:s3",
AwsRegion: ctx.Locals("region").(string),
EventTime: time.Now().Format(time.RFC3339),
EventName: meta.EventName,
UserIdentity: EventUserIdentity{
PrincipalId: acc.Access,
},
RequestParameters: EventRequestParams{
SourceIPAddress: ctx.IP(),
},
ResponseElements: EventResponseElements{
RequestId: ctx.Get("X-Amz-Request-Id"),
HostId: ctx.Get("X-Amz-Id-2"),
},
S3: EventS3Data{
S3SchemaVersion: "1.0",
ConfigurationId: configId,
Bucket: EventS3BucketData{
Name: bucket,
OwnerIdentity: EventUserIdentity{
PrincipalId: meta.BucketOwner,
},
Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
},
GlacierEventData: EventGlacierData{
// Not supported
RestoreEventData: EventRestoreData{},
Object: EventObjectData{
Key: object,
Size: meta.ObjectSize,
ETag: meta.ObjectETag,
VersionId: meta.VersionId,
Sequencer: genSequencer(),
},
},
GlacierEventData: EventGlacierData{
// Not supported
RestoreEventData: EventRestoreData{},
},
},
}
return json.Marshal(event)
}
func generateTestEvent() ([]byte, error) {

View File

@@ -17,7 +17,6 @@ package s3event
import (
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strings"
@@ -26,21 +25,19 @@ import (
type EventType string
const (
EventObjectCreated EventType = "s3:ObjectCreated:*" // ObjectCreated
EventObjectCreatedPut EventType = "s3:ObjectCreated:Put"
EventObjectCreatedPost EventType = "s3:ObjectCreated:Post"
EventObjectCreatedCopy EventType = "s3:ObjectCreated:Copy"
EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
EventObjectRemoved EventType = "s3:ObjectRemoved:*"
EventObjectRemovedDelete EventType = "s3:ObjectRemoved:Delete"
EventObjectRemovedDeleteObjects EventType = "s3:ObjectRemoved:DeleteObjects" // non AWS custom type for DeleteObjects
EventObjectTagging EventType = "s3:ObjectTagging:*" // ObjectTagging
EventObjectTaggingPut EventType = "s3:ObjectTagging:Put"
EventObjectTaggingDelete EventType = "s3:ObjectTagging:Delete"
EventObjectAclPut EventType = "s3:ObjectAcl:Put"
EventObjectRestore EventType = "s3:ObjectRestore:*" // ObjectRestore
EventObjectRestorePost EventType = "s3:ObjectRestore:Post"
EventObjectRestoreCompleted EventType = "s3:ObjectRestore:Completed"
EventObjectCreated EventType = "s3:ObjectCreated:*" // ObjectCreated
EventObjectCreatedPut EventType = "s3:ObjectCreated:Put"
EventObjectCreatedPost EventType = "s3:ObjectCreated:Post"
EventObjectCreatedCopy EventType = "s3:ObjectCreated:Copy"
EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
EventObjectDeleted EventType = "s3:ObjectRemoved:Delete" // ObjectRemoved
EventObjectTagging EventType = "s3:ObjectTagging:*" // ObjectTagging
EventObjectTaggingPut EventType = "s3:ObjectTagging:Put"
EventObjectTaggingDelete EventType = "s3:ObjectTagging:Delete"
EventObjectAclPut EventType = "s3:ObjectAcl:Put"
EventObjectRestore EventType = "s3:ObjectRestore:*" // ObjectRestore
EventObjectRestorePost EventType = "s3:ObjectRestore:Post"
EventObjectRestoreCompleted EventType = "s3:ObjectRestore:Completed"
// EventObjectRestorePost EventType = "s3:ObjectRestore:Post"
// EventObjectRestoreDelete EventType = "s3:ObjectRestore:Delete"
)
@@ -51,26 +48,24 @@ func (event EventType) IsValid() bool {
}
var supportedEventFilters = map[EventType]struct{}{
EventObjectCreated: {},
EventObjectCreatedPut: {},
EventObjectCreatedPost: {},
EventObjectCreatedCopy: {},
EventCompleteMultipartUpload: {},
EventObjectRemoved: {},
EventObjectRemovedDelete: {},
EventObjectRemovedDeleteObjects: {},
EventObjectTagging: {},
EventObjectTaggingPut: {},
EventObjectTaggingDelete: {},
EventObjectAclPut: {},
EventObjectRestore: {},
EventObjectRestorePost: {},
EventObjectRestoreCompleted: {},
EventObjectCreated: {},
EventObjectCreatedPut: {},
EventObjectCreatedPost: {},
EventObjectCreatedCopy: {},
EventCompleteMultipartUpload: {},
EventObjectDeleted: {},
EventObjectTagging: {},
EventObjectTaggingPut: {},
EventObjectTaggingDelete: {},
EventObjectAclPut: {},
EventObjectRestore: {},
EventObjectRestorePost: {},
EventObjectRestoreCompleted: {},
}
type EventFilter map[EventType]bool
func parseEventFiltersFile(path string) (EventFilter, error) {
func parseEventFilters(path string) (EventFilter, error) {
// if no filter config file path is specified return nil map
if path == "" {
return nil, nil
@@ -88,12 +83,8 @@ func parseEventFiltersFile(path string) (EventFilter, error) {
}
defer file.Close()
return parseEventFilters(file)
}
func parseEventFilters(r io.Reader) (EventFilter, error) {
var filter EventFilter
if err := json.NewDecoder(r).Decode(&filter); err != nil {
if err := json.NewDecoder(file).Decode(&filter); err != nil {
return nil, err
}
@@ -121,7 +112,7 @@ func (ef EventFilter) Filter(event EventType) bool {
}
// check wildcard match
wildCardEv := EventType(string(event[:strings.LastIndex(string(event), ":")+1]) + "*")
wildCardEv := EventType(string(event[strings.LastIndex(string(event), ":")+1]) + "*")
wildcard, found := ef[wildCardEv]
if found {
return wildcard

View File

@@ -1,52 +0,0 @@
package s3event
import (
"strings"
"testing"
)
func TestFilterWildcardCreated(t *testing.T) {
filterString := `{"s3:ObjectCreated:*": true}`
strReader := strings.NewReader(filterString)
ef, err := parseEventFilters(strReader)
if err != nil {
t.Fatalf("failed to parse event filter: %v", err)
}
created := []string{
"s3:ObjectCreated:Put",
"s3:ObjectCreated:Post",
"s3:ObjectCreated:Copy",
"s3:ObjectCreated:CompleteMultipartUpload",
}
for _, event := range created {
allowed := ef.Filter(EventType(event))
if !allowed {
t.Errorf("expected event to be allowed: %s", event)
}
}
}
func TestFilterWildcardRemoved(t *testing.T) {
filterString := `{"s3:ObjectRemoved:*": true}`
strReader := strings.NewReader(filterString)
ef, err := parseEventFilters(strReader)
if err != nil {
t.Fatalf("failed to parse event filter: %v", err)
}
removed := []string{
"s3:ObjectRemoved:Delete",
"s3:ObjectRemoved:DeleteObjects",
}
for _, event := range removed {
allowed := ef.Filter(EventType(event))
if !allowed {
t.Errorf("expected event to be allowed: %s", event)
}
}
}

View File

@@ -16,8 +16,6 @@ package s3event
import (
"context"
"encoding/json"
"encoding/xml"
"fmt"
"os"
"sync"
@@ -25,7 +23,6 @@ import (
"github.com/gofiber/fiber/v2"
"github.com/segmentio/kafka-go"
"github.com/versity/versitygw/s3response"
)
var sequencer = 0
@@ -81,29 +78,12 @@ func (ks *Kafka) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
return
}
if meta.EventName == EventObjectRemovedDeleteObjects {
var dObj s3response.DeleteObjects
if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
fmt.Fprintf(os.Stderr, "failed to parse delete objects input payload: %v\n", err.Error())
return
}
// Events aren't send in correct order
for _, obj := range dObj.Objects {
key := *obj.Key
schema := createEventSchema(ctx, meta, ConfigurationIdWebhook)
schema.Records[0].S3.Object.Key = key
schema.Records[0].S3.Object.VersionId = obj.VersionId
go ks.send(schema)
}
schema, err := createEventSchema(ctx, meta, ConfigurationIdKafka)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to create kafka event: %v\n", err.Error())
return
}
schema := createEventSchema(ctx, meta, ConfigurationIdWebhook)
go ks.send(schema)
}
@@ -111,20 +91,14 @@ func (ks *Kafka) Close() error {
return ks.writer.Close()
}
func (ks *Kafka) send(event EventSchema) {
eventBytes, err := json.Marshal(event)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to parse event data: %v\n", err.Error())
return
}
func (ks *Kafka) send(event []byte) {
message := kafka.Message{
Key: []byte(ks.key),
Value: eventBytes,
Value: event,
}
ctx := context.Background()
err = ks.writer.WriteMessages(ctx, message)
err := ks.writer.WriteMessages(ctx, message)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to send kafka event: %v\n", err.Error())
}

View File

@@ -15,15 +15,12 @@
package s3event
import (
"encoding/json"
"encoding/xml"
"fmt"
"os"
"sync"
"github.com/gofiber/fiber/v2"
"github.com/nats-io/nats.go"
"github.com/versity/versitygw/s3response"
)
type NatsEventSender struct {
@@ -68,29 +65,12 @@ func (ns *NatsEventSender) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
return
}
if meta.EventName == EventObjectRemovedDeleteObjects {
var dObj s3response.DeleteObjects
if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
fmt.Fprintf(os.Stderr, "failed to parse delete objects input payload: %v\n", err.Error())
return
}
// Events aren't send in correct order
for _, obj := range dObj.Objects {
key := *obj.Key
schema := createEventSchema(ctx, meta, ConfigurationIdWebhook)
schema.Records[0].S3.Object.Key = key
schema.Records[0].S3.Object.VersionId = obj.VersionId
go ns.send(schema)
}
schema, err := createEventSchema(ctx, meta, ConfigurationIdNats)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to create nats event: %v\n", err.Error())
return
}
schema := createEventSchema(ctx, meta, ConfigurationIdWebhook)
go ns.send(schema)
}
@@ -99,13 +79,8 @@ func (ns *NatsEventSender) Close() error {
return nil
}
func (ns *NatsEventSender) send(event EventSchema) {
eventBytes, err := json.Marshal(event)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to parse event data: %v\n", err.Error())
return
}
err = ns.client.Publish(ns.topic, eventBytes)
func (ns *NatsEventSender) send(event []byte) {
err := ns.client.Publish(ns.topic, event)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to send nats event: %v\n", err.Error())
}

View File

@@ -16,8 +16,6 @@ package s3event
import (
"bytes"
"encoding/json"
"encoding/xml"
"fmt"
"net"
"net/http"
@@ -26,7 +24,6 @@ import (
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3response"
)
type Webhook struct {
@@ -80,29 +77,12 @@ func (w *Webhook) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
return
}
if meta.EventName == EventObjectRemovedDeleteObjects {
var dObj s3response.DeleteObjects
if err := xml.Unmarshal(ctx.Body(), &dObj); err != nil {
fmt.Fprintf(os.Stderr, "failed to parse delete objects input payload: %v\n", err.Error())
return
}
// Events aren't send in correct order
for _, obj := range dObj.Objects {
key := *obj.Key
schema := createEventSchema(ctx, meta, ConfigurationIdWebhook)
schema.Records[0].S3.Object.Key = key
schema.Records[0].S3.Object.VersionId = obj.VersionId
go w.send(schema)
}
schema, err := createEventSchema(ctx, meta, ConfigurationIdWebhook)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to create webhook event: %v\n", err.Error())
return
}
schema := createEventSchema(ctx, meta, ConfigurationIdWebhook)
go w.send(schema)
}
@@ -110,14 +90,8 @@ func (w *Webhook) Close() error {
return nil
}
func (w *Webhook) send(event EventSchema) {
eventBytes, err := json.Marshal(event)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to parse event data: %v\n", err.Error())
return
}
req, err := http.NewRequest(http.MethodPost, w.url, bytes.NewReader(eventBytes))
func (w *Webhook) send(event []byte) {
req, err := http.NewRequest(http.MethodPost, w.url, bytes.NewReader(event))
if err != nil {
fmt.Fprintf(os.Stderr, "failed to create webhook event request: %v\n", err.Error())
return

View File

@@ -35,13 +35,11 @@ type LogMeta struct {
BucketOwner string
ObjectSize int64
Action string
HttpStatus int
}
type LogConfig struct {
LogFile string
WebhookURL string
AdminLogFile string
LogFile string
WebhookURL string
}
type LogFields struct {
@@ -73,66 +71,18 @@ type LogFields struct {
AclRequired string
}
type AdminLogFields struct {
Time time.Time
RemoteIP string
Requester string
RequestID string
Operation string
RequestURI string
HttpStatus int
ErrorCode string
BytesSent int
TotalTime int64
TurnAroundTime int64
Referer string
UserAgent string
SignatureVersion string
CipherSuite string
AuthenticationType string
TLSVersion string
}
type Loggers struct {
S3Logger AuditLogger
AdminLogger AuditLogger
}
func InitLogger(cfg *LogConfig) (*Loggers, error) {
func InitLogger(cfg *LogConfig) (AuditLogger, error) {
if cfg.WebhookURL != "" && cfg.LogFile != "" {
return nil, fmt.Errorf("there should be specified one of the following: file, webhook")
}
loggers := new(Loggers)
switch {
case cfg.WebhookURL != "":
fmt.Printf("initializing S3 access logs with '%v' webhook url\n", cfg.WebhookURL)
l, err := InitWebhookLogger(cfg.WebhookURL)
if err != nil {
return nil, err
}
loggers.S3Logger = l
case cfg.LogFile != "":
fmt.Printf("initializing S3 access logs with '%v' file\n", cfg.LogFile)
l, err := InitFileLogger(cfg.LogFile)
if err != nil {
return nil, err
}
loggers.S3Logger = l
if cfg.WebhookURL != "" {
return InitWebhookLogger(cfg.WebhookURL)
}
if cfg.LogFile != "" {
return InitFileLogger(cfg.LogFile)
}
if cfg.AdminLogFile != "" {
fmt.Printf("initializing admin access logs with '%v' file\n", cfg.AdminLogFile)
l, err := InitAdminFileLogger(cfg.AdminLogFile)
if err != nil {
return nil, err
}
loggers.AdminLogger = l
}
return loggers, nil
return nil, nil
}
func genID() string {

View File

@@ -1,151 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3log
import (
"crypto/tls"
"fmt"
"os"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
)
// FileLogger is a local file audit log
type AdminFileLogger struct {
FileLogger
}
var _ AuditLogger = &AdminFileLogger{}
// InitFileLogger initializes audit logs to local file
func InitAdminFileLogger(logname string) (AuditLogger, error) {
f, err := os.OpenFile(logname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return nil, fmt.Errorf("open log: %w", err)
}
f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
return &AdminFileLogger{FileLogger: FileLogger{logfile: logname, f: f}}, nil
}
// Log sends log message to file logger
func (f *AdminFileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
f.mu.Lock()
defer f.mu.Unlock()
if f.gotErr {
return
}
lf := AdminLogFields{}
access := "-"
reqURI := ctx.OriginalURL()
errorCode := ""
startTime := ctx.Locals("startTime").(time.Time)
tlsConnState := ctx.Context().TLSConnectionState()
if tlsConnState != nil {
lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
}
if err != nil {
errorCode = err.Error()
}
switch ctx.Locals("account").(type) {
case auth.Account:
access = ctx.Locals("account").(auth.Account).Access
}
lf.Time = time.Now()
lf.RemoteIP = ctx.IP()
lf.Requester = access
lf.RequestID = genID()
lf.Operation = meta.Action
lf.RequestURI = reqURI
lf.HttpStatus = meta.HttpStatus
lf.ErrorCode = errorCode
lf.BytesSent = len(body)
lf.TotalTime = time.Since(startTime).Milliseconds()
lf.TurnAroundTime = time.Since(startTime).Milliseconds()
lf.Referer = ctx.Get("Referer")
lf.UserAgent = ctx.Get("User-Agent")
lf.SignatureVersion = "SigV4"
lf.AuthenticationType = "AuthHeader"
f.writeLog(lf)
}
func (f *AdminFileLogger) writeLog(lf AdminLogFields) {
if lf.RemoteIP == "" {
lf.RemoteIP = "-"
}
if lf.Requester == "" {
lf.Requester = "-"
}
if lf.Operation == "" {
lf.Operation = "-"
}
if lf.RequestURI == "" {
lf.RequestURI = "-"
}
if lf.ErrorCode == "" {
lf.ErrorCode = "-"
}
if lf.Referer == "" {
lf.Referer = "-"
}
if lf.UserAgent == "" {
lf.UserAgent = "-"
}
if lf.CipherSuite == "" {
lf.CipherSuite = "-"
}
if lf.TLSVersion == "" {
lf.TLSVersion = "-"
}
log := fmt.Sprintf("%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v\n",
fmt.Sprintf("[%v]", lf.Time.Format(timeFormat)),
lf.RemoteIP,
lf.Requester,
lf.RequestID,
lf.Operation,
lf.RequestURI,
lf.HttpStatus,
lf.ErrorCode,
lf.BytesSent,
lf.TotalTime,
lf.TurnAroundTime,
lf.Referer,
lf.UserAgent,
lf.SignatureVersion,
lf.CipherSuite,
lf.AuthenticationType,
lf.TLSVersion,
)
_, err := f.f.WriteString(log)
if err != nil {
fmt.Fprintf(os.Stderr, "error writing to log file: %v\n", err)
// TODO: do we need to terminate on log error?
// set err for now so that we don't spew errors
f.gotErr = true
}
}

Some files were not shown because too many files have changed in this diff Show More