Compare commits

..

1 Commits

Author SHA1 Message Date
Ben McClelland
17b42c369a add wiki assets 2023-06-10 19:49:40 -07:00
445 changed files with 26 additions and 110242 deletions

View File

@@ -1,46 +0,0 @@
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
cmd/versitygw/versitygw
/versitygw
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Go workspace file
go.work
# ignore IntelliJ directories
.idea
# auto generated VERSION file
VERSION
# build output
/versitygw.spec
/versitygw.spec.in
*.tar
*.tar.gz
**/rand.data
/profile.txt
dist/
# Release config files
/.github
# Docker configuration files
*Dockerfile
/docker-compose.yml
# read files
/LICENSE
/NOTICE
/CODE_OF_CONDUCT.md
/README.md

View File

@@ -1,8 +0,0 @@
POSIX_PORT=7071
PROXY_PORT=7070
ACCESS_KEY_ID=user
SECRET_ACCESS_KEY=pass
IAM_DIR=.
SETUP_DIR=.
AZ_ACCOUNT_NAME=devstoreaccount1
AZ_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==

View File

@@ -1,23 +0,0 @@
---
name: Bug Report
about: Create a report to help us improve
title: '[Bug] - <Short Description>'
labels: bug
assignees: ''
---
**Describe the bug**
<!-- A clear and concise description of what the bug is. -->
**To Reproduce**
<!-- Steps to reproduce the behavior. -->
**Expected behavior**
<!-- A clear and concise description of what you expected to happen. -->
**Server Version**
<!-- output of: './versitygw -version && uname -a' -->
**Additional context**
<!-- Describe s3 client and version if applicable.

View File

@@ -1,14 +0,0 @@
---
name: Feature Request
about: Suggest an idea for this project
title: '[Feature] - <Short Description>'
labels: enhancement
assignees: ''
---
**Describe the solution you'd like**
<!-- A clear and concise description of what you want to happen. -->
**Additional context**
<!-- Add any other context or screenshots about the feature request here. -->

View File

@@ -1,33 +0,0 @@
---
name: Test Case Request
about: Request new test cases or additional test coverage
title: '[Test Case] - <Short Description>'
labels: 'testcase'
assignees: ''
---
## Description
<!-- Please provide a detailed description of the test case or test coverage request. -->
## Purpose
<!-- Explain why this test case is important and what it aims to achieve. -->
## Scope
<!-- Describe the scope of the test case, including any specific functionalities, features, or modules that should be tested. -->
## Acceptance Criteria
<!-- List the criteria that must be met for the test case to be considered complete. -->
1.
2.
3.
## Additional Context
<!-- Add any other context or screenshots about the feature request here. -->
## Resources
<!-- Provide any resources, documentation, or links that could help in writing the test case. -->
**Thank you for contributing to our project!**

25
.github/SECURITY.md vendored
View File

@@ -1,25 +0,0 @@
# Security Policy
## Reporting a Vulnerability
If you discover a security vulnerability in `versitygw`, we strongly encourage you to report it privately and responsibly.
Please do **not** create public issues or pull requests that contain details about the vulnerability.
Instead, report the issue using GitHub's private **Security Advisories** feature:
- Go to [versitygw's Security Advisories page](https://github.com/versity/versitygw/security/advisories)
- Click on **"Report a vulnerability"**
We aim to respond within **2 business days** and work with you to quickly resolve the issue.
## Supported Versions
| Version | Supported |
| --------------- | --------- |
| Latest (v1.x.x) | ✅ |
| Older versions | ❌ |
## Responsible Disclosure
We appreciate responsible disclosures and are committed to fixing vulnerabilities in a timely manner. Thank you for helping keep `versitygw` secure.

View File

@@ -1,14 +0,0 @@
version: 2
updates:
- package-ecosystem: "gomod"
directory: "/"
schedule:
interval: "weekly"
groups:
dev-dependencies:
patterns:
- "*"
allow:
# Allow both direct and indirect updates for all packages
- dependency-type: "all"

View File

@@ -1,37 +0,0 @@
name: azurite functional tests
permissions: {}
on: pull_request
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 'stable'
id: go
- name: Set up Docker Compose
run: |
docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . up -d azurite azuritegw
- name: Wait for Azurite to be ready
run: sleep 40
- name: Get Dependencies
run: |
go mod download
- name: Build and Run
run: |
make
./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow --azure
- name: Shut down services
run: |
docker compose -f tests/docker-compose.yml --env-file .env.dev --project-directory . down azurite azuritegw

View File

@@ -1,29 +0,0 @@
name: docker bats tests
permissions: {}
on: pull_request
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build Docker Image
run: |
cp tests/.env.docker.default tests/.env.docker
cp tests/.secrets.default tests/.secrets
# see https://github.com/versity/versitygw/issues/1034
docker build \
--build-arg="GO_LIBRARY=go1.23.1.linux-amd64.tar.gz" \
--build-arg="AWS_CLI=awscli-exe-linux-x86_64.zip" \
--build-arg="MC_FOLDER=linux-amd64" \
--progress=plain \
-f tests/Dockerfile_test_bats \
-t bats_test .
- name: Run Docker Container
run: |
docker compose -f tests/docker-compose-bats.yml --project-directory . \
up --exit-code-from s3api_np_only s3api_np_only

View File

@@ -1,56 +0,0 @@
name: Publish Docker image
on:
release:
types: [published]
jobs:
push_to_registries:
name: Push Docker image to multiple registries
runs-on: ubuntu-latest
permissions:
packages: write
contents: read
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Log in to the Container registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v5
with:
images: |
versity/versitygw
ghcr.io/${{ github.repository }}
- name: Build and push Docker images
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64,linux/arm64
build-args: |
VERSION=${{ github.event.release.tag_name }}
TIME=${{ github.event.release.published_at }}
BUILD=${{ github.sha }}

View File

@@ -1,31 +0,0 @@
name: functional tests
permissions: {}
on: pull_request
jobs:
build:
name: RunTests
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 'stable'
id: go
- name: Get Dependencies
run: |
go mod download
- name: Build and Run
run: |
make testbin
./runtests.sh
- name: Coverage Report
run: |
go tool covdata percent -i=/tmp/covdata

View File

@@ -1,58 +0,0 @@
name: general
permissions: {}
on: pull_request
jobs:
build:
name: Go Basic Checks
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 'stable'
id: go
- name: Verify all files pass gofmt formatting
run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then gofmt -s -d .; exit 1; fi
- name: Get dependencies
run: |
go get -v -t -d ./...
- name: Test
run: go test -coverprofile profile.txt -race -v -timeout 30s -tags=github ./...
- name: Install govulncheck
run: go install golang.org/x/vuln/cmd/govulncheck@latest
shell: bash
- name: Run govulncheck
run: govulncheck ./...
shell: bash
verify-build:
name: Verify Build Targets
needs: build
runs-on: ubuntu-latest
strategy:
matrix:
os: [darwin, freebsd, linux]
arch: [amd64, arm64]
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 'stable'
- name: Build for ${{ matrix.os }}/${{ matrix.arch }}
run: |
GOOS=${{ matrix.os }} GOARCH=${{ matrix.arch }} go build -o versitygw-${{ matrix.os }}-${{ matrix.arch }} cmd/versitygw/*.go

View File

@@ -1,34 +0,0 @@
name: goreleaser
permissions:
contents: write
on:
push:
# run only against tags
tags:
- '*'
jobs:
goreleaser:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Fetch tags
run: git fetch --force --tags
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: stable
- name: Run Releaser
uses: goreleaser/goreleaser-action@v6
with:
distribution: goreleaser
version: '~> v2'
args: release --clean
env:
GITHUB_TOKEN: ${{ secrets.TOKEN }}

View File

@@ -1,13 +0,0 @@
name: host style tests
permissions: {}
on: pull_request
jobs:
build-and-run:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: run host-style tests
run: make test-host-style

View File

@@ -1,17 +0,0 @@
name: shellcheck
permissions: {}
on: pull_request
jobs:
build:
name: Run shellcheck
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Run checks
run: |
shellcheck --version
shellcheck -e SC1091 tests/*.sh tests/*/*.sh

View File

@@ -1,25 +0,0 @@
name: staticcheck
permissions: {}
on: pull_request
jobs:
build:
name: Check
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 'stable'
id: go
- name: "staticcheck"
uses: dominikh/staticcheck-action@v1
with:
version: "latest"

View File

@@ -1,263 +0,0 @@
name: system tests
permissions: {}
on: pull_request
jobs:
build:
name: RunTests
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- set: "mc, posix, non-file count, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "mc-non-file-count"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "mc, posix, file count, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "mc-file-count"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "REST, posix, non-static, base|acl|multipart|put-object, folder IAM"
IAM_TYPE: folder
RUN_SET: "rest-base,rest-acl,rest-multipart,rest-put-object"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "REST, posix, non-static, chunked|checksum|versioning|bucket, folder IAM"
IAM_TYPE: folder
RUN_SET: "rest-chunked,rest-checksum,rest-versioning,rest-bucket,rest-list-buckets,rest-create-bucket,rest-head-bucket"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "REST, posix, non-static, not implemented|rest-delete-bucket-ownership-controls|rest-delete-bucket-tagging, folder IAM"
IAM_TYPE: folder
RUN_SET: "rest-not-implemented,rest-delete-bucket-ownership-controls,rest-delete-bucket-tagging"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "REST, posix, static, base|acl|multipart|put-object, folder IAM"
IAM_TYPE: folder
RUN_SET: "rest-base,rest-acl,rest-multipart,rest-put-object"
RECREATE_BUCKETS: "false"
DELETE_BUCKETS_AFTER_TEST: "false"
BACKEND: "posix"
- set: "REST, posix, static, chunked|checksum|versioning|bucket, folder IAM"
IAM_TYPE: folder
RUN_SET: "rest-chunked,rest-checksum,rest-versioning,rest-bucket,rest-list-buckets,rest-create-bucket,rest-head-bucket"
RECREATE_BUCKETS: "false"
DELETE_BUCKETS_AFTER_TEST: "false"
BACKEND: "posix"
- set: "REST, posix, static, not implemented|rest-delete-bucket-ownership-controls|rest-delete-bucket-tagging, folder IAM"
IAM_TYPE: folder
RUN_SET: "rest-not-implemented,rest-delete-bucket-ownership-controls,rest-delete-bucket-tagging"
RECREATE_BUCKETS: "false"
DELETE_BUCKETS_AFTER_TEST: "false"
BACKEND: "posix"
- set: "s3, posix, non-file count, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3-non-file-count"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "s3, posix, file count, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3-file-count"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "s3api, posix, bucket|object|multipart, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "s3api, posix, policy, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-policy"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "s3api, posix, user, non-static, s3 IAM"
IAM_TYPE: s3
RUN_SET: "s3api-user"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "s3api, posix, bucket, static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-bucket"
RECREATE_BUCKETS: "false"
DELETE_BUCKETS_AFTER_TEST: "false"
BACKEND: "posix"
- set: "s3api, posix, multipart, static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-multipart"
RECREATE_BUCKETS: "false"
DELETE_BUCKETS_AFTER_TEST: "false"
BACKEND: "posix"
- set: "s3api, posix, object, static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-object"
RECREATE_BUCKETS: "false"
DELETE_BUCKETS_AFTER_TEST: "false"
BACKEND: "posix"
- set: "s3api, posix, policy, static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-policy"
RECREATE_BUCKETS: "false"
DELETE_BUCKETS_AFTER_TEST: "false"
BACKEND: "posix"
- set: "s3api, posix, user, static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3api-user"
RECREATE_BUCKETS: "false"
DELETE_BUCKETS_AFTER_TEST: "false"
BACKEND: "posix"
# TODO fix/debug s3 gateway
#- set: "s3api, s3, multipart|object, non-static, folder IAM"
# IAM_TYPE: folder
# RUN_SET: "s3api-bucket,s3api-object,s3api-multipart"
# RECREATE_BUCKETS: "true"
# BACKEND: "s3"
#- set: "s3api, s3, policy|user, non-static, folder IAM"
# IAM_TYPE: folder
# RUN_SET: "s3api-policy,s3api-user"
# RECREATE_BUCKETS: "true"
# BACKEND: "s3"
- set: "s3cmd, posix, file count, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3cmd-file-count"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "s3cmd, posix, non-user, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3cmd-non-user"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "s3cmd, posix, user, non-static, folder IAM"
IAM_TYPE: folder
RUN_SET: "s3cmd-user"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
- set: "setup/remove static buckets scripts"
IAM_TYPE: folder
RUN_SET: "setup-remove-static"
RECREATE_BUCKETS: "true"
DELETE_BUCKETS_AFTER_TEST: "true"
BACKEND: "posix"
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "stable"
id: go
- name: Get Dependencies
run: |
go get -v -t -d ./...
- name: Install BATS
run: |
git clone https://github.com/bats-core/bats-core.git
cd bats-core && ./install.sh $HOME
git clone https://github.com/bats-core/bats-support.git ${{ github.workspace }}/tests/bats-support
git clone https://github.com/ztombol/bats-assert.git ${{ github.workspace }}/tests/bats-assert
- name: Install s3cmd
run: |
sudo apt-get update
sudo apt-get install s3cmd
- name: Install mc
run: |
curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
chmod 755 /usr/local/bin/mc
- name: Install xml libraries (for rest)
run: |
sudo apt-get update
sudo apt-get install libxml2-utils xmlstarlet
# see https://github.com/versity/versitygw/issues/1034
- name: Install AWS cli
run: |
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip"
unzip -o awscliv2.zip
./aws/install -i ${{ github.workspace }}/aws-cli -b ${{ github.workspace }}/bin
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
- name: Build and run
env:
IAM_TYPE: ${{ matrix.IAM_TYPE }}
RUN_SET: ${{ matrix.RUN_SET }}
AWS_PROFILE: versity
VERSITY_EXE: ${{ github.workspace }}/versitygw
RUN_VERSITYGW: true
BACKEND: ${{ matrix.BACKEND }}
RECREATE_BUCKETS: ${{ matrix.RECREATE_BUCKETS }}
DELETE_BUCKETS_AFTER_TEST: ${{ matrix.DELETE_BUCKETS_AFTER_TEST }}
CERT: ${{ github.workspace }}/cert.pem
KEY: ${{ github.workspace }}/versitygw.pem
LOCAL_FOLDER: /tmp/gw
BUCKET_ONE_NAME: versity-gwtest-bucket-one
BUCKET_TWO_NAME: versity-gwtest-bucket-two
USERS_FOLDER: /tmp/iam
USERS_BUCKET: versity-gwtest-iam
AWS_ENDPOINT_URL: https://127.0.0.1:7070
PORT: 7070
S3CMD_CONFIG: tests/s3cfg.local.default
MC_ALIAS: versity
LOG_LEVEL: 4
GOCOVERDIR: ${{ github.workspace }}/cover
USERNAME_ONE: ABCDEFG
PASSWORD_ONE: 1234567
USERNAME_TWO: HIJKLMN
PASSWORD_TWO: 8901234
TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
REMOVE_TEST_FILE_FOLDER: true
VERSIONING_DIR: ${{ github.workspace }}/versioning
COMMAND_LOG: command.log
TIME_LOG: time.log
PYTHON_ENV_FOLDER: ${{ github.workspace }}/env
AUTOGENERATE_USERS: true
USER_AUTOGENERATION_PREFIX: github-actions-test-
run: |
make testbin
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
export AWS_REGION=us-east-1
export AWS_ACCESS_KEY_ID_TWO=user
export AWS_SECRET_ACCESS_KEY_TWO=pass
export AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
aws configure set aws_region $AWS_REGION --profile versity
mkdir $LOCAL_FOLDER
export WORKSPACE=$GITHUB_WORKSPACE
openssl genpkey -algorithm RSA -out $KEY -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key $KEY -out $CERT -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
mkdir $GOCOVERDIR $USERS_FOLDER
if [[ $RECREATE_BUCKETS == "false" ]]; then
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/setup_static.sh
fi
BYPASS_ENV_FILE=true ${{ github.workspace }}/tests/run.sh $RUN_SET
- name: Time report
run: |
if [ -e ${{ github.workspace }}/time.log ]; then
cat ${{ github.workspace }}/time.log
fi
- name: Coverage report
run: |
go tool covdata percent -i=cover

48
.gitignore vendored
View File

@@ -7,8 +7,6 @@
*.dll
*.so
*.dylib
cmd/versitygw/versitygw
/versitygw
# Test binary, built with `go test -c`
*.test
@@ -21,49 +19,3 @@ cmd/versitygw/versitygw
# Go workspace file
go.work
# ignore IntelliJ directories
.idea
# ignore VS code directories
.vscode
# auto generated VERSION file
VERSION
# build output
/versitygw.spec
*.tar
*.tar.gz
**/rand.data
/profile.txt
dist/
# secrets file for local github-actions testing
tests/.secrets*
# IAM users files often created in testing
users.json
users.json.backup
# env files for testing
**/.env*
**/!.env.default
# s3cmd config files (testing)
tests/s3cfg.local*
tests/!s3cfg.local.default
# keys
*.pem
# patches
*.patch
# grafana's local database (kept on filesystem for survival between instantiations)
metrics-exploration/grafana_data/**
# bats tools
/tests/bats-assert
/tests/bats-support

View File

@@ -1,128 +0,0 @@
version: 2
before:
hooks:
- go mod tidy
builds:
- goos:
- linux
- darwin
- freebsd
# windows is untested, we can start doing windows releases
# if someone is interested in taking on testing
# - windows
env:
# disable cgo to fix glibc issues: https://github.com/golang/go/issues/58550
# once we need to enable this, we will need to do per distro releases
- CGO_ENABLED=0
main: ./cmd/versitygw
binary: versitygw
goarch:
- amd64
- arm64
ldflags:
- -X=main.Build={{.Commit}} -X=main.BuildTime={{.Date}} -X=main.Version={{.Version}}
archives:
- formats: [ 'tar.gz' ]
# this name template makes the OS and Arch compatible with the results of uname.
name_template: >-
{{ .ProjectName }}_v{{ .Version }}_
{{- title .Os }}_
{{- if eq .Arch "amd64" }}x86_64
{{- else if eq .Arch "386" }}i386
{{- else }}{{ .Arch }}{{ end }}
{{- if .Arm }}v{{ .Arm }}{{ end }}
# Set this to true if you want all files in the archive to be in a single directory.
# If set to true and you extract the archive 'goreleaser_Linux_arm64.tar.gz',
# you'll get a folder 'goreleaser_Linux_arm64'.
# If set to false, all files are extracted separately.
# You can also set it to a custom folder name (templating is supported).
wrap_in_directory: true
# use zip for windows archives
format_overrides:
- goos: windows
formats: [ 'zip' ]
# Additional files/globs you want to add to the archive.
#
# Default: [ 'LICENSE*', 'README*', 'CHANGELOG', 'license*', 'readme*', 'changelog']
# Templates: allowed
files:
- README.md
- LICENSE
- NOTICE
checksum:
name_template: 'checksums.txt'
snapshot:
version_template: "{{ incpatch .Version }}-{{.ShortCommit}}"
changelog:
sort: asc
filters:
exclude:
- '^docs:'
- '^test:'
- '^Merge '
nfpms:
- id: packages
package_name: versitygw
vendor: Versity Software
homepage: https://github.com/versity/versitygw
maintainer: Ben McClelland <ben.mcclelland@versity.com>
description: |-
The Versity S3 Gateway.
A high-performance tool facilitating translation between AWS S3 API
requests and various backend storage systems, including POSIX file
backend storage. Its stateless architecture enables deployment in
clusters for increased throughput, distributing requests across gateways
for optimal performance. With a focus on modularity, it supports future
extensions for additional backend systems.
license: Apache 2.0
ids:
- versitygw
formats:
- deb
- rpm
umask: 0o002
bindir: /usr/bin
epoch: "1"
release: "1"
rpm:
group: "System Environment/Daemons"
# RPM specific scripts.
scripts:
# The pretrans script runs before all RPM package transactions / stages.
#pretrans: ./extra/pretrans.sh
# The posttrans script runs after all RPM package transactions / stages.
posttrans: ./extra/posttrans.sh
contents:
- src: extra/versitygw@.service
dst: /lib/systemd/system/versitygw@.service
- src: extra/example.conf
dst: /etc/versitygw.d/example.conf
type: config
- dst: /etc/versitygw.d
type: dir
file_info:
mode: 0700
# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
# vim: set ts=2 sw=2 tw=0 fo=cnqoj

View File

@@ -1,128 +0,0 @@
# Contributor Covenant Code of Conduct
## Our Pledge
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, religion, or sexual identity
and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to a positive environment for our
community include:
* Demonstrating empathy and kindness toward other people
* Being respectful of differing opinions, viewpoints, and experiences
* Giving and gracefully accepting constructive feedback
* Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
* Focusing on what is best not just for us as individuals, but for the
overall community
Examples of unacceptable behavior include:
* The use of sexualized language or imagery, and sexual attention or
advances of any kind
* Trolling, insulting or derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or email
address, without their explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Enforcement Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
versitygw@versity.com.
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series
of actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or
permanent ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within
the community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.0, available at
https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
Community Impact Guidelines were inspired by [Mozilla's code of conduct
enforcement ladder](https://github.com/mozilla/diversity).
[homepage]: https://www.contributor-covenant.org
For answers to common questions about this code of conduct, see the FAQ at
https://www.contributor-covenant.org/faq. Translations are available at
https://www.contributor-covenant.org/translations.

View File

@@ -1,35 +0,0 @@
FROM golang:latest
# Set build arguments with default values
ARG VERSION="none"
ARG BUILD="none"
ARG TIME="none"
# Set environment variables
ENV VERSION=${VERSION}
ENV BUILD=${BUILD}
ENV TIME=${TIME}
WORKDIR /app
COPY go.mod ./
RUN go mod download
COPY ./ ./
WORKDIR /app/cmd/versitygw
ENV CGO_ENABLED=0
RUN go build -ldflags "-X=main.Build=${BUILD} -X=main.BuildTime=${TIME} -X=main.Version=${VERSION}" -o versitygw
FROM alpine:latest
# These arguments can be overriden when building the image
ARG IAM_DIR=/tmp/vgw
ARG SETUP_DIR=/tmp/vgw
RUN mkdir -p $IAM_DIR
RUN mkdir -p $SETUP_DIR
COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
ENTRYPOINT [ "/app/versitygw" ]

104
Makefile
View File

@@ -1,104 +0,0 @@
# Copyright 2023 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Go parameters
GOCMD=go
GOBUILD=$(GOCMD) build
GOCLEAN=$(GOCMD) clean
GOTEST=$(GOCMD) test
# docker-compose
DCCMD=docker-compose
DOCKERCOMPOSE=$(DCCMD) -f tests/docker-compose.yml --env-file .env.dev --project-directory .
BIN=versitygw
VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
BUILD := $(shell git rev-parse --short HEAD || echo release-rpm)
TIME := `date -u '+%Y-%m-%d_%I:%M:%S%p'`
LDFLAGS=-ldflags "-X=main.Build=$(BUILD) -X=main.BuildTime=$(TIME) -X=main.Version=$(VERSION)"
all: build
build: $(BIN)
.PHONY: $(BIN)
$(BIN):
$(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go
testbin:
$(GOBUILD) $(LDFLAGS) -o $(BIN) -cover -race cmd/$(BIN)/*.go
.PHONY: test
test:
$(GOTEST) ./...
.PHONY: check
check:
# note this requires staticcheck be in your PATH:
# export PATH=$PATH:~/go/bin
# go install honnef.co/go/tools/cmd/staticcheck@latest
staticcheck ./...
golint ./...
gofmt -s -l .
.PHONY: clean
clean:
$(GOCLEAN)
.PHONY: cleanall
cleanall: clean
rm -f $(BIN)
rm -f versitygw-*.tar
rm -f versitygw-*.tar.gz
TARFILE = $(BIN)-$(VERSION).tar
dist:
echo $(VERSION) >VERSION
git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
rm -f VERSION
gzip -f $(TARFILE)
.PHONY: snapshot
snapshot:
# brew install goreleaser/tap/goreleaser
goreleaser release --snapshot --skip publish --clean
# Creates and runs S3 gateway instance in a docker container
.PHONY: up-posix
up-posix:
$(DOCKERCOMPOSE) up posix
# Creates and runs S3 gateway proxy instance in a docker container
.PHONY: up-proxy
up-proxy:
$(DOCKERCOMPOSE) up proxy
# Creates and runs S3 gateway to azurite instance in a docker container
.PHONY: up-azurite
up-azurite:
$(DOCKERCOMPOSE) up azurite azuritegw
# Creates and runs both S3 gateway and proxy server instances in docker containers
.PHONY: up-app
up-app:
$(DOCKERCOMPOSE) up
# Run the host-style tests in docker containers
.PHONY: test-host-style
test-host-style:
docker compose -f tests/host-style-tests/docker-compose.yml up --build --abort-on-container-exit --exit-code-from test

2
NOTICE
View File

@@ -1,2 +0,0 @@
versitygw - Versity S3 Gateway
Copyright 2023 Versity Software

View File

@@ -1,87 +0,0 @@
# The Versity S3 Gateway:<br/>A High-Performance S3 Translation Service
<picture>
<source media="(prefers-color-scheme: dark)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo-white.svg">
<source media="(prefers-color-scheme: light)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo.svg">
<a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
</picture>
[![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/versity/versitygw)](https://goreportcard.com/report/github.com/versity/versitygw) [![Go Reference](https://pkg.go.dev/badge/github.com/versity/versitygw.svg)](https://pkg.go.dev/github.com/versity/versitygw)
### Binary release builds
Download [latest release](https://github.com/versity/versitygw/releases)
| Linux/amd64 | Linux/arm64 | MacOS/amd64 | MacOS/arm64 | BSD/amd64 | BSD/arm64 |
|:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
| ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
### Use Cases
* Turn your local filesystem into an S3 server with a single command!
* Proxy S3 requests to S3 storage
* Simple to deploy S3 server with a single command
* Protocol compatibility in `posix` allows common access to files via posix or S3
* Simplified interface for adding new storage system support
### News
Check out latest wiki articles: [https://github.com/versity/versitygw/wiki/Articles](https://github.com/versity/versitygw/wiki/Articles)
### Mailing List
Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).
### Documentation
See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
### Need help?
Ask questions in the [community discussions](https://github.com/versity/versitygw/discussions).
<br>
Contact [Versity Sales](https://www.versity.com/contact/) to discuss enterprise support.
### Overview
Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage, Versitys open source ScoutFS filesystem, Azure Blob Storage, and other S3 servers.
The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateways stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance.
The S3 HTTP(S) server and routing is implemented using the [Fiber](https://gofiber.io) web framework. This framework is actively developed with a focus on performance. S3 API compatibility leverages the official [aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) whenever possible for maximum service compatibility with AWS S3.
## Getting Started
See the [Quickstart](https://github.com/versity/versitygw/wiki/Quickstart) documentation.
### Run the gateway with posix backend:
```
mkdir /tmp/vgw
ROOT_ACCESS_KEY="testuser" ROOT_SECRET_KEY="secret" ./versitygw --port :10000 posix /tmp/vgw
```
This will enable an S3 server on the current host listening on port 10000 and hosting the directory `/tmp/vgw`.
To get the usage output, run the following:
```
./versitygw --help
```
The command format is
```
versitygw [global options] command [command options] [arguments...]
```
The [global options](https://github.com/versity/versitygw/wiki/Global-Options) are specified before the backend type and the backend options are specified after.
***
#### Versity gives you clarity and control over your archival storage, so you can allocate more resources to your core mission.
### Contact
![versity logo](https://www.versity.com/wp-content/uploads/2022/12/cropped-android-chrome-512x512-1-32x32.png)
info@versity.com <br />
+1 844 726 8826
### @versitysoftware
[![linkedin](https://github.com/versity/versitygw/blob/assets/assets/linkedin.jpg)](https://www.linkedin.com/company/versity/) &nbsp;
[![twitter](https://github.com/versity/versitygw/blob/assets/assets/twitter.jpg)](https://twitter.com/VersitySoftware) &nbsp;
[![facebook](https://github.com/versity/versitygw/blob/assets/assets/facebook.jpg)](https://www.facebook.com/versitysoftware) &nbsp;
[![instagram](https://github.com/versity/versitygw/blob/assets/assets/instagram.jpg)](https://www.instagram.com/versitysoftware/) &nbsp;

BIN
assets/facebook.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 470 B

BIN
assets/instagram.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 461 B

BIN
assets/linkedin.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 391 B

13
assets/logo-white.svg Normal file
View File

@@ -0,0 +1,13 @@
<svg width="209" height="73" viewBox="0 0 209 73" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M47.427 30.0292C42.767 30.9134 40.4568 30.3528 40.2354 29.406C40.1625 29.0919 40.2887 28.6812 40.6122 28.1848C42.2518 25.6629 47.9677 22.1097 54.5119 19.5417C57.6668 18.3011 60.7686 17.3735 63.4818 16.8593C68.1376 15.976 70.4445 16.5363 70.6646 17.4827C70.7388 17.7976 70.6123 18.2078 70.2897 18.7048C68.6525 21.2235 62.9369 24.7783 56.3908 27.3497C53.2389 28.5879 50.1396 29.5145 47.4284 30.0287L47.427 30.0292Z" fill="white"/>
<path d="M48.4866 39.9975C42.4383 40.4446 38.4817 39.0018 38.7133 37.4258C38.763 37.0777 39.0232 36.6989 39.4845 36.301C41.0086 34.9865 44.833 33.4548 49.9427 32.9982C50.1102 32.9836 50.2764 32.9697 50.4411 32.9574C56.4783 32.5107 60.4292 33.9536 60.1996 35.5285C60.1487 35.8758 59.8879 36.2549 59.4247 36.6544C57.8987 37.9699 54.0743 39.5016 48.9726 39.9577C48.8092 39.9718 48.648 39.9852 48.4871 39.997L48.4866 39.9975Z" fill="white"/>
<path d="M44.5318 47.6969C44.5093 44.8455 42.0643 42.5515 39.0706 42.5727C37.1309 42.5866 35.4376 43.57 34.4912 45.0354L34.4904 45.0349L34.4755 45.0595C34.4417 45.1127 34.4089 45.1664 34.3773 45.2211L29.6958 52.93L10.1292 20.2803C8.6447 17.8035 5.33413 16.9425 2.73347 18.3555C0.192505 19.737 -0.72811 22.7797 0.616915 25.2285L0.615815 25.2295L0.643322 25.2748C0.666427 25.3162 0.688702 25.3579 0.712632 25.3987C0.736837 25.4396 0.763518 25.4786 0.788548 25.5179L28.8811 72.4019L28.8822 72.3956C29.031 72.6689 29.3281 72.8577 29.673 72.8577C29.9979 72.8577 30.2801 72.6909 30.4363 72.4433L30.4374 72.4467L43.8764 50.2044L43.8731 50.2028C44.2991 49.4586 44.5387 48.6041 44.5318 47.6969Z" fill="white"/>
<path d="M69.6803 0.677278C68.2968 -1.32435 60.4824 1.26118 52.2266 6.45006C43.9703 11.641 38.3998 17.4709 39.7834 19.472C41.1669 21.4739 48.9818 18.8889 57.2376 13.6979C65.4929 8.50826 71.0644 2.67916 69.6803 0.677278Z" fill="white"/>
<path d="M108.898 20.9902L101.884 39.4324H101.798L94.2404 20.9902H87.6453L99.6694 51.7645H103.261L115.498 20.9902H108.898Z" fill="white"/>
<path d="M118.544 38.6442C118.626 38.1404 118.765 37.6502 118.962 37.1708C119.155 36.6966 119.415 36.2812 119.733 35.921C120.056 35.5652 120.444 35.2711 120.903 35.0454C121.363 34.8191 121.884 34.7061 122.468 34.7061C123.024 34.7061 123.523 34.8191 123.97 35.0454C124.415 35.2711 124.807 35.5704 125.141 35.9406C125.473 36.3138 125.741 36.7284 125.932 37.1912C126.127 37.6563 126.269 38.1404 126.35 38.6442H118.544ZM129.504 33.9109C128.711 32.956 127.701 32.2004 126.479 31.6458C125.252 31.0894 123.803 30.8105 122.135 30.8105C120.464 30.8105 119.009 31.0807 117.772 31.6236C116.534 32.1708 115.504 32.9168 114.681 33.8708C113.86 34.8261 113.242 35.9458 112.823 37.233C112.405 38.5185 112.197 39.9027 112.197 41.3869C112.197 42.8712 112.433 44.238 112.907 45.4838C113.38 46.7301 114.055 47.8124 114.933 48.726C115.807 49.6369 116.88 50.3485 118.147 50.8501C119.415 51.3556 120.838 51.6087 122.427 51.6087C124.6 51.6087 126.509 51.0785 128.166 50.0223C129.822 48.9665 131.014 47.4522 131.737 45.4838L126.35 44.6495C125.96 45.4181 125.454 46.0528 124.827 46.5552C124.2 47.0607 123.399 47.3139 122.427 47.3139C121.676 47.3139 121.036 47.1525 120.503 46.8397C119.975 46.5226 119.559 46.1185 119.256 45.6122C118.948 45.1158 118.722 44.5568 118.587 43.94C118.445 43.3197 118.376 42.7098 118.376 42.1055H131.863V41.4282C131.863 39.9705 131.668 38.5907 131.277 37.2935C130.887 35.9915 130.298 34.8648 129.504 33.9109Z" fill="white"/>
<path d="M145.732 30.8105C144.645 30.8105 143.674 31.0424 142.81 31.5039C141.947 31.9694 141.223 32.6245 140.638 33.4759H140.557V31.4487H134.71V50.9689H140.557V42.1821C140.557 41.4413 140.597 40.684 140.68 39.9127C140.765 39.1471 140.959 38.4511 141.265 37.8294C141.572 37.2039 142.023 36.7028 142.621 36.3178C143.222 35.9345 144.021 35.7392 145.025 35.7392C145.915 35.7392 146.721 35.9806 147.448 36.4565L148.155 31.2864C147.765 31.1546 147.367 31.0424 146.966 30.9489C146.562 30.8562 146.152 30.8105 145.732 30.8105Z" fill="white"/>
<path d="M162.394 40.9323C161.99 40.4929 161.537 40.1267 161.033 39.8387C160.538 39.5464 160.021 39.2923 159.492 39.0792C158.962 38.8669 158.447 38.679 157.948 38.505C157.444 38.3323 156.993 38.1483 156.591 37.9503C156.187 37.7498 155.868 37.5314 155.631 37.2935C155.395 37.0529 155.273 36.7497 155.273 36.3773C155.273 35.9001 155.477 35.5025 155.883 35.1837C156.283 34.8661 156.722 34.7056 157.195 34.7056C157.779 34.7056 158.339 34.8396 158.865 35.1063C159.396 35.3707 159.868 35.6883 160.286 36.0598L162.789 32.8403C161.955 32.1482 160.919 31.6401 159.681 31.3069C158.44 30.9754 157.28 30.8105 156.193 30.8105C155.219 30.8105 154.336 30.9706 153.539 31.2864C152.751 31.6062 152.067 32.0425 151.496 32.6002C150.925 33.1574 150.479 33.8269 150.16 34.6086C149.839 35.3929 149.68 36.2312 149.68 37.1321C149.68 37.9273 149.799 38.5985 150.034 39.1405C150.272 39.6869 150.577 40.1549 150.952 40.5525C151.327 40.9519 151.765 41.2816 152.268 41.5483C152.772 41.8141 153.269 42.0511 153.772 42.263C154.273 42.4761 154.775 42.6636 155.273 42.8381C155.776 43.0112 156.216 43.2044 156.591 43.4167C156.965 43.6281 157.271 43.8738 157.509 44.1522C157.743 44.432 157.861 44.7821 157.861 45.2032C157.861 45.8396 157.634 46.3547 157.175 46.7371C156.716 47.1194 156.167 47.3139 155.525 47.3139C154.661 47.3139 153.861 47.0868 153.124 46.6348C152.39 46.1863 151.739 45.6552 151.181 45.0479L148.511 48.3444C149.399 49.3802 150.479 50.1841 151.747 50.7531C153.011 51.3234 154.342 51.6087 155.735 51.6087C156.819 51.6087 157.841 51.4508 158.805 51.1293C159.76 50.81 160.605 50.3555 161.329 49.7591C162.051 49.1635 162.63 48.4388 163.06 47.591C163.494 46.7414 163.709 45.7735 163.709 44.6886C163.709 43.8408 163.59 43.1117 163.352 42.4992C163.118 41.8915 162.797 41.369 162.394 40.9323Z" fill="white"/>
<path d="M169.093 22.4395C168.2 22.4395 167.45 22.7401 166.841 23.3317C166.233 23.9237 165.926 24.6332 165.926 25.458C165.926 26.3045 166.233 27.0171 166.841 27.5952C167.45 28.1777 168.2 28.4674 169.093 28.4674C169.98 28.4674 170.731 28.1777 171.339 27.5952C171.951 27.0171 172.257 26.3045 172.257 25.458C172.257 24.6332 171.951 23.9237 171.339 23.3317C170.731 22.7401 169.98 22.4395 169.093 22.4395Z" fill="white"/>
<path d="M172.014 31.4492H166.167V50.9694H172.014V31.4492Z" fill="white"/>
<path d="M202.482 31.4492L197.425 41.9607L192.336 31.4492H185.418H184.587H184.212V22.9521H178.364V31.4492H174.405V36.139H178.364V50.9693H184.212V36.139H188.688L194.422 47.9882L189.519 57.6672H195.949L209 31.4492H202.482Z" fill="white"/>
</svg>

After

Width:  |  Height:  |  Size: 6.4 KiB

13
assets/logo.svg Normal file
View File

@@ -0,0 +1,13 @@
<svg width="150" height="53" viewBox="0 0 150 53" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M33.903 21.4665C30.5718 22.0985 28.9204 21.6977 28.7621 21.021C28.71 20.7965 28.8003 20.5028 29.0315 20.148C30.2036 18.3452 34.2896 15.8052 38.9676 13.9695C41.2229 13.0827 43.4402 12.4196 45.3797 12.052C48.7079 11.4206 50.357 11.8211 50.5143 12.4977C50.5673 12.7228 50.4769 13.016 50.2463 13.3713C49.076 15.1718 44.9902 17.7129 40.3107 19.551C38.0576 20.4362 35.8421 21.0985 33.904 21.4661L33.903 21.4665Z" fill="#191B2A"/>
<path d="M34.6603 28.592C30.3367 28.9116 27.5083 27.8802 27.6739 26.7536C27.7095 26.5048 27.8955 26.234 28.2252 25.9495C29.3147 25.0099 32.0485 23.915 35.7012 23.5886C35.8209 23.5781 35.9397 23.5682 36.0575 23.5594C40.3731 23.2401 43.1974 24.2715 43.0332 25.3973C42.9969 25.6456 42.8105 25.9166 42.4794 26.2021C41.3885 27.1426 38.6547 28.2375 35.0077 28.5635C34.8909 28.5736 34.7757 28.5832 34.6607 28.5916L34.6603 28.592Z" fill="#191B2A"/>
<path d="M31.8333 34.096C31.8172 32.0577 30.0694 30.4178 27.9294 30.433C26.5428 30.4429 25.3324 31.1459 24.6558 32.1934L24.6552 32.193L24.6446 32.2106C24.6204 32.2487 24.597 32.287 24.5744 32.3262L21.2279 37.8368L7.24078 14.4974C6.17961 12.7269 3.81307 12.1114 1.95401 13.1214C0.137611 14.109 -0.520485 16.2841 0.440998 18.0346L0.440212 18.0353L0.459875 18.0677C0.476391 18.0973 0.492314 18.1271 0.509421 18.1563C0.526723 18.1855 0.545796 18.2134 0.563688 18.2415L20.6455 51.7562L20.6463 51.7517C20.7527 51.947 20.965 52.082 21.2116 52.082C21.4438 52.082 21.6455 51.9628 21.7572 51.7858L21.758 51.7882L31.3648 35.8884L31.3624 35.8873C31.667 35.3553 31.8382 34.7444 31.8333 34.096Z" fill="#191B2A"/>
<path d="M49.8105 0.484148C48.8215 -0.9467 43.2354 0.90155 37.3338 4.61078C31.4318 8.32151 27.4498 12.489 28.4388 13.9194C29.4278 15.3505 35.0143 13.5026 40.9159 9.79187C46.8171 6.08208 50.7999 1.91518 49.8105 0.484148Z" fill="#191B2A"/>
<path d="M77.8452 15.0049L72.8313 28.1882H72.7699L67.3671 15.0049H62.6526L71.248 37.0037H73.8157L82.5626 15.0049H77.8452Z" fill="#191B2A"/>
<path d="M84.7403 27.625C84.7988 27.2649 84.898 26.9144 85.0391 26.5717C85.1775 26.2328 85.363 25.9358 85.5906 25.6783C85.8211 25.424 86.0986 25.2138 86.4268 25.0524C86.7556 24.8907 87.1281 24.8098 87.5454 24.8098C87.9428 24.8098 88.2996 24.8907 88.619 25.0524C88.9373 25.2138 89.2178 25.4277 89.4565 25.6923C89.6938 25.9591 89.8848 26.2555 90.0216 26.5863C90.1614 26.9188 90.2623 27.2649 90.3207 27.625H84.7403ZM92.575 24.2414C92.0085 23.5588 91.2866 23.0187 90.4128 22.6222C89.5358 22.2245 88.5001 22.0251 87.3074 22.0251C86.113 22.0251 85.073 22.2183 84.1885 22.6063C83.3034 22.9975 82.5674 23.5308 81.9787 24.2128C81.3923 24.8956 80.9502 25.6961 80.6504 26.6162C80.352 27.5351 80.2031 28.5246 80.2031 29.5856C80.2031 30.6466 80.3723 31.6237 80.7108 32.5143C81.0488 33.4052 81.5314 34.1788 82.1593 34.8319C82.7839 35.483 83.5512 35.9918 84.4566 36.3503C85.363 36.7116 86.3801 36.8926 87.516 36.8926C89.0695 36.8926 90.434 36.5135 91.6183 35.7585C92.8022 35.0038 93.6544 33.9214 94.1716 32.5143L90.3207 31.9178C90.0419 32.4673 89.6798 32.921 89.2318 33.2801C88.7838 33.6415 88.2111 33.8225 87.516 33.8225C86.9795 33.8225 86.5218 33.7071 86.1411 33.4835C85.7636 33.2568 85.4658 32.9679 85.249 32.606C85.029 32.2512 84.8677 31.8516 84.7713 31.4106C84.6695 30.9672 84.6202 30.5312 84.6202 30.0993H94.2611V29.6151C94.2611 28.5731 94.1217 27.5867 93.8425 26.6594C93.5636 25.7287 93.1428 24.9233 92.575 24.2414Z" fill="#191B2A"/>
<path d="M104.175 22.0251C103.398 22.0251 102.704 22.1909 102.087 22.5208C101.47 22.8536 100.952 23.3219 100.534 23.9304H100.476V22.4813H96.2964V36.4352H100.476V30.154C100.476 29.6245 100.505 29.0831 100.564 28.5318C100.625 27.9845 100.763 27.4869 100.982 27.0425C101.202 26.5954 101.524 26.2372 101.952 25.962C102.381 25.688 102.952 25.5484 103.67 25.5484C104.306 25.5484 104.883 25.721 105.402 26.0611L105.908 22.3653C105.629 22.2711 105.345 22.1909 105.058 22.124C104.769 22.0578 104.476 22.0251 104.175 22.0251Z" fill="#191B2A"/>
<path d="M116.086 29.2606C115.797 28.9465 115.474 28.6847 115.114 28.4789C114.759 28.2699 114.39 28.0883 114.012 27.9359C113.633 27.7842 113.265 27.6498 112.908 27.5255C112.548 27.402 112.225 27.2705 111.938 27.129C111.649 26.9856 111.421 26.8295 111.252 26.6594C111.083 26.4875 110.996 26.2707 110.996 26.0045C110.996 25.6634 111.142 25.3792 111.432 25.1513C111.718 24.9242 112.031 24.8095 112.37 24.8095C112.787 24.8095 113.187 24.9053 113.563 25.0959C113.944 25.285 114.28 25.512 114.579 25.7775L116.369 23.4761C115.772 22.9813 115.032 22.6182 114.147 22.38C113.26 22.143 112.43 22.0251 111.653 22.0251C110.958 22.0251 110.326 22.1396 109.756 22.3653C109.193 22.5939 108.704 22.9058 108.296 23.3044C107.887 23.7028 107.569 24.1814 107.341 24.7402C107.111 25.3008 106.998 25.9001 106.998 26.5441C106.998 27.1125 107.083 27.5923 107.251 27.9798C107.421 28.3703 107.639 28.7049 107.907 28.9891C108.175 29.2746 108.489 29.5103 108.848 29.7009C109.208 29.8909 109.563 30.0604 109.923 30.2119C110.281 30.3642 110.64 30.4983 110.996 30.623C111.355 30.7467 111.67 30.8848 111.938 31.0365C112.206 31.1877 112.424 31.3634 112.594 31.5624C112.762 31.7623 112.846 32.0127 112.846 32.3137C112.846 32.7686 112.684 33.1368 112.355 33.4101C112.027 33.6835 111.635 33.8225 111.176 33.8225C110.559 33.8225 109.987 33.6601 109.46 33.337C108.935 33.0164 108.47 32.6368 108.071 32.2027L106.162 34.5591C106.797 35.2995 107.569 35.8742 108.475 36.2809C109.379 36.6886 110.331 36.8926 111.326 36.8926C112.101 36.8926 112.832 36.7797 113.521 36.5499C114.204 36.3217 114.807 35.9967 115.325 35.5704C115.841 35.1447 116.255 34.6266 116.562 34.0205C116.872 33.4132 117.026 32.7213 117.026 31.9458C117.026 31.3397 116.941 30.8186 116.771 30.3807C116.604 29.9463 116.375 29.5728 116.086 29.2606Z" fill="#191B2A"/>
<path d="M120.875 16.041C120.237 16.041 119.701 16.2559 119.265 16.6788C118.83 17.102 118.611 17.6092 118.611 18.1988C118.611 18.8039 118.83 19.3133 119.265 19.7266C119.701 20.1429 120.237 20.35 120.875 20.35C121.509 20.35 122.046 20.1429 122.481 19.7266C122.918 19.3133 123.137 18.8039 123.137 18.1988C123.137 17.6092 122.918 17.102 122.481 16.6788C122.046 16.2559 121.509 16.041 120.875 16.041Z" fill="#191B2A"/>
<path d="M122.963 22.4814H118.784V36.4353H122.963V22.4814Z" fill="#191B2A"/>
<path d="M144.743 22.4813L141.128 29.9954L137.49 22.4813H132.545H131.951H131.683V16.4072H127.503V22.4813H124.673V25.8338H127.503V36.4351H131.683V25.8338H134.882L138.982 34.3041L135.477 41.223H140.073L149.402 22.4813H144.743Z" fill="#191B2A"/>
</svg>

After

Width:  |  Height:  |  Size: 6.4 KiB

BIN
assets/scoutam_gateway.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 66 KiB

BIN
assets/twitter.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 448 B

BIN
assets/workflow.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 69 KiB

View File

@@ -1,189 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"context"
"encoding/json"
"errors"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
)
func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
if opts.IsRoot {
return nil
}
if opts.Acc.Role == RoleAdmin {
return nil
}
// Verify destination bucket access
if err := VerifyAccess(ctx, be, opts); err != nil {
return err
}
// Verify source bucket access
srcBucket, srcObject, found := strings.Cut(copySource, "/")
if !found {
return s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)
}
// Get source bucket ACL
srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
if err != nil {
return err
}
var srcBucketAcl ACL
if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
return err
}
if err := VerifyAccess(ctx, be, AccessOptions{
Acl: srcBucketAcl,
AclPermission: PermissionRead,
IsRoot: opts.IsRoot,
Acc: opts.Acc,
Bucket: srcBucket,
Object: srcObject,
Action: GetObjectAction,
}); err != nil {
return err
}
return nil
}
type AccessOptions struct {
Acl ACL
AclPermission Permission
IsRoot bool
Acc Account
Bucket string
Object string
Action Action
Readonly bool
IsPublicRequest bool
}
func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
// Skip the access check for public bucket requests
if opts.IsPublicRequest {
return nil
}
if opts.Readonly {
if opts.AclPermission == PermissionWrite || opts.AclPermission == PermissionWriteAcp {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
}
if opts.IsRoot {
return nil
}
if opts.Acc.Role == RoleAdmin {
return nil
}
policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
if policyErr != nil {
if !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
return policyErr
}
} else {
return VerifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action)
}
if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
return err
}
return nil
}
// Detects if the action is policy related
// e.g.
// 'GetBucketPolicy', 'PutBucketPolicy'
func isPolicyAction(action Action) bool {
return action == GetBucketPolicyAction || action == PutBucketPolicyAction
}
// VerifyPublicAccess checks if the bucket is publically accessible by ACL or Policy
func VerifyPublicAccess(ctx context.Context, be backend.Backend, action Action, permission Permission, bucket, object string) error {
// ACL disabled
policy, err := be.GetBucketPolicy(ctx, bucket)
if err != nil && !errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
return err
}
if err == nil {
err = VerifyPublicBucketPolicy(policy, bucket, object, action)
if err == nil {
// if ACLs are disabled, and the bucket grants public access,
// policy actions should return 'MethodNotAllowed'
if isPolicyAction(action) {
return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return nil
}
}
// if the action is not in the ACL whitelist the access is denied
_, ok := publicACLAllowedActions[action]
if !ok {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
err = VerifyPublicBucketACL(ctx, be, bucket, action, permission)
if err != nil {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
return nil
}
func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
// Owner check
if acct.Access == acl.Owner {
return nil
}
// Root user has access over almost everything
if isRoot {
return nil
}
// Admin user case
if acct.Role == RoleAdmin {
return nil
}
// Return access denied in all other cases
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
type PublicACLAllowedActions map[Action]struct{}
var publicACLAllowedActions PublicACLAllowedActions = PublicACLAllowedActions{
ListBucketAction: struct{}{},
PutObjectAction: struct{}{},
ListBucketMultipartUploadsAction: struct{}{},
DeleteObjectAction: struct{}{},
ListBucketVersionsAction: struct{}{},
GetObjectAction: struct{}{},
GetObjectAttributesAction: struct{}{},
GetObjectAclAction: struct{}{},
}

View File

@@ -1,495 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"context"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
)
type ACL struct {
Owner string
Grantees []Grantee
}
// IsPublic specifies if the acl grants public read access
func (acl *ACL) IsPublic(permission Permission) bool {
for _, grt := range acl.Grantees {
if grt.Permission == permission && grt.Type == types.TypeGroup && grt.Access == "all-users" {
return true
}
}
return false
}
type Grantee struct {
Permission Permission
Access string
Type types.Type
}
type GetBucketAclOutput struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy"`
Owner *types.Owner
AccessControlList AccessControlList
}
type PutBucketAclInput struct {
Bucket *string
ACL types.BucketCannedACL
AccessControlPolicy *AccessControlPolicy
GrantFullControl *string
GrantRead *string
GrantReadACP *string
GrantWrite *string
GrantWriteACP *string
}
type AccessControlPolicy struct {
AccessControlList AccessControlList `xml:"AccessControlList"`
Owner *types.Owner
}
func (acp *AccessControlPolicy) Validate() error {
if !acp.AccessControlList.isValid() {
return s3err.GetAPIError(s3err.ErrMalformedACL)
}
// The Owner can't be nil
if acp.Owner == nil {
return s3err.GetAPIError(s3err.ErrMalformedACL)
}
// The Owner ID can't be empty
if acp.Owner.ID == nil || *acp.Owner.ID == "" {
return s3err.GetAPIError(s3err.ErrMalformedACL)
}
return nil
}
type AccessControlList struct {
Grants []Grant `xml:"Grant"`
}
// Validates the AccessControlList
func (acl *AccessControlList) isValid() bool {
for _, el := range acl.Grants {
if !el.isValid() {
return false
}
}
return true
}
type Permission string
const (
PermissionFullControl Permission = "FULL_CONTROL"
PermissionWrite Permission = "WRITE"
PermissionWriteAcp Permission = "WRITE_ACP"
PermissionRead Permission = "READ"
PermissionReadAcp Permission = "READ_ACP"
)
// Check if the permission is valid
func (p Permission) isValid() bool {
return p == PermissionFullControl ||
p == PermissionRead ||
p == PermissionReadAcp ||
p == PermissionWrite ||
p == PermissionWriteAcp
}
type Grant struct {
Grantee *Grt `xml:"Grantee"`
Permission Permission `xml:"Permission"`
}
// Checks if Grant is valid
func (g *Grant) isValid() bool {
return g.Permission.isValid() && g.Grantee.isValid()
}
type Grt struct {
XMLNS string `xml:"xmlns:xsi,attr"`
Type types.Type `xml:"xsi:type,attr"`
ID string `xml:"ID"`
}
// Custom Unmarshalling for Grt to parse xsi:type properly
func (g *Grt) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Iterate through the XML tokens to process the attributes
for _, attr := range start.Attr {
// Check if the attribute is xsi:type and belongs to the xsi namespace
if attr.Name.Space == "http://www.w3.org/2001/XMLSchema-instance" && attr.Name.Local == "type" {
g.Type = types.Type(attr.Value)
}
// Handle xmlns:xsi
if attr.Name.Local == "xmlns:xsi" {
g.XMLNS = attr.Value
}
}
// Decode the inner XML elements like ID
for {
t, err := d.Token()
if err != nil {
return err
}
switch se := t.(type) {
case xml.StartElement:
if se.Name.Local == "ID" {
if err := d.DecodeElement(&g.ID, &se); err != nil {
return err
}
}
case xml.EndElement:
if se.Name.Local == start.Name.Local {
return nil
}
}
}
}
// Validates Grt
func (g *Grt) isValid() bool {
// Validate the Type
// Only these 2 types are supported in the gateway
if g.Type != types.TypeCanonicalUser && g.Type != types.TypeGroup {
return false
}
// The ID prop shouldn't be empty
if g.ID == "" {
return false
}
return true
}
func ParseACL(data []byte) (ACL, error) {
if len(data) == 0 {
return ACL{}, nil
}
var acl ACL
if err := json.Unmarshal(data, &acl); err != nil {
return acl, fmt.Errorf("parse acl: %w", err)
}
return acl, nil
}
func ParseACLOutput(data []byte, owner string) (GetBucketAclOutput, error) {
grants := []Grant{}
if len(data) == 0 {
return GetBucketAclOutput{
Owner: &types.Owner{
ID: &owner,
},
AccessControlList: AccessControlList{
Grants: grants,
},
}, nil
}
var acl ACL
if err := json.Unmarshal(data, &acl); err != nil {
return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
}
for _, elem := range acl.Grantees {
acs := elem.Access
grants = append(grants, Grant{
Grantee: &Grt{
XMLNS: "http://www.w3.org/2001/XMLSchema-instance",
ID: acs,
Type: elem.Type,
},
Permission: elem.Permission,
})
}
return GetBucketAclOutput{
Owner: &types.Owner{
ID: &acl.Owner,
},
AccessControlList: AccessControlList{
Grants: grants,
},
}, nil
}
func UpdateACL(input *PutBucketAclInput, acl ACL, iam IAMService, isAdmin bool) ([]byte, error) {
if input == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
defaultGrantees := []Grantee{
{
Permission: PermissionFullControl,
Access: acl.Owner,
Type: types.TypeCanonicalUser,
},
}
// if the ACL is specified, set the ACL, else replace the grantees
if input.ACL != "" {
switch input.ACL {
case types.BucketCannedACLPublicRead:
defaultGrantees = append(defaultGrantees, Grantee{
Permission: PermissionRead,
Access: "all-users",
Type: types.TypeGroup,
})
case types.BucketCannedACLPublicReadWrite:
defaultGrantees = append(defaultGrantees, []Grantee{
{
Permission: PermissionRead,
Access: "all-users",
Type: types.TypeGroup,
},
{
Permission: PermissionWrite,
Access: "all-users",
Type: types.TypeGroup,
},
}...)
}
} else {
accs := []string{}
if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{}
if input.GrantFullControl != nil && *input.GrantFullControl != "" {
fullControlList = splitUnique(*input.GrantFullControl, ",")
for _, str := range fullControlList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: PermissionFullControl,
Type: types.TypeCanonicalUser,
})
}
}
if input.GrantRead != nil && *input.GrantRead != "" {
readList = splitUnique(*input.GrantRead, ",")
for _, str := range readList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: PermissionRead,
Type: types.TypeCanonicalUser,
})
}
}
if input.GrantReadACP != nil && *input.GrantReadACP != "" {
readACPList = splitUnique(*input.GrantReadACP, ",")
for _, str := range readACPList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: PermissionReadAcp,
Type: types.TypeCanonicalUser,
})
}
}
if input.GrantWrite != nil && *input.GrantWrite != "" {
writeList = splitUnique(*input.GrantWrite, ",")
for _, str := range writeList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: PermissionWrite,
Type: types.TypeCanonicalUser,
})
}
}
if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
writeACPList = splitUnique(*input.GrantWriteACP, ",")
for _, str := range writeACPList {
defaultGrantees = append(defaultGrantees, Grantee{
Access: str,
Permission: PermissionWriteAcp,
Type: types.TypeCanonicalUser,
})
}
}
accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
} else {
cache := make(map[string]bool)
for _, grt := range input.AccessControlPolicy.AccessControlList.Grants {
if grt.Grantee == nil || grt.Grantee.ID == "" || grt.Permission == "" {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
access := grt.Grantee.ID
defaultGrantees = append(defaultGrantees, Grantee{
Access: access,
Permission: grt.Permission,
Type: types.TypeCanonicalUser,
})
if _, ok := cache[access]; !ok {
cache[access] = true
accs = append(accs, access)
}
}
}
// Check if the specified accounts exist
accList, err := CheckIfAccountsExist(accs, iam)
if err != nil {
return nil, err
}
if len(accList) > 0 {
return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
}
}
acl.Grantees = defaultGrantees
result, err := json.Marshal(acl)
if err != nil {
return nil, err
}
return result, nil
}
func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
result := []string{}
for _, acc := range accs {
_, err := iam.GetUserAccount(acc)
if err != nil {
if err == ErrNoSuchUser || err == s3err.GetAPIError(s3err.ErrAdminUserNotFound) {
result = append(result, acc)
continue
}
if errors.Is(err, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)) {
return nil, err
}
return nil, fmt.Errorf("check user account: %w", err)
}
}
return result, nil
}
func splitUnique(s, divider string) []string {
elements := strings.Split(s, divider)
uniqueElements := make(map[string]bool)
result := make([]string, 0, len(elements))
for _, element := range elements {
if _, ok := uniqueElements[element]; !ok {
result = append(result, element)
uniqueElements[element] = true
}
}
return result
}
func verifyACL(acl ACL, access string, permission Permission) error {
grantee := Grantee{
Access: access,
Permission: permission,
Type: types.TypeCanonicalUser,
}
granteeFullCtrl := Grantee{
Access: access,
Permission: PermissionFullControl,
Type: types.TypeCanonicalUser,
}
granteeAllUsers := Grantee{
Access: "all-users",
Permission: permission,
Type: types.TypeGroup,
}
isFound := false
for _, grt := range acl.Grantees {
if grt == grantee || grt == granteeFullCtrl || grt == granteeAllUsers {
isFound = true
break
}
}
if isFound {
return nil
}
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
// Verifies if the bucket acl grants public access
func VerifyPublicBucketACL(ctx context.Context, be backend.Backend, bucket string, action Action, permission Permission) error {
aclBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{
Bucket: &bucket,
})
if err != nil {
return err
}
acl, err := ParseACL(aclBytes)
if err != nil {
return err
}
if !acl.IsPublic(permission) {
return ErrAccessDenied
}
return nil
}
// UpdateBucketACLOwner sets default ACL with new owner and removes
// any previous bucket policy that was in place
func UpdateBucketACLOwner(ctx context.Context, be backend.Backend, bucket, newOwner string) error {
acl := ACL{
Owner: newOwner,
Grantees: []Grantee{
{
Permission: PermissionFullControl,
Access: newOwner,
Type: types.TypeCanonicalUser,
},
},
}
result, err := json.Marshal(acl)
if err != nil {
return fmt.Errorf("marshal ACL: %w", err)
}
err = be.PutBucketAcl(ctx, bucket, result)
if err != nil {
return err
}
return be.DeleteBucketPolicy(ctx, bucket)
}

View File

@@ -1,338 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"encoding/xml"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
)
// headerRegex is the regexp to validate http header names
var headerRegex = regexp.MustCompile(`^[!#$%&'*+\-.^_` + "`" + `|~0-9A-Za-z]+$`)
type CORSHeader string
type CORSHTTPMethod string
// IsValid validates the CORS http header
// the rules are based on http RFC
// https://datatracker.ietf.org/doc/html/rfc7230#section-3.2
//
// Empty values are considered as valid
func (ch CORSHeader) IsValid() bool {
return ch == "" || headerRegex.MatchString(ch.String())
}
// String converts the header value to 'string'
func (ch CORSHeader) String() string {
return string(ch)
}
// ToLower converts the header to lower case
func (ch CORSHeader) ToLower() string {
return strings.ToLower(string(ch))
}
// IsValid validates the cors http request method:
// the methods are case sensitive
func (cm CORSHTTPMethod) IsValid() bool {
return cm.IsEmpty() || cm == http.MethodGet || cm == http.MethodHead || cm == http.MethodPut ||
cm == http.MethodPost || cm == http.MethodDelete
}
// IsEmpty checks if the cors method is an empty string
func (cm CORSHTTPMethod) IsEmpty() bool {
return cm == ""
}
// String converts the method value to 'string'
func (cm CORSHTTPMethod) String() string {
return string(cm)
}
type CORSConfiguration struct {
Rules []CORSRule `xml:"CORSRule"`
}
// Validate validates the cors configuration rules
func (cc *CORSConfiguration) Validate() error {
if cc == nil || cc.Rules == nil {
debuglogger.Logf("invalid CORS configuration")
return s3err.GetAPIError(s3err.ErrMalformedXML)
}
if len(cc.Rules) == 0 {
debuglogger.Logf("empty CORS config rules")
return s3err.GetAPIError(s3err.ErrMalformedXML)
}
// validate each CORS rule
for _, rule := range cc.Rules {
if err := rule.Validate(); err != nil {
return err
}
}
return nil
}
type CORSAllowanceConfig struct {
Origin string
Methods string
ExposedHeaders string
AllowCredentials string
AllowHeaders string
MaxAge *int32
}
// IsAllowed walks through the CORS rules and finds the first one allowing access.
// If no rule grants access, returns 'AccessForbidden'
func (cc *CORSConfiguration) IsAllowed(origin string, method CORSHTTPMethod, headers []CORSHeader) (*CORSAllowanceConfig, error) {
// if method is empty, anyways cors is forbidden
// skip, without going through the rules
if method.IsEmpty() {
debuglogger.Logf("empty Access-Control-Request-Method")
return nil, s3err.GetAPIError(s3err.ErrCORSForbidden)
}
for _, rule := range cc.Rules {
// find the first rule granting access
if isAllowed, wilcardOrigin := rule.Match(origin, method, headers); isAllowed {
o := origin
allowCredentials := "true"
if wilcardOrigin {
o = "*"
allowCredentials = "false"
}
return &CORSAllowanceConfig{
Origin: o,
AllowCredentials: allowCredentials,
Methods: rule.GetAllowedMethods(),
ExposedHeaders: rule.GetExposeHeaders(),
AllowHeaders: buildAllowedHeaders(headers),
MaxAge: rule.MaxAgeSeconds,
}, nil
}
}
// if no matching rule is found, return AccessForbidden
return nil, s3err.GetAPIError(s3err.ErrCORSForbidden)
}
type CORSRule struct {
AllowedMethods []CORSHTTPMethod `xml:"AllowedMethod"`
AllowedHeaders []CORSHeader `xml:"AllowedHeader"`
ExposeHeaders []CORSHeader `xml:"ExposeHeader"`
AllowedOrigins []string `xml:"AllowedOrigin"`
ID *string
MaxAgeSeconds *int32
}
// Validate validates and returns error if CORS configuration has invalid rule
func (cr *CORSRule) Validate() error {
// validate CORS allowed headers
for _, header := range cr.AllowedHeaders {
if !header.IsValid() {
debuglogger.Logf("invalid CORS allowed header: %s", header)
return s3err.GetInvalidCORSHeaderErr(header.String())
}
}
// validate CORS allowed methods
for _, method := range cr.AllowedMethods {
if !method.IsValid() {
debuglogger.Logf("invalid CORS allowed method: %s", method)
return s3err.GetUnsopportedCORSMethodErr(method.String())
}
}
// validate CORS expose headers
for _, header := range cr.ExposeHeaders {
if !header.IsValid() {
debuglogger.Logf("invalid CORS exposed header: %s", header)
return s3err.GetInvalidCORSHeaderErr(header.String())
}
}
return nil
}
// Match matches the provided origin, method and headers with the
// CORS configuration rule
// if the matching origin is "*", it returns true as the first argument
func (cr *CORSRule) Match(origin string, method CORSHTTPMethod, headers []CORSHeader) (bool, bool) {
wildcardOrigin := false
originFound := false
// check if the provided origin exists in CORS AllowedOrigins
for _, or := range cr.AllowedOrigins {
if wildcardMatch(or, origin) {
originFound = true
if or == "*" {
// mark wildcardOrigin as true, if "*" is found in AllowedOrigins
wildcardOrigin = true
}
break
}
}
if !originFound {
return false, false
}
// cache the CORS AllowedMethods in a map
allowedMethods := cacheCORSMethods(cr.AllowedMethods)
// check if the provided method exists in CORS AllowedMethods
if _, ok := allowedMethods[method]; !ok {
return false, false
}
// check is CORS rule allowed headers match
// with the requested allowed headers
for _, reqHeader := range headers {
match := false
for _, header := range cr.AllowedHeaders {
if wildcardMatch(header.ToLower(), reqHeader.ToLower()) {
match = true
break
}
}
if !match {
return false, false
}
}
return true, wildcardOrigin
}
// GetExposeHeaders returns comma separated CORS expose headers
func (cr *CORSRule) GetExposeHeaders() string {
var result strings.Builder
for i, h := range cr.ExposeHeaders {
if i > 0 {
result.WriteString(", ")
}
result.WriteString(h.String())
}
return result.String()
}
// buildAllowedHeaders builds a comma separated string from []CORSHeader
func buildAllowedHeaders(headers []CORSHeader) string {
var result strings.Builder
for i, h := range headers {
if i > 0 {
result.WriteString(", ")
}
result.WriteString(h.ToLower())
}
return result.String()
}
// GetAllowedMethods returns comma separated CORS allowed methods
func (cr *CORSRule) GetAllowedMethods() string {
var result strings.Builder
for i, m := range cr.AllowedMethods {
if i > 0 {
result.WriteString(", ")
}
result.WriteString(m.String())
}
return result.String()
}
// ParseCORSOutput parses raw bytes to 'CORSConfiguration'
func ParseCORSOutput(data []byte) (*CORSConfiguration, error) {
var config CORSConfiguration
err := xml.Unmarshal(data, &config)
if err != nil {
debuglogger.Logf("unmarshal cors output: %v", err)
return nil, fmt.Errorf("failed to parse cors config: %w", err)
}
return &config, nil
}
func cacheCORSMethods(input []CORSHTTPMethod) map[CORSHTTPMethod]struct{} {
result := make(map[CORSHTTPMethod]struct{}, len(input))
for _, el := range input {
result[el] = struct{}{}
}
return result
}
// ParseCORSHeaders parses/validates Access-Control-Request-Headers
// and returns []CORSHeaders
func ParseCORSHeaders(headers string) ([]CORSHeader, error) {
result := []CORSHeader{}
if headers == "" {
return result, nil
}
headersSplitted := strings.Split(headers, ",")
for _, h := range headersSplitted {
corsHeader := CORSHeader(strings.TrimSpace(h))
if corsHeader == "" || !corsHeader.IsValid() {
debuglogger.Logf("invalid access control header: %s", h)
return nil, s3err.GetInvalidCORSRequestHeaderErr(h)
}
result = append(result, corsHeader)
}
return result, nil
}
func wildcardMatch(pattern, input string) bool {
pIdx, sIdx := 0, 0
starIdx, matchIdx := -1, 0
for sIdx < len(input) {
if pIdx < len(pattern) && pattern[pIdx] == input[sIdx] {
// exact match of current char
sIdx++
pIdx++
} else if pIdx < len(pattern) && pattern[pIdx] == '*' {
// remember star position
starIdx = pIdx
matchIdx = sIdx
pIdx++
} else if starIdx != -1 {
// backtrack: try to match more characters with '*'
pIdx = starIdx + 1
matchIdx++
sIdx = matchIdx
} else {
return false
}
}
// skip trailing stars
for pIdx < len(pattern) && pattern[pIdx] == '*' {
pIdx++
}
return pIdx == len(pattern)
}

View File

@@ -1,736 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"net/http"
"testing"
"github.com/stretchr/testify/assert"
"github.com/versity/versitygw/s3err"
)
func TestCORSHeader_IsValid(t *testing.T) {
tests := []struct {
name string
header CORSHeader
want bool
}{
{"empty", "", true},
{"valid", "X-Custom-Header", true},
{"invalid_1", "Invalid Header", false},
{"invalid_2", "invalid/header", false},
{"invalid_3", "Invalid\tHeader", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := tt.header.IsValid(); got != tt.want {
t.Errorf("IsValid() = %v, want %v", got, tt.want)
}
})
}
}
func TestCORSHTTPMethod_IsValid(t *testing.T) {
tests := []struct {
name string
method CORSHTTPMethod
want bool
}{
{"empty valid", "", true},
{"GET valid", http.MethodGet, true},
{"HEAD valid", http.MethodHead, true},
{"PUT valid", http.MethodPut, true},
{"POST valid", http.MethodPost, true},
{"DELETE valid", http.MethodDelete, true},
{"get valid", "get", false},
{"put valid", "put", false},
{"post valid", "post", false},
{"head valid", "head", false},
{"invalid", "FOO", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := tt.method.IsValid(); got != tt.want {
t.Errorf("IsValid() = %v, want %v", got, tt.want)
}
})
}
}
func TestCORSHeader_ToLower(t *testing.T) {
tests := []struct {
name string
header CORSHeader
want string
}{
{
name: "already lowercase",
header: CORSHeader("content-type"),
want: "content-type",
},
{
name: "mixed case",
header: CORSHeader("X-CuStOm-HeAdEr"),
want: "x-custom-header",
},
{
name: "uppercase",
header: CORSHeader("AUTHORIZATION"),
want: "authorization",
},
{
name: "empty string",
header: CORSHeader(""),
want: "",
},
{
name: "numeric and symbols",
header: CORSHeader("X-123-HEADER"),
want: "x-123-header",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.header.ToLower()
assert.Equal(t, tt.want, got)
})
}
}
func TestCORSHTTPMethod_IsEmpty(t *testing.T) {
tests := []struct {
name string
method CORSHTTPMethod
want bool
}{
{
name: "empty string is empty",
method: CORSHTTPMethod(""),
want: true,
},
{
name: "GET method is not empty",
method: CORSHTTPMethod("GET"),
want: false,
},
{
name: "random string is not empty",
method: CORSHTTPMethod("FOO"),
want: false,
},
{
name: "lowercase get is not empty (case sensitive)",
method: CORSHTTPMethod("get"),
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.method.IsEmpty()
assert.Equal(t, tt.want, got)
})
}
}
func TestCORSConfiguration_Validate(t *testing.T) {
tests := []struct {
name string
cfg *CORSConfiguration
want error
}{
{"nil config", nil, s3err.GetAPIError(s3err.ErrMalformedXML)},
{"nil rules", &CORSConfiguration{}, s3err.GetAPIError(s3err.ErrMalformedXML)},
{"empty rules", &CORSConfiguration{Rules: []CORSRule{}}, s3err.GetAPIError(s3err.ErrMalformedXML)},
{"invalid rule", &CORSConfiguration{Rules: []CORSRule{{AllowedHeaders: []CORSHeader{"Invalid Header"}}}}, s3err.GetInvalidCORSHeaderErr("Invalid Header")},
{"valid rule", &CORSConfiguration{Rules: []CORSRule{{
AllowedOrigins: []string{"origin"},
AllowedHeaders: []CORSHeader{"X-Test"},
AllowedMethods: []CORSHTTPMethod{http.MethodGet},
ExposeHeaders: []CORSHeader{"X-Expose"},
}}}, nil},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.cfg.Validate()
assert.EqualValues(t, tt.want, err)
})
}
}
func TestCORSConfiguration_IsAllowed(t *testing.T) {
type input struct {
cfg *CORSConfiguration
origin string
method CORSHTTPMethod
headers []CORSHeader
}
type output struct {
result *CORSAllowanceConfig
err error
}
tests := []struct {
name string
input input
output output
}{
{
name: "allowed exact origin",
input: input{
cfg: &CORSConfiguration{Rules: []CORSRule{{
AllowedOrigins: []string{"http://allowed.com"},
AllowedMethods: []CORSHTTPMethod{http.MethodGet},
AllowedHeaders: []CORSHeader{"X-Test"},
}}},
origin: "http://allowed.com",
method: http.MethodGet,
headers: []CORSHeader{"X-Test"},
},
output: output{
result: &CORSAllowanceConfig{
Origin: "http://allowed.com",
AllowCredentials: "true",
Methods: http.MethodGet,
AllowHeaders: "x-test",
ExposedHeaders: "",
MaxAge: nil,
},
err: nil,
},
},
{
name: "allowed wildcard origin",
input: input{
cfg: &CORSConfiguration{Rules: []CORSRule{{
AllowedOrigins: []string{"*"},
AllowedMethods: []CORSHTTPMethod{http.MethodGet},
AllowedHeaders: []CORSHeader{"X-Test"},
}}},
origin: "anything",
method: http.MethodGet,
headers: []CORSHeader{"X-Test"},
},
output: output{
result: &CORSAllowanceConfig{
Origin: "*",
AllowCredentials: "false",
AllowHeaders: "x-test",
Methods: http.MethodGet,
ExposedHeaders: "",
MaxAge: nil,
},
err: nil,
},
},
{
name: "forbidden no matching origin",
input: input{
cfg: &CORSConfiguration{Rules: []CORSRule{{
AllowedOrigins: []string{"http://nope.com"},
}}},
origin: "http://not-allowed.com",
method: http.MethodGet,
},
output: output{
result: nil,
err: s3err.GetAPIError(s3err.ErrCORSForbidden),
},
},
{
name: "forbidden method not allowed",
input: input{
cfg: &CORSConfiguration{Rules: []CORSRule{{
AllowedOrigins: []string{"http://allowed.com"},
AllowedMethods: []CORSHTTPMethod{http.MethodPost},
AllowedHeaders: []CORSHeader{"X-Test"},
}}},
origin: "http://allowed.com",
method: http.MethodGet,
headers: []CORSHeader{"X-Test"},
},
output: output{
result: nil,
err: s3err.GetAPIError(s3err.ErrCORSForbidden),
},
},
{
name: "forbidden header not allowed",
input: input{
cfg: &CORSConfiguration{Rules: []CORSRule{{
AllowedOrigins: []string{"http://allowed.com"},
AllowedMethods: []CORSHTTPMethod{http.MethodGet},
AllowedHeaders: []CORSHeader{"X-Test"},
}}},
origin: "http://allowed.com",
method: http.MethodGet,
headers: []CORSHeader{"X-Nope"},
},
output: output{
result: nil,
err: s3err.GetAPIError(s3err.ErrCORSForbidden),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.input.cfg.IsAllowed(tt.input.origin, tt.input.method, tt.input.headers)
assert.EqualValues(t, tt.output.err, err)
assert.EqualValues(t, tt.output.result, got)
})
}
}
func TestCORSRule_Validate(t *testing.T) {
tests := []struct {
name string
rule CORSRule
want error
}{
{
name: "valid rule",
rule: CORSRule{
AllowedOrigins: []string{"http://allowed.com"},
AllowedMethods: []CORSHTTPMethod{http.MethodGet},
AllowedHeaders: []CORSHeader{"X-Test"},
},
want: nil,
},
{
name: "invalid allowed methods",
rule: CORSRule{
AllowedOrigins: []string{"http://allowed.com"},
AllowedMethods: []CORSHTTPMethod{"invalid_method"},
AllowedHeaders: []CORSHeader{"X-Test"},
},
want: s3err.GetUnsopportedCORSMethodErr("invalid_method"),
},
{
name: "invalid allowed header",
rule: CORSRule{
AllowedOrigins: []string{"http://allowed.com"},
AllowedMethods: []CORSHTTPMethod{http.MethodGet},
AllowedHeaders: []CORSHeader{"Invalid Header"},
},
want: s3err.GetInvalidCORSHeaderErr("Invalid Header"),
},
{
name: "invalid allowed header",
rule: CORSRule{
AllowedOrigins: []string{"http://allowed.com"},
AllowedMethods: []CORSHTTPMethod{http.MethodGet},
AllowedHeaders: []CORSHeader{"Content-Length"},
ExposeHeaders: []CORSHeader{"Content-Encoding", "invalid header"},
},
want: s3err.GetInvalidCORSHeaderErr("invalid header"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.rule.Validate()
assert.EqualValues(t, tt.want, err)
})
}
}
func TestCORSRule_Match(t *testing.T) {
type input struct {
rule CORSRule
origin string
method CORSHTTPMethod
headers []CORSHeader
}
type output struct {
isAllowed bool
isWildcard bool
}
tests := []struct {
name string
input input
output output
}{
{
name: "exact origin and method match",
input: input{
rule: CORSRule{
AllowedOrigins: []string{"http://allowed.com"},
AllowedMethods: []CORSHTTPMethod{http.MethodGet},
AllowedHeaders: []CORSHeader{"X-Test"},
},
origin: "http://allowed.com",
method: http.MethodGet,
headers: []CORSHeader{"X-Test"},
},
output: output{isAllowed: true, isWildcard: false},
},
{
name: "wildcard origin match",
input: input{
rule: CORSRule{
AllowedOrigins: []string{"*"},
AllowedMethods: []CORSHTTPMethod{http.MethodPost},
AllowedHeaders: []CORSHeader{"X-Test"},
},
origin: "http://random.com",
method: http.MethodPost,
headers: []CORSHeader{"X-Test"},
},
output: output{isAllowed: true, isWildcard: true},
},
{
name: "wildcard containing origin match",
input: input{
rule: CORSRule{
AllowedOrigins: []string{"http://random*"},
AllowedMethods: []CORSHTTPMethod{http.MethodPost},
AllowedHeaders: []CORSHeader{"X-Test"},
},
origin: "http://random.com",
method: http.MethodPost,
headers: []CORSHeader{"X-Test"},
},
output: output{isAllowed: true, isWildcard: false},
},
{
name: "wildcard allowed headers match",
input: input{
rule: CORSRule{
AllowedOrigins: []string{"http://something.com"},
AllowedMethods: []CORSHTTPMethod{http.MethodPost},
AllowedHeaders: []CORSHeader{"X-*"},
},
origin: "http://something.com",
method: http.MethodPost,
headers: []CORSHeader{"X-Test", "X-Something", "X-Anyting"},
},
output: output{isAllowed: true, isWildcard: false},
},
{
name: "origin mismatch",
input: input{
rule: CORSRule{
AllowedOrigins: []string{"http://allowed.com"},
AllowedMethods: []CORSHTTPMethod{http.MethodGet},
AllowedHeaders: []CORSHeader{"X-Test"},
},
origin: "http://notallowed.com",
method: http.MethodGet,
headers: []CORSHeader{"X-Test"},
},
output: output{isAllowed: false, isWildcard: false},
},
{
name: "method mismatch",
input: input{
rule: CORSRule{
AllowedOrigins: []string{"http://allowed.com"},
AllowedMethods: []CORSHTTPMethod{http.MethodPost},
AllowedHeaders: []CORSHeader{"X-Test"},
},
origin: "http://allowed.com",
method: http.MethodGet,
headers: []CORSHeader{"X-Test"},
},
output: output{isAllowed: false, isWildcard: false},
},
{
name: "header mismatch",
input: input{
rule: CORSRule{
AllowedOrigins: []string{"http://allowed.com"},
AllowedMethods: []CORSHTTPMethod{http.MethodGet},
AllowedHeaders: []CORSHeader{"X-Test"},
},
origin: "http://allowed.com",
method: http.MethodGet,
headers: []CORSHeader{"X-Other"},
},
output: output{isAllowed: false, isWildcard: false},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
isAllowed, wild := tt.input.rule.Match(tt.input.origin, tt.input.method, tt.input.headers)
assert.Equal(t, tt.output.isAllowed, isAllowed)
assert.Equal(t, tt.output.isWildcard, wild)
})
}
}
func TestGetExposeHeaders(t *testing.T) {
tests := []struct {
name string
rule CORSRule
want string
}{
{"multiple headers", CORSRule{ExposeHeaders: []CORSHeader{"Content-Length", "Content-Type", "Content-Encoding"}}, "Content-Length, Content-Type, Content-Encoding"},
{"single header", CORSRule{ExposeHeaders: []CORSHeader{"Authorization"}}, "Authorization"},
{"no headers", CORSRule{}, ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.rule.GetExposeHeaders()
assert.Equal(t, tt.want, got)
})
}
}
func TestBuildAllowedHeaders(t *testing.T) {
tests := []struct {
name string
headers []CORSHeader
want string
}{
{
name: "empty slice returns empty string",
headers: []CORSHeader{},
want: "",
},
{
name: "single header lowercase",
headers: []CORSHeader{"Content-Type"},
want: "content-type",
},
{
name: "multiple headers lowercased with commas",
headers: []CORSHeader{"Content-Type", "X-Custom-Header", "Authorization"},
want: "content-type, x-custom-header, authorization",
},
{
name: "already lowercase header",
headers: []CORSHeader{"accept"},
want: "accept",
},
{
name: "mixed case headers",
headers: []CORSHeader{"ACCEPT", "x-Powered-By"},
want: "accept, x-powered-by",
},
{
name: "empty header value",
headers: []CORSHeader{""},
want: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := buildAllowedHeaders(tt.headers)
assert.Equal(t, tt.want, got)
})
}
}
func TestGetAllowedMethods(t *testing.T) {
tests := []struct {
name string
rule CORSRule
want string
}{
{"multiple methods", CORSRule{AllowedMethods: []CORSHTTPMethod{http.MethodGet, http.MethodPost, http.MethodPut}}, "GET, POST, PUT"},
{"single method", CORSRule{AllowedMethods: []CORSHTTPMethod{http.MethodGet}}, "GET"},
{"no methods", CORSRule{}, ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.rule.GetAllowedMethods()
assert.Equal(t, tt.want, got)
})
}
}
func TestParseCORSOutput(t *testing.T) {
tests := []struct {
name string
data string
want bool
}{
{"valid", `<CORSConfiguration><CORSRule></CORSRule></CORSConfiguration>`, true},
{"invalid xml", `<CORSConfiguration><CORSRule>`, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg, err := ParseCORSOutput([]byte(tt.data))
if (err == nil) != tt.want {
t.Errorf("ParseCORSOutput() err = %v, want success=%v", err, tt.want)
}
if tt.want && cfg == nil {
t.Errorf("Expected non-nil config")
}
})
}
}
func TestCacheCORSProps(t *testing.T) {
tests := []struct {
name string
in []CORSHTTPMethod
want map[string]struct{}
}{
{
name: "empty CORSHTTPMethod slice",
in: []CORSHTTPMethod{},
want: map[string]struct{}{},
},
{
name: "single CORSHTTPMethod",
in: []CORSHTTPMethod{http.MethodGet},
want: map[string]struct{}{http.MethodGet: {}},
},
{
name: "multiple CORSHTTPMethods",
in: []CORSHTTPMethod{http.MethodGet, http.MethodPost, http.MethodPut},
want: map[string]struct{}{
http.MethodGet: {},
http.MethodPost: {},
http.MethodPut: {},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := cacheCORSMethods(tt.in)
assert.Equal(t, len(tt.want), len(got))
for key := range tt.want {
_, ok := got[CORSHTTPMethod(key)]
assert.True(t, ok)
}
})
}
}
func TestParseCORSHeaders(t *testing.T) {
tests := []struct {
name string
in string
want []CORSHeader
err error
}{
{
name: "empty string",
in: "",
want: []CORSHeader{},
err: nil,
},
{
name: "single valid header",
in: "X-Test",
want: []CORSHeader{"X-Test"},
err: nil,
},
{
name: "multiple valid headers with spaces",
in: "X-Test, Content-Type, Authorization",
want: []CORSHeader{"X-Test", "Content-Type", "Authorization"},
err: nil,
},
{
name: "header with leading/trailing spaces",
in: " X-Test ",
want: []CORSHeader{"X-Test"},
err: nil,
},
{
name: "contains invalid header",
in: "X-Test, Invalid Header, Content-Type",
want: nil,
err: s3err.GetInvalidCORSRequestHeaderErr(" Invalid Header"),
},
{
name: "only invalid header",
in: "Invalid Header",
want: nil,
err: s3err.GetInvalidCORSRequestHeaderErr("Invalid Header"),
},
{
name: "multiple commas in a row",
in: "X-Test,,Content-Type",
want: nil,
err: s3err.GetInvalidCORSRequestHeaderErr(""),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ParseCORSHeaders(tt.in)
assert.EqualValues(t, tt.err, err)
assert.Equal(t, tt.want, got)
})
}
}
func TestWildcardMatch(t *testing.T) {
tests := []struct {
name string
pattern string
input string
want bool
}{
// Exact match, no wildcards
{"exact match", "hello", "hello", true},
{"exact mismatch", "hello", "hell", false},
// Single '*' matching zero chars
{"star matches zero chars", "he*lo", "helo", true},
// Single '*' matching multiple chars
{"star matches multiple chars", "he*o", "heyyyyyo", true},
// '*' at start
{"star at start", "*world", "hello world", true},
// '*' at end
{"star at end", "hello*", "hello there", true},
// '*' matches whole string
{"only star", "*", "anything", true},
{"only star empty", "*", "", true},
// Multiple '*'s
{"multiple stars", "a*b*c", "axxxbzzzzyc", true},
{"multiple stars no match", "a*b*c", "axxxbzzzzy", false},
// Backtracking needed
{"backtracking required", "a*b*c", "ab123c", true},
// No match with star present
{"star but mismatch", "he*world", "hey there", false},
// Trailing stars in pattern
{"trailing stars match", "abc**", "abc", true},
{"trailing stars match longer", "abc**", "abccc", true},
// Empty pattern cases
{"empty pattern and empty input", "", "", true},
{"empty pattern non-empty input", "", "a", false},
{"only stars pattern with empty input", "***", "", true},
// Pattern longer than input
{"pattern longer no star", "abcd", "abc", false},
// Input longer but no star
{"input longer no star", "abc", "abcd", false},
// Complex interleaved match
{"complex interleaved", "*a*b*cd*", "xxaYYbZZcd123", true},
// Star match at the end after mismatch
{"mismatch then star match", "ab*xyz", "abzzzxyz", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := wildcardMatch(tt.pattern, tt.input)
assert.Equal(t, tt.want, got)
})
}
}

View File

@@ -1,290 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"github.com/versity/versitygw/s3err"
)
var ErrAccessDenied = errors.New("access denied")
type policyErr string
func (p policyErr) Error() string {
return string(p)
}
const (
policyErrResourceMismatch = policyErr("Action does not apply to any resource(s) in statement")
policyErrInvalidResource = policyErr("Policy has invalid resource")
policyErrInvalidPrincipal = policyErr("Invalid principal in policy")
policyErrInvalidAction = policyErr("Policy has invalid action")
policyErrInvalidPolicy = policyErr("This policy contains invalid Json")
policyErrInvalidFirstChar = policyErr("Policies must be valid JSON and the first byte must be '{'")
policyErrEmptyStatement = policyErr("Could not parse the policy: Statement is empty!")
policyErrMissingStatmentField = policyErr("Missing required field Statement")
)
type BucketPolicy struct {
Statement []BucketPolicyItem `json:"Statement"`
}
func (bp *BucketPolicy) UnmarshalJSON(data []byte) error {
var tmp struct {
Statement *[]BucketPolicyItem `json:"Statement"`
}
if err := json.Unmarshal(data, &tmp); err != nil {
return err
}
// If Statement is nil (not present in JSON), return an error
if tmp.Statement == nil {
return policyErrMissingStatmentField
}
// Assign the parsed value to the actual struct
bp.Statement = *tmp.Statement
return nil
}
func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
for _, statement := range bp.Statement {
err := statement.Validate(bucket, iam)
if err != nil {
return err
}
}
return nil
}
func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
var isAllowed bool
for _, statement := range bp.Statement {
if statement.findMatch(principal, action, resource) {
switch statement.Effect {
case BucketPolicyAccessTypeAllow:
isAllowed = true
case BucketPolicyAccessTypeDeny:
return false
}
}
}
return isAllowed
}
// IsPublicFor checks if the bucket policy statements contain
// an entity granting public access to the given resource and action
func (bp *BucketPolicy) isPublicFor(resource string, action Action) bool {
var isAllowed bool
for _, statement := range bp.Statement {
if statement.isPublicFor(resource, action) {
switch statement.Effect {
case BucketPolicyAccessTypeAllow:
isAllowed = true
case BucketPolicyAccessTypeDeny:
return false
}
}
}
return isAllowed
}
// IsPublic checks if one of bucket policy statments grant
// public access to ALL users
func (bp *BucketPolicy) IsPublic() bool {
for _, statement := range bp.Statement {
if statement.isPublic() {
return true
}
}
return false
}
type BucketPolicyItem struct {
Effect BucketPolicyAccessType `json:"Effect"`
Principals Principals `json:"Principal"`
Actions Actions `json:"Action"`
Resources Resources `json:"Resource"`
}
func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
if err := bpi.Effect.Validate(); err != nil {
return err
}
if err := bpi.Principals.Validate(iam); err != nil {
return err
}
if err := bpi.Resources.Validate(bucket); err != nil {
return err
}
containsObjectAction := bpi.Resources.ContainsObjectPattern()
containsBucketAction := bpi.Resources.ContainsBucketPattern()
for action := range bpi.Actions {
isObjectAction := action.IsObjectAction()
if isObjectAction == nil {
break
}
if *isObjectAction && !containsObjectAction {
return policyErrResourceMismatch
}
if !*isObjectAction && !containsBucketAction {
return policyErrResourceMismatch
}
}
return nil
}
func (bpi *BucketPolicyItem) findMatch(principal string, action Action, resource string) bool {
if bpi.Principals.Contains(principal) && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource) {
return true
}
return false
}
// isPublicFor checks if the bucket policy statemant grants public access
// for given resource and action
func (bpi *BucketPolicyItem) isPublicFor(resource string, action Action) bool {
return bpi.Principals.isPublic() && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource)
}
// isPublic checks if the statement grants public access
// to ALL users
func (bpi *BucketPolicyItem) isPublic() bool {
return bpi.Principals.isPublic()
}
func getMalformedPolicyError(err error) error {
return s3err.APIError{
Code: "MalformedPolicy",
Description: err.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
}
// ParsePolicyDocument parses raw bytes to 'BucketPolicy'
func ParsePolicyDocument(data []byte) (*BucketPolicy, error) {
var policy BucketPolicy
if err := json.Unmarshal(data, &policy); err != nil {
var pe policyErr
if errors.As(err, &pe) {
return nil, getMalformedPolicyError(err)
}
return nil, getMalformedPolicyError(policyErrInvalidPolicy)
}
return &policy, nil
}
func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) error {
if len(policyBin) == 0 || policyBin[0] != '{' {
return getMalformedPolicyError(policyErrInvalidFirstChar)
}
policy, err := ParsePolicyDocument(policyBin)
if err != nil {
return err
}
if len(policy.Statement) == 0 {
return getMalformedPolicyError(policyErrEmptyStatement)
}
if err := policy.Validate(bucket, iam); err != nil {
return getMalformedPolicyError(err)
}
return nil
}
func VerifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
var bucketPolicy BucketPolicy
if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
return fmt.Errorf("failed to parse the bucket policy: %w", err)
}
resource := bucket
if object != "" {
resource += "/" + object
}
if !bucketPolicy.isAllowed(access, action, resource) {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
return nil
}
// Checks if the bucket policy grants public access
func VerifyPublicBucketPolicy(policy []byte, bucket, object string, action Action) error {
var bucketPolicy BucketPolicy
if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
return err
}
resource := bucket
if object != "" {
resource += "/" + object
}
if !bucketPolicy.isPublicFor(resource, action) {
return ErrAccessDenied
}
return nil
}
// matchPattern checks if the input string matches the given pattern with wildcard(`*`) and any character(`?`).
// - `?` matches exactly one occurrence of any character.
// - `*` matches arbitrary many (including zero) occurrences of any character.
func matchPattern(pattern, input string) bool {
pIdx, sIdx := 0, 0
starIdx, matchIdx := -1, 0
for sIdx < len(input) {
if pIdx < len(pattern) && (pattern[pIdx] == '?' || pattern[pIdx] == input[sIdx]) {
sIdx++
pIdx++
} else if pIdx < len(pattern) && pattern[pIdx] == '*' {
starIdx = pIdx
matchIdx = sIdx
pIdx++
} else if starIdx != -1 {
pIdx = starIdx + 1
matchIdx++
sIdx = matchIdx
} else {
return false
}
}
for pIdx < len(pattern) && pattern[pIdx] == '*' {
pIdx++
}
return pIdx == len(pattern)
}

View File

@@ -1,313 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"encoding/json"
"strings"
)
type Action string
const (
GetBucketAclAction Action = "s3:GetBucketAcl"
CreateBucketAction Action = "s3:CreateBucket"
PutBucketAclAction Action = "s3:PutBucketAcl"
DeleteBucketAction Action = "s3:DeleteBucket"
PutBucketVersioningAction Action = "s3:PutBucketVersioning"
GetBucketVersioningAction Action = "s3:GetBucketVersioning"
PutBucketPolicyAction Action = "s3:PutBucketPolicy"
GetBucketPolicyAction Action = "s3:GetBucketPolicy"
DeleteBucketPolicyAction Action = "s3:DeleteBucketPolicy"
AbortMultipartUploadAction Action = "s3:AbortMultipartUpload"
ListMultipartUploadPartsAction Action = "s3:ListMultipartUploadParts"
ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
PutObjectAction Action = "s3:PutObject"
GetObjectAction Action = "s3:GetObject"
GetObjectVersionAction Action = "s3:GetObjectVersion"
DeleteObjectAction Action = "s3:DeleteObject"
GetObjectAclAction Action = "s3:GetObjectAcl"
GetObjectAttributesAction Action = "s3:GetObjectAttributes"
PutObjectAclAction Action = "s3:PutObjectAcl"
RestoreObjectAction Action = "s3:RestoreObject"
GetBucketTaggingAction Action = "s3:GetBucketTagging"
PutBucketTaggingAction Action = "s3:PutBucketTagging"
GetObjectTaggingAction Action = "s3:GetObjectTagging"
PutObjectTaggingAction Action = "s3:PutObjectTagging"
DeleteObjectTaggingAction Action = "s3:DeleteObjectTagging"
ListBucketVersionsAction Action = "s3:ListBucketVersions"
ListBucketAction Action = "s3:ListBucket"
GetBucketObjectLockConfigurationAction Action = "s3:GetBucketObjectLockConfiguration"
PutBucketObjectLockConfigurationAction Action = "s3:PutBucketObjectLockConfiguration"
GetObjectLegalHoldAction Action = "s3:GetObjectLegalHold"
PutObjectLegalHoldAction Action = "s3:PutObjectLegalHold"
GetObjectRetentionAction Action = "s3:GetObjectRetention"
PutObjectRetentionAction Action = "s3:PutObjectRetention"
BypassGovernanceRetentionAction Action = "s3:BypassGovernanceRetention"
PutBucketOwnershipControlsAction Action = "s3:PutBucketOwnershipControls"
GetBucketOwnershipControlsAction Action = "s3:GetBucketOwnershipControls"
PutBucketCorsAction Action = "s3:PutBucketCORS"
GetBucketCorsAction Action = "s3:GetBucketCORS"
PutAnalyticsConfigurationAction Action = "s3:PutAnalyticsConfiguration"
GetAnalyticsConfigurationAction Action = "s3:GetAnalyticsConfiguration"
PutEncryptionConfigurationAction Action = "s3:PutEncryptionConfiguration"
GetEncryptionConfigurationAction Action = "s3:GetEncryptionConfiguration"
PutIntelligentTieringConfigurationAction Action = "s3:PutIntelligentTieringConfiguration"
GetIntelligentTieringConfigurationAction Action = "s3:GetIntelligentTieringConfiguration"
PutInventoryConfigurationAction Action = "s3:PutInventoryConfiguration"
GetInventoryConfigurationAction Action = "s3:GetInventoryConfiguration"
PutLifecycleConfigurationAction Action = "s3:PutLifecycleConfiguration"
GetLifecycleConfigurationAction Action = "s3:GetLifecycleConfiguration"
PutBucketLoggingAction Action = "s3:PutBucketLogging"
GetBucketLoggingAction Action = "s3:GetBucketLogging"
PutBucketRequestPaymentAction Action = "s3:PutBucketRequestPayment"
GetBucketRequestPaymentAction Action = "s3:GetBucketRequestPayment"
PutMetricsConfigurationAction Action = "s3:PutMetricsConfiguration"
GetMetricsConfigurationAction Action = "s3:GetMetricsConfiguration"
PutReplicationConfigurationAction Action = "s3:PutReplicationConfiguration"
GetReplicationConfigurationAction Action = "s3:GetReplicationConfiguration"
PutBucketPublicAccessBlockAction Action = "s3:PutBucketPublicAccessBlock"
GetBucketPublicAccessBlockAction Action = "s3:GetBucketPublicAccessBlock"
PutBucketNotificationAction Action = "s3:PutBucketNotification"
GetBucketNotificationAction Action = "s3:GetBucketNotification"
PutAccelerateConfigurationAction Action = "s3:PutAccelerateConfiguration"
GetAccelerateConfigurationAction Action = "s3:GetAccelerateConfiguration"
PutBucketWebsiteAction Action = "s3:PutBucketWebsite"
GetBucketWebsiteAction Action = "s3:GetBucketWebsite"
GetBucketPolicyStatusAction Action = "s3:GetBucketPolicyStatus"
GetBucketLocationAction Action = "s3:GetBucketLocation"
AllActions Action = "s3:*"
)
var supportedActionList = map[Action]struct{}{
GetBucketAclAction: {},
CreateBucketAction: {},
PutBucketAclAction: {},
DeleteBucketAction: {},
PutBucketVersioningAction: {},
GetBucketVersioningAction: {},
PutBucketPolicyAction: {},
GetBucketPolicyAction: {},
DeleteBucketPolicyAction: {},
AbortMultipartUploadAction: {},
ListMultipartUploadPartsAction: {},
ListBucketMultipartUploadsAction: {},
PutObjectAction: {},
GetObjectAction: {},
GetObjectVersionAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
PutObjectAclAction: {},
RestoreObjectAction: {},
GetBucketTaggingAction: {},
PutBucketTaggingAction: {},
GetObjectTaggingAction: {},
PutObjectTaggingAction: {},
DeleteObjectTaggingAction: {},
ListBucketVersionsAction: {},
ListBucketAction: {},
GetBucketObjectLockConfigurationAction: {},
PutBucketObjectLockConfigurationAction: {},
GetObjectLegalHoldAction: {},
PutObjectLegalHoldAction: {},
GetObjectRetentionAction: {},
PutObjectRetentionAction: {},
BypassGovernanceRetentionAction: {},
PutBucketOwnershipControlsAction: {},
GetBucketOwnershipControlsAction: {},
PutBucketCorsAction: {},
GetBucketCorsAction: {},
PutAnalyticsConfigurationAction: {},
GetAnalyticsConfigurationAction: {},
PutEncryptionConfigurationAction: {},
GetEncryptionConfigurationAction: {},
PutIntelligentTieringConfigurationAction: {},
GetIntelligentTieringConfigurationAction: {},
PutInventoryConfigurationAction: {},
GetInventoryConfigurationAction: {},
PutLifecycleConfigurationAction: {},
GetLifecycleConfigurationAction: {},
PutBucketLoggingAction: {},
GetBucketLoggingAction: {},
PutBucketRequestPaymentAction: {},
GetBucketRequestPaymentAction: {},
PutMetricsConfigurationAction: {},
GetMetricsConfigurationAction: {},
PutReplicationConfigurationAction: {},
GetReplicationConfigurationAction: {},
PutBucketPublicAccessBlockAction: {},
GetBucketPublicAccessBlockAction: {},
PutBucketNotificationAction: {},
GetBucketNotificationAction: {},
PutAccelerateConfigurationAction: {},
GetAccelerateConfigurationAction: {},
PutBucketWebsiteAction: {},
GetBucketWebsiteAction: {},
GetBucketPolicyStatusAction: {},
GetBucketLocationAction: {},
AllActions: {},
}
var supportedObjectActionList = map[Action]struct{}{
AbortMultipartUploadAction: {},
ListMultipartUploadPartsAction: {},
PutObjectAction: {},
GetObjectAction: {},
GetObjectVersionAction: {},
DeleteObjectAction: {},
GetObjectAclAction: {},
GetObjectAttributesAction: {},
PutObjectAclAction: {},
RestoreObjectAction: {},
GetObjectTaggingAction: {},
PutObjectTaggingAction: {},
DeleteObjectTaggingAction: {},
GetObjectLegalHoldAction: {},
PutObjectLegalHoldAction: {},
GetObjectRetentionAction: {},
PutObjectRetentionAction: {},
BypassGovernanceRetentionAction: {},
AllActions: {},
}
// Validates Action: it should either wildcard match with supported actions list or be in it
func (a Action) IsValid() error {
if !strings.HasPrefix(string(a), "s3:") {
return policyErrInvalidAction
}
if a == AllActions {
return nil
}
// first check for an exact match
if _, ok := supportedActionList[a]; ok {
return nil
}
// walk through the supported actions and try wildcard match
for action := range supportedActionList {
if action.Match(a) {
return nil
}
}
return policyErrInvalidAction
}
func getBoolPtr(bl bool) *bool {
return &bl
}
// String converts the action to string
func (a Action) String() string {
return string(a)
}
// Match wildcard matches the given pattern to the action
func (a Action) Match(pattern Action) bool {
return matchPattern(pattern.String(), a.String())
}
// Checks if the action is object action
// nil points to 's3:*'
func (a Action) IsObjectAction() *bool {
if a == AllActions {
return nil
}
// first find an exact match
if _, ok := supportedObjectActionList[a]; ok {
return &ok
}
for action := range supportedObjectActionList {
if action.Match(a) {
return getBoolPtr(true)
}
}
return getBoolPtr(false)
}
type Actions map[Action]struct{}
// Override UnmarshalJSON method to decode both []string and string properties
func (a *Actions) UnmarshalJSON(data []byte) error {
ss := []string{}
var err error
if err = json.Unmarshal(data, &ss); err == nil {
if len(ss) == 0 {
return policyErrInvalidAction
}
*a = make(Actions)
for _, s := range ss {
err = a.Add(s)
if err != nil {
return err
}
}
} else {
var s string
if err = json.Unmarshal(data, &s); err == nil {
if s == "" {
return policyErrInvalidAction
}
*a = make(Actions)
err = a.Add(s)
if err != nil {
return err
}
}
}
return err
}
// Validates and adds a new Action to Actions map
func (a Actions) Add(str string) error {
action := Action(str)
err := action.IsValid()
if err != nil {
return err
}
a[action] = struct{}{}
return nil
}
// FindMatch tries to match the given action to the actions list
func (a Actions) FindMatch(action Action) bool {
_, ok := a[AllActions]
if ok {
return true
}
// First O(1) check for non wildcard actions
_, found := a[action]
if found {
return true
}
// search for a wildcard match
for act := range a {
if action.Match(act) {
return true
}
}
return false
}

View File

@@ -1,175 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
)
func TestAction_IsValid(t *testing.T) {
tests := []struct {
name string
action Action
wantErr bool
}{
{"valid exact action", GetObjectAction, false},
{"valid all actions", AllActions, false},
{"invalid prefix", "invalid:Action", true},
{"unsupported action 1", "s3:Unsupported", true},
{"unsupported action 2", "s3:HeadObject", true},
{"valid wildcard match 1", "s3:Get*", false},
{"valid wildcard match 2", "s3:*Object*", false},
{"valid wildcard match 3", "s3:*Multipart*", false},
{"any char match 1", "s3:Get?bject", false},
{"any char match 2", "s3:Get??bject", true},
{"any char match 3", "s3:???", true},
{"mixed match 1", "s3:Get?*", false},
{"mixed match 2", "s3:*Object?????", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.action.IsValid()
if tt.wantErr {
assert.EqualValues(t, policyErrInvalidAction, err)
} else {
assert.NoError(t, err)
}
})
}
}
func TestAction_String(t *testing.T) {
a := Action("s3:TestAction")
assert.Equal(t, "s3:TestAction", a.String())
}
func TestAction_Match(t *testing.T) {
tests := []struct {
name string
action Action
pattern Action
want bool
}{
{"exact match", "s3:GetObject", "s3:GetObject", true},
{"wildcard match", "s3:GetObject", "s3:Get*", true},
{"wildcard mismatch", "s3:PutObject", "s3:Get*", false},
{"any character match", "s3:Get1", "s3:Get?", true},
{"any character mismatch", "s3:Get12", "s3:Get?", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.action.Match(tt.pattern)
assert.Equal(t, tt.want, got)
})
}
}
func TestAction_IsObjectAction(t *testing.T) {
tests := []struct {
name string
action Action
want *bool
}{
{"all actions", AllActions, nil},
{"object action exact", GetObjectAction, getBoolPtr(true)},
{"object action wildcard", "s3:Get*", getBoolPtr(true)},
{"non object action", GetBucketAclAction, getBoolPtr(false)},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.action.IsObjectAction()
if tt.want == nil {
assert.Nil(t, got)
} else {
assert.NotNil(t, got)
assert.Equal(t, *tt.want, *got)
}
})
}
}
func TestActions_UnmarshalJSON(t *testing.T) {
tests := []struct {
name string
input string
wantErr bool
}{
{"valid slice", `["s3:GetObject","s3:PutObject"]`, false},
{"empty slice", `[]`, true},
{"invalid action in slice", `["s3:Invalid"]`, true},
{"valid string", `"s3:GetObject"`, false},
{"empty string", `""`, true},
{"invalid string", `"s3:Invalid"`, true},
{"invalid json", `{}`, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var a Actions
err := json.Unmarshal([]byte(tt.input), &a)
if tt.wantErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}
func TestActions_Add(t *testing.T) {
tests := []struct {
name string
action string
wantErr bool
}{
{"valid add", "s3:GetObject", false},
{"invalid add", "s3:InvalidAction", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
a := make(Actions)
err := a.Add(tt.action)
if tt.wantErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
_, ok := a[Action(tt.action)]
assert.True(t, ok)
}
})
}
}
func TestActions_FindMatch(t *testing.T) {
tests := []struct {
name string
actions Actions
check Action
want bool
}{
{"all actions present", Actions{AllActions: {}}, GetObjectAction, true},
{"exact match", Actions{GetObjectAction: {}}, GetObjectAction, true},
{"wildcard match", Actions{"s3:Get*": {}}, GetObjectAction, true},
{"no match", Actions{"s3:Put*": {}}, GetObjectAction, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.actions.FindMatch(tt.check)
assert.Equal(t, tt.want, got)
})
}
}

View File

@@ -1,35 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import "fmt"
type BucketPolicyAccessType string
const (
BucketPolicyAccessTypeDeny BucketPolicyAccessType = "Deny"
BucketPolicyAccessTypeAllow BucketPolicyAccessType = "Allow"
)
// Checks policy statement Effect to be valid ("Deny", "Allow")
func (bpat BucketPolicyAccessType) Validate() error {
switch bpat {
case BucketPolicyAccessTypeAllow, BucketPolicyAccessTypeDeny:
return nil
}
//lint:ignore ST1005 Reason: This error message is intended for end-user clarity and follows their expectations
return fmt.Errorf("Invalid effect: %v", bpat)
}

View File

@@ -1,57 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestBucketPolicyAccessType_Validate(t *testing.T) {
tests := []struct {
name string
input BucketPolicyAccessType
wantErr bool
errMsg string
}{
{
name: "valid allow",
input: BucketPolicyAccessTypeAllow,
wantErr: false,
},
{
name: "valid deny",
input: BucketPolicyAccessTypeDeny,
wantErr: false,
},
{
name: "invalid type",
input: BucketPolicyAccessType("InvalidValue"),
wantErr: true,
errMsg: "Invalid effect: InvalidValue",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.input.Validate()
if tt.wantErr {
assert.EqualError(t, err, tt.errMsg)
} else {
assert.NoError(t, err)
}
})
}
}

View File

@@ -1,130 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"encoding/json"
)
type Principals map[string]struct{}
func (p Principals) Add(key string) {
p[key] = struct{}{}
}
// Override UnmarshalJSON method to decode both []string and string properties
func (p *Principals) UnmarshalJSON(data []byte) error {
ss := []string{}
var s string
var k struct {
AWS string
}
var err error
if err = json.Unmarshal(data, &ss); err == nil {
if len(ss) == 0 {
return policyErrInvalidPrincipal
}
*p = make(Principals)
for _, s := range ss {
p.Add(s)
}
return nil
} else if err = json.Unmarshal(data, &s); err == nil {
if s == "" {
return policyErrInvalidPrincipal
}
*p = make(Principals)
p.Add(s)
return nil
} else if err = json.Unmarshal(data, &k); err == nil {
if k.AWS == "" {
return policyErrInvalidPrincipal
}
*p = make(Principals)
p.Add(k.AWS)
return nil
} else {
var sk struct {
AWS []string
}
if err = json.Unmarshal(data, &sk); err == nil {
if len(sk.AWS) == 0 {
return policyErrInvalidPrincipal
}
*p = make(Principals)
for _, s := range sk.AWS {
p.Add(s)
}
}
}
return err
}
// Converts Principals map to a slice, by omitting "*"
func (p Principals) ToSlice() []string {
principals := []string{}
for p := range p {
if p == "*" {
continue
}
principals = append(principals, p)
}
return principals
}
// Validates Principals by checking user account access keys existence
func (p Principals) Validate(iam IAMService) error {
_, containsWildCard := p["*"]
if containsWildCard {
if len(p) == 1 {
return nil
}
return policyErrInvalidPrincipal
}
accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
if err != nil {
return err
}
if len(accs) > 0 {
return policyErrInvalidPrincipal
}
return nil
}
func (p Principals) Contains(userAccess string) bool {
// "*" means it matches for any user account
_, ok := p["*"]
if ok {
return true
}
_, found := p[userAccess]
return found
}
// Bucket policy grants public access, if it contains
// a wildcard match to all the users
func (p Principals) isPublic() bool {
_, ok := p["*"]
return ok
}

View File

@@ -1,106 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
)
func TestPrincipals_Add(t *testing.T) {
p := make(Principals)
p.Add("user1")
_, ok := p["user1"]
assert.True(t, ok)
}
func TestPrincipals_UnmarshalJSON(t *testing.T) {
tests := []struct {
name string
input string
want Principals
wantErr bool
}{
{"valid slice", `["user1","user2"]`, Principals{"user1": {}, "user2": {}}, false},
{"empty slice", `[]`, nil, true},
{"valid string", `"user1"`, Principals{"user1": {}}, false},
{"empty string", `""`, nil, true},
{"valid AWS object", `{"AWS":"user1"}`, Principals{"user1": {}}, false},
{"empty AWS object", `{"AWS":""}`, nil, true},
{"valid AWS array", `{"AWS":["user1","user2"]}`, Principals{"user1": {}, "user2": {}}, false},
{"empty AWS array", `{"AWS":[]}`, nil, true},
{"invalid json", `{invalid}`, nil, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var p Principals
err := json.Unmarshal([]byte(tt.input), &p)
if tt.wantErr {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, tt.want, p)
}
})
}
}
func TestPrincipals_ToSlice(t *testing.T) {
p := Principals{"user1": {}, "user2": {}, "*": {}}
got := p.ToSlice()
assert.Contains(t, got, "user1")
assert.Contains(t, got, "user2")
assert.NotContains(t, got, "*")
}
func TestPrincipals_Validate(t *testing.T) {
iamSingle := NewIAMServiceSingle(Account{
Access: "user1",
})
tests := []struct {
name string
principals Principals
mockIAM IAMService
err error
}{
{"only wildcard", Principals{"*": {}}, iamSingle, nil},
{"wildcard and user", Principals{"*": {}, "user1": {}}, iamSingle, policyErrInvalidPrincipal},
{"accounts exist returns err", Principals{"user2": {}, "user3": {}}, iamSingle, policyErrInvalidPrincipal},
{"accounts exist non-empty", Principals{"user1": {}}, iamSingle, nil},
{"accounts valid", Principals{"user1": {}}, iamSingle, nil},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.principals.Validate(tt.mockIAM)
assert.EqualValues(t, tt.err, err)
})
}
}
func TestPrincipals_Contains(t *testing.T) {
p := Principals{"user1": {}}
assert.True(t, p.Contains("user1"))
assert.False(t, p.Contains("user2"))
p = Principals{"*": {}}
assert.True(t, p.Contains("anyuser"))
}
func TestPrincipals_isPublic(t *testing.T) {
assert.True(t, Principals{"*": {}}.isPublic())
assert.False(t, Principals{"user1": {}}.isPublic())
}

View File

@@ -1,134 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"encoding/json"
"strings"
)
type Resources map[string]struct{}
const ResourceArnPrefix = "arn:aws:s3:::"
// Override UnmarshalJSON method to decode both []string and string properties
func (r *Resources) UnmarshalJSON(data []byte) error {
ss := []string{}
var err error
if err = json.Unmarshal(data, &ss); err == nil {
if len(ss) == 0 {
return policyErrInvalidResource
}
*r = make(Resources)
for _, s := range ss {
err = r.Add(s)
if err != nil {
return err
}
}
} else {
var s string
if err = json.Unmarshal(data, &s); err == nil {
if s == "" {
return policyErrInvalidResource
}
*r = make(Resources)
err = r.Add(s)
if err != nil {
return err
}
}
}
return err
}
// Adds and validates a new resource to Resources map
func (r Resources) Add(rc string) error {
ok, pattern := isValidResource(rc)
if !ok {
return policyErrInvalidResource
}
r[pattern] = struct{}{}
return nil
}
// Checks if the resources contain object pattern
func (r Resources) ContainsObjectPattern() bool {
for resource := range r {
if resource == "*" || strings.Contains(resource, "/") {
return true
}
}
return false
}
// Checks if the resources contain bucket pattern
func (r Resources) ContainsBucketPattern() bool {
for resource := range r {
if resource == "*" || !strings.Contains(resource, "/") {
return true
}
}
return false
}
// Bucket resources should start with bucket name: arn:aws:s3:::MyBucket/*
func (r Resources) Validate(bucket string) error {
for resource := range r {
if !strings.HasPrefix(resource, bucket) {
return policyErrInvalidResource
}
}
return nil
}
func (r Resources) FindMatch(resource string) bool {
for res := range r {
if r.Match(res, resource) {
return true
}
}
return false
}
// Match matches the given input resource with the pattern
func (r Resources) Match(pattern, input string) bool {
return matchPattern(pattern, input)
}
// Checks the resource to have arn prefix and not starting with /
func isValidResource(rc string) (isValid bool, pattern string) {
if !strings.HasPrefix(rc, ResourceArnPrefix) {
return false, ""
}
res := strings.TrimPrefix(rc, ResourceArnPrefix)
if res == "" {
return false, ""
}
// The resource can't start with / (bucket name comes first)
if strings.HasPrefix(res, "/") {
return false, ""
}
return true, res
}

View File

@@ -1,182 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"encoding/json"
"testing"
)
func TestUnmarshalJSON(t *testing.T) {
var r Resources
cases := []struct {
input string
expected int
wantErr bool
}{
{`"arn:aws:s3:::my-bucket/*"`, 1, false},
{`["arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket"]`, 2, false},
{`""`, 0, true},
{`[]`, 0, true},
{`["invalid-bucket"]`, 0, true},
}
for _, tc := range cases {
r = Resources{}
err := json.Unmarshal([]byte(tc.input), &r)
if (err != nil) != tc.wantErr {
t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
}
if len(r) != tc.expected {
t.Errorf("Expected %d resources, got %d", tc.expected, len(r))
}
}
}
func TestAdd(t *testing.T) {
r := Resources{}
cases := []struct {
input string
wantErr bool
}{
{"arn:aws:s3:::valid-bucket/*", false},
{"arn:aws:s3:::valid-bucket/object", false},
{"invalid-bucket/*", true},
{"/invalid-start", true},
}
for _, tc := range cases {
err := r.Add(tc.input)
if (err != nil) != tc.wantErr {
t.Errorf("Unexpected error status for input %s: %v", tc.input, err)
}
}
}
func TestContainsObjectPattern(t *testing.T) {
cases := []struct {
resources []string
expected bool
}{
{[]string{"arn:aws:s3:::my-bucket/my-object"}, true},
{[]string{"arn:aws:s3:::my-bucket/*"}, true},
{[]string{"arn:aws:s3:::my-bucket"}, false},
}
for _, tc := range cases {
r := Resources{}
for _, res := range tc.resources {
r.Add(res)
}
if r.ContainsObjectPattern() != tc.expected {
t.Errorf("Expected object pattern to be %v for %v", tc.expected, tc.resources)
}
}
}
func TestContainsBucketPattern(t *testing.T) {
cases := []struct {
resources []string
expected bool
}{
{[]string{"arn:aws:s3:::my-bucket"}, true},
{[]string{"arn:aws:s3:::my-bucket/*"}, false},
{[]string{"arn:aws:s3:::my-bucket/object"}, false},
}
for _, tc := range cases {
r := Resources{}
for _, res := range tc.resources {
r.Add(res)
}
if r.ContainsBucketPattern() != tc.expected {
t.Errorf("Expected bucket pattern to be %v for %v", tc.expected, tc.resources)
}
}
}
func TestValidate(t *testing.T) {
cases := []struct {
resources []string
bucket string
expected bool
}{
{[]string{"arn:aws:s3:::valid-bucket/*"}, "valid-bucket", true},
{[]string{"arn:aws:s3:::wrong-bucket/*"}, "valid-bucket", false},
{[]string{"arn:aws:s3:::valid-bucket/*", "arn:aws:s3:::valid-bucket/object/*"}, "valid-bucket", true},
}
for _, tc := range cases {
r := Resources{}
for _, res := range tc.resources {
r.Add(res)
}
if (r.Validate(tc.bucket) == nil) != tc.expected {
t.Errorf("Expected validation to be %v for bucket %s", tc.expected, tc.bucket)
}
}
}
func TestFindMatch(t *testing.T) {
cases := []struct {
resources []string
input string
expected bool
}{
{[]string{"arn:aws:s3:::my-bucket/*"}, "my-bucket/my-object", true},
{[]string{"arn:aws:s3:::my-bucket/object"}, "other-bucket/my-object", false},
{[]string{"arn:aws:s3:::my-bucket/object"}, "my-bucket/object", true},
{[]string{"arn:aws:s3:::my-bucket/*", "arn:aws:s3:::other-bucket/*"}, "other-bucket/something", true},
}
for _, tc := range cases {
r := Resources{}
for _, res := range tc.resources {
r.Add(res)
}
if r.FindMatch(tc.input) != tc.expected {
t.Errorf("Expected FindMatch to be %v for input %s", tc.expected, tc.input)
}
}
}
func TestMatch(t *testing.T) {
r := Resources{}
cases := []struct {
pattern string
input string
expected bool
}{
{"my-bucket/*", "my-bucket/object", true},
{"my-bucket/?bject", "my-bucket/object", true},
{"my-bucket/*", "other-bucket/object", false},
{"*", "any-bucket/object", true},
{"my-bucket/*", "my-bucket/subdir/object", true},
{"my-bucket/*", "other-bucket", false},
{"my-bucket/*/*", "my-bucket/hello", false},
{"my-bucket/*/*", "my-bucket/hello/world", true},
{"foo/???/bar", "foo/qux/bar", true},
{"foo/???/bar", "foo/quxx/bar", false},
{"foo/???/bar/*/?", "foo/qux/bar/hello/g", true},
{"foo/???/bar/*/?", "foo/qux/bar/hello/smth", false},
}
for _, tc := range cases {
if r.Match(tc.pattern, tc.input) != tc.expected {
t.Errorf("Match(%s, %s) failed, expected %v", tc.pattern, tc.input, tc.expected)
}
}
}

View File

@@ -1,191 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"errors"
"fmt"
"time"
"github.com/versity/versitygw/s3err"
)
type Role string
const (
RoleUser Role = "user"
RoleAdmin Role = "admin"
RoleUserPlus Role = "userplus"
)
func (r Role) IsValid() bool {
switch r {
case RoleAdmin:
return true
case RoleUser:
return true
case RoleUserPlus:
return true
default:
return false
}
}
// Account is a gateway IAM account
type Account struct {
Access string `json:"access"`
Secret string `json:"secret"`
Role Role `json:"role"`
UserID int `json:"userID"`
GroupID int `json:"groupID"`
}
type ListUserAccountsResult struct {
Accounts []Account
}
// Mutable props, which could be changed when updating an IAM account
type MutableProps struct {
Secret *string `json:"secret"`
Role Role `json:"role"`
UserID *int `json:"userID"`
GroupID *int `json:"groupID"`
}
func (m MutableProps) Validate() error {
if m.Role != "" && !m.Role.IsValid() {
return s3err.GetAPIError(s3err.ErrAdminInvalidUserRole)
}
return nil
}
func updateAcc(acc *Account, props MutableProps) {
if props.Secret != nil {
acc.Secret = *props.Secret
}
if props.GroupID != nil {
acc.GroupID = *props.GroupID
}
if props.UserID != nil {
acc.UserID = *props.UserID
}
if props.Role != "" {
acc.Role = props.Role
}
}
// IAMService is the interface for all IAM service implementations
//
//go:generate moq -out ../s3api/controllers/iam_moq_test.go -pkg controllers . IAMService
type IAMService interface {
CreateAccount(account Account) error
GetUserAccount(access string) (Account, error)
UpdateUserAccount(access string, props MutableProps) error
DeleteUserAccount(access string) error
ListUserAccounts() ([]Account, error)
Shutdown() error
}
var (
// ErrUserExists is returned when the user already exists
ErrUserExists = errors.New("user already exists")
// ErrNoSuchUser is returned when the user does not exist
ErrNoSuchUser = errors.New("user not found")
)
type Opts struct {
RootAccount Account
Dir string
LDAPServerURL string
LDAPBindDN string
LDAPPassword string
LDAPQueryBase string
LDAPObjClasses string
LDAPAccessAtr string
LDAPSecretAtr string
LDAPRoleAtr string
LDAPUserIdAtr string
LDAPGroupIdAtr string
VaultEndpointURL string
VaultSecretStoragePath string
VaultAuthMethod string
VaultMountPath string
VaultRootToken string
VaultRoleId string
VaultRoleSecret string
VaultServerCert string
VaultClientCert string
VaultClientCertKey string
S3Access string
S3Secret string
S3Region string
S3Bucket string
S3Endpoint string
S3DisableSSlVerfiy bool
CacheDisable bool
CacheTTL int
CachePrune int
IpaHost string
IpaVaultName string
IpaUser string
IpaPassword string
IpaInsecure bool
}
func New(o *Opts) (IAMService, error) {
var svc IAMService
var err error
switch {
case o.Dir != "":
svc, err = NewInternal(o.RootAccount, o.Dir)
fmt.Printf("initializing internal IAM with %q\n", o.Dir)
case o.LDAPServerURL != "":
svc, err = NewLDAPService(o.RootAccount, o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr, o.LDAPUserIdAtr,
o.LDAPGroupIdAtr, o.LDAPObjClasses)
fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
case o.S3Endpoint != "":
svc, err = NewS3(o.RootAccount, o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
o.S3Endpoint, o.S3DisableSSlVerfiy)
fmt.Printf("initializing S3 IAM with '%v/%v'\n",
o.S3Endpoint, o.S3Bucket)
case o.VaultEndpointURL != "":
svc, err = NewVaultIAMService(o.RootAccount, o.VaultEndpointURL, o.VaultSecretStoragePath,
o.VaultAuthMethod, o.VaultMountPath, o.VaultRootToken, o.VaultRoleId, o.VaultRoleSecret,
o.VaultServerCert, o.VaultClientCert, o.VaultClientCertKey)
fmt.Printf("initializing Vault IAM with %q\n", o.VaultEndpointURL)
case o.IpaHost != "":
svc, err = NewIpaIAMService(o.RootAccount, o.IpaHost, o.IpaVaultName, o.IpaUser, o.IpaPassword, o.IpaInsecure)
fmt.Printf("initializing IPA IAM with %q\n", o.IpaHost)
default:
// if no iam options selected, default to the single user mode
fmt.Println("No IAM service configured, enabling single account mode")
return NewIAMServiceSingle(o.RootAccount), nil
}
if err != nil {
return nil, err
}
if o.CacheDisable {
return svc, nil
}
return NewCache(svc,
time.Duration(o.CacheTTL)*time.Second,
time.Duration(o.CachePrune)*time.Second), nil
}

View File

@@ -1,204 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"context"
"strings"
"sync"
"time"
)
// IAMCache is an in memory cache of the IAM accounts
// with expiration. This helps to alleviate the load on
// the real IAM service if the gateway is handling
// many requests. This forwards account updates to the
// underlying service, and returns cached results while
// the in memory account is not expired.
type IAMCache struct {
service IAMService
iamcache *icache
cancel context.CancelFunc
}
var _ IAMService = &IAMCache{}
type item struct {
value Account
exp time.Time
}
type icache struct {
sync.RWMutex
expire time.Duration
items map[string]item
}
func (i *icache) set(k string, v Account) {
cpy := v
i.Lock()
i.items[k] = item{
exp: time.Now().Add(i.expire),
value: cpy,
}
i.Unlock()
}
func (i *icache) get(k string) (Account, bool) {
i.RLock()
v, ok := i.items[k]
i.RUnlock()
if !ok || !v.exp.After(time.Now()) {
return Account{}, false
}
return v.value, true
}
func (i *icache) update(k string, props MutableProps) {
i.Lock()
defer i.Unlock()
item, found := i.items[k]
if found {
updateAcc(&item.value, props)
// refresh the expiration date
item.exp = time.Now().Add(i.expire)
i.items[k] = item
}
}
func (i *icache) Delete(k string) {
i.Lock()
delete(i.items, k)
i.Unlock()
}
func (i *icache) gcCache(ctx context.Context, interval time.Duration) {
for {
if ctx.Err() != nil {
break
}
now := time.Now()
i.Lock()
// prune expired entries
for k, v := range i.items {
if now.After(v.exp) {
delete(i.items, k)
}
}
i.Unlock()
// sleep for the clean interval or context cancelation,
// whichever comes first
select {
case <-ctx.Done():
case <-time.After(interval):
}
}
}
// NewCache initializes an IAM cache for the provided service. The expireTime
// is the duration a cache entry can be valid, and the cleanupInterval is
// how often to scan cache and cleanup expired entries.
func NewCache(service IAMService, expireTime, cleanupInterval time.Duration) *IAMCache {
i := &IAMCache{
service: service,
iamcache: &icache{
items: make(map[string]item),
expire: expireTime,
},
}
ctx, cancel := context.WithCancel(context.Background())
go i.iamcache.gcCache(ctx, cleanupInterval)
i.cancel = cancel
return i
}
// CreateAccount send create to IAM service and creates an account cache entry
func (c *IAMCache) CreateAccount(account Account) error {
err := c.service.CreateAccount(account)
if err != nil {
return err
}
// we need a copy of account to be able to store beyond the
// lifetime of the request, otherwise Fiber will reuse and corrupt
// these entries
acct := Account{
Access: strings.Clone(account.Access),
Secret: strings.Clone(account.Secret),
Role: Role(strings.Clone(string(account.Role))),
}
c.iamcache.set(acct.Access, acct)
return nil
}
// GetUserAccount retrieves the cache account if it is in the cache and not
// expired. Otherwise retrieves from underlying IAM service and caches
// result for the expire duration.
func (c *IAMCache) GetUserAccount(access string) (Account, error) {
acct, found := c.iamcache.get(access)
if found {
return acct, nil
}
a, err := c.service.GetUserAccount(access)
if err != nil {
return Account{}, err
}
c.iamcache.set(access, a)
return a, nil
}
// DeleteUserAccount deletes account from IAM service and cache
func (c *IAMCache) DeleteUserAccount(access string) error {
err := c.service.DeleteUserAccount(access)
if err != nil {
return err
}
c.iamcache.Delete(access)
return nil
}
func (c *IAMCache) UpdateUserAccount(access string, props MutableProps) error {
err := c.service.UpdateUserAccount(access, props)
if err != nil {
return err
}
c.iamcache.update(access, props)
return nil
}
// ListUserAccounts is a passthrough to the underlying service and
// does not make use of the cache
func (c *IAMCache) ListUserAccounts() ([]Account, error) {
return c.service.ListUserAccounts()
}
// Shutdown graceful termination of service
func (c *IAMCache) Shutdown() error {
c.cancel()
return nil
}

View File

@@ -1,354 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"sync"
"time"
)
const (
iamFile = "users.json"
iamBackupFile = "users.json.backup"
)
// IAMServiceInternal manages the internal IAM service
type IAMServiceInternal struct {
// This mutex will help with racing updates to the IAM data
// from multiple requests to this gateway instance, but
// will not help with racing updates to multiple load balanced
// gateway instances. This is a limitation of the internal
// IAM service. All account updates should be sent to a single
// gateway instance if possible.
sync.RWMutex
dir string
rootAcc Account
}
// UpdateAcctFunc accepts the current data and returns the new data to be stored
type UpdateAcctFunc func([]byte) ([]byte, error)
// iAMConfig stores all internal IAM accounts
type iAMConfig struct {
AccessAccounts map[string]Account `json:"accessAccounts"`
}
var _ IAMService = &IAMServiceInternal{}
// NewInternal creates a new instance for the Internal IAM service
func NewInternal(rootAcc Account, dir string) (*IAMServiceInternal, error) {
i := &IAMServiceInternal{
dir: dir,
rootAcc: rootAcc,
}
err := i.initIAM()
if err != nil {
return nil, fmt.Errorf("init iam: %w", err)
}
return i, nil
}
// CreateAccount creates a new IAM account. Returns an error if the account
// already exists.
func (s *IAMServiceInternal) CreateAccount(account Account) error {
if account.Access == s.rootAcc.Access {
return ErrUserExists
}
s.Lock()
defer s.Unlock()
return s.storeIAM(func(data []byte) ([]byte, error) {
conf, err := parseIAM(data)
if err != nil {
return nil, fmt.Errorf("get iam data: %w", err)
}
_, ok := conf.AccessAccounts[account.Access]
if ok {
return nil, ErrUserExists
}
conf.AccessAccounts[account.Access] = account
b, err := json.Marshal(conf)
if err != nil {
return nil, fmt.Errorf("failed to serialize iam: %w", err)
}
return b, nil
})
}
// GetUserAccount retrieves account info for the requested user. Returns
// ErrNoSuchUser if the account does not exist.
func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
if access == s.rootAcc.Access {
return s.rootAcc, nil
}
s.RLock()
defer s.RUnlock()
conf, err := s.getIAM()
if err != nil {
return Account{}, fmt.Errorf("get iam data: %w", err)
}
acct, ok := conf.AccessAccounts[access]
if !ok {
return Account{}, ErrNoSuchUser
}
return acct, nil
}
// UpdateUserAccount updates the specified user account fields. Returns
// ErrNoSuchUser if the account does not exist.
func (s *IAMServiceInternal) UpdateUserAccount(access string, props MutableProps) error {
s.Lock()
defer s.Unlock()
return s.storeIAM(func(data []byte) ([]byte, error) {
conf, err := parseIAM(data)
if err != nil {
return nil, fmt.Errorf("get iam data: %w", err)
}
acc, found := conf.AccessAccounts[access]
if !found {
return nil, ErrNoSuchUser
}
updateAcc(&acc, props)
conf.AccessAccounts[access] = acc
b, err := json.Marshal(conf)
if err != nil {
return nil, fmt.Errorf("failed to serialize iam: %w", err)
}
return b, nil
})
}
// DeleteUserAccount deletes the specified user account. Does not check if
// account exists.
func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
s.Lock()
defer s.Unlock()
return s.storeIAM(func(data []byte) ([]byte, error) {
conf, err := parseIAM(data)
if err != nil {
return nil, fmt.Errorf("get iam data: %w", err)
}
delete(conf.AccessAccounts, access)
b, err := json.Marshal(conf)
if err != nil {
return nil, fmt.Errorf("failed to serialize iam: %w", err)
}
return b, nil
})
}
// ListUserAccounts lists all the user accounts stored.
func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
s.RLock()
defer s.RUnlock()
conf, err := s.getIAM()
if err != nil {
return []Account{}, fmt.Errorf("get iam data: %w", err)
}
keys := make([]string, 0, len(conf.AccessAccounts))
for k := range conf.AccessAccounts {
keys = append(keys, k)
}
sort.Strings(keys)
var accs []Account
for _, k := range keys {
accs = append(accs, Account{
Access: k,
Secret: conf.AccessAccounts[k].Secret,
Role: conf.AccessAccounts[k].Role,
UserID: conf.AccessAccounts[k].UserID,
GroupID: conf.AccessAccounts[k].GroupID,
})
}
return accs, nil
}
// Shutdown graceful termination of service
func (s *IAMServiceInternal) Shutdown() error {
return nil
}
const (
iamMode = 0600
)
func (s *IAMServiceInternal) initIAM() error {
fname := filepath.Join(s.dir, iamFile)
_, err := os.ReadFile(fname)
if errors.Is(err, fs.ErrNotExist) {
b, err := json.Marshal(iAMConfig{AccessAccounts: map[string]Account{}})
if err != nil {
return fmt.Errorf("marshal default iam: %w", err)
}
err = os.WriteFile(fname, b, iamMode)
if err != nil {
return fmt.Errorf("write default iam: %w", err)
}
}
return nil
}
func (s *IAMServiceInternal) getIAM() (iAMConfig, error) {
b, err := s.readIAMData()
if err != nil {
return iAMConfig{}, err
}
return parseIAM(b)
}
func parseIAM(b []byte) (iAMConfig, error) {
var conf iAMConfig
if err := json.Unmarshal(b, &conf); err != nil {
return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
}
if conf.AccessAccounts == nil {
conf.AccessAccounts = make(map[string]Account)
}
return conf, nil
}
const (
backoff = 100 * time.Millisecond
maxretry = 300
)
func (s *IAMServiceInternal) readIAMData() ([]byte, error) {
// We are going to be racing with other running gateways without any
// coordination. So we might find the file does not exist at times.
// For this case we need to retry for a while assuming the other gateway
// will eventually write the file. If it doesn't after the max retries,
// then we will return the error.
retries := 0
for {
b, err := os.ReadFile(filepath.Join(s.dir, iamFile))
if errors.Is(err, fs.ErrNotExist) {
// racing with someone else updating
// keep retrying after backoff
retries++
if retries < maxretry {
time.Sleep(backoff)
continue
}
return nil, fmt.Errorf("read iam file: %w", err)
}
if err != nil {
return nil, err
}
return b, nil
}
}
func (s *IAMServiceInternal) storeIAM(update UpdateAcctFunc) error {
// We are going to be racing with other running gateways without any
// coordination. So the strategy here is to read the current file data,
// update the data, write back out to a temp file, then rename the
// temp file to the original file. This rename will replace the
// original file with the new file. This is atomic and should always
// allow for a consistent view of the data. There is a small
// window where the file could be read and then updated by
// another process. In this case any updates the other process did
// will be lost. This is a limitation of the internal IAM service.
// This should be rare, and even when it does happen should result
// in a valid IAM file, just without the other process's updates.
iamFname := filepath.Join(s.dir, iamFile)
backupFname := filepath.Join(s.dir, iamBackupFile)
b, err := os.ReadFile(iamFname)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("read iam file: %w", err)
}
// save copy of data
datacopy := make([]byte, len(b))
copy(datacopy, b)
// make a backup copy in case something happens
err = s.writeUsingTempFile(b, backupFname)
if err != nil {
return fmt.Errorf("write backup iam file: %w", err)
}
b, err = update(b)
if err != nil {
return fmt.Errorf("update iam data: %w", err)
}
err = s.writeUsingTempFile(b, iamFname)
if err != nil {
return fmt.Errorf("write iam file: %w", err)
}
return nil
}
func (s *IAMServiceInternal) writeUsingTempFile(b []byte, fname string) error {
f, err := os.CreateTemp(s.dir, iamFile)
if err != nil {
return fmt.Errorf("create temp file: %w", err)
}
defer os.Remove(f.Name())
_, err = f.Write(b)
f.Close()
if err != nil {
return fmt.Errorf("write temp file: %w", err)
}
err = os.Rename(f.Name(), fname)
if err != nil {
return fmt.Errorf("rename temp file: %w", err)
}
return nil
}

View File

@@ -1,496 +0,0 @@
// Copyright 2025 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/cookiejar"
"net/url"
"slices"
"strconv"
"strings"
"syscall"
"time"
"github.com/versity/versitygw/debuglogger"
)
const IpaVersion = "2.254"
type IpaIAMService struct {
client http.Client
id int
version string
host string
vaultName string
username string
password string
kraTransportKey *rsa.PublicKey
rootAcc Account
}
var _ IAMService = &IpaIAMService{}
func NewIpaIAMService(rootAcc Account, host, vaultName, username, password string, isInsecure bool) (*IpaIAMService, error) {
ipa := IpaIAMService{
id: 0,
version: IpaVersion,
host: host,
vaultName: vaultName,
username: username,
password: password,
rootAcc: rootAcc,
}
jar, err := cookiejar.New(nil)
if err != nil {
// this should never happen
return nil, fmt.Errorf("cookie jar creation: %w", err)
}
mTLSConfig := &tls.Config{InsecureSkipVerify: isInsecure}
tr := &http.Transport{
TLSClientConfig: mTLSConfig,
Proxy: http.ProxyFromEnvironment,
}
ipa.client = http.Client{Jar: jar, Transport: tr}
err = ipa.login()
if err != nil {
return nil, fmt.Errorf("ipa login failed: %w", err)
}
req, err := ipa.newRequest("vaultconfig_show/1", []string{}, map[string]any{"all": true})
if err != nil {
return nil, fmt.Errorf("ipa vaultconfig_show: %w", err)
}
vaultConfig := struct {
Kra_Server_Server []string
Transport_Cert Base64EncodedWrapped
Wrapping_default_algorithm string
Wrapping_supported_algorithms []string
}{}
err = ipa.rpc(req, &vaultConfig)
if err != nil {
return nil, fmt.Errorf("ipa vault config: %w", err)
}
cert, err := x509.ParseCertificate(vaultConfig.Transport_Cert)
if err != nil {
return nil, fmt.Errorf("ipa cannot parse vault certificate: %w", err)
}
ipa.kraTransportKey = cert.PublicKey.(*rsa.PublicKey)
isSupported := slices.Contains(vaultConfig.Wrapping_supported_algorithms, "aes-128-cbc")
if !isSupported {
return nil,
fmt.Errorf("IPA vault does not support aes-128-cbc. Only %v supported",
vaultConfig.Wrapping_supported_algorithms)
}
return &ipa, nil
}
func (ipa *IpaIAMService) CreateAccount(account Account) error {
return fmt.Errorf("not implemented")
}
func (ipa *IpaIAMService) GetUserAccount(access string) (Account, error) {
if access == ipa.rootAcc.Access {
return ipa.rootAcc, nil
}
req, err := ipa.newRequest("user_show/1", []string{access}, map[string]any{})
if err != nil {
return Account{}, fmt.Errorf("ipa user_show: %w", err)
}
userResult := struct {
Gidnumber []string
Uidnumber []string
}{}
err = ipa.rpc(req, &userResult)
if err != nil {
return Account{}, err
}
uid, err := strconv.Atoi(userResult.Uidnumber[0])
if err != nil {
return Account{}, fmt.Errorf("ipa uid invalid: %w", err)
}
gid, err := strconv.Atoi(userResult.Gidnumber[0])
if err != nil {
return Account{}, fmt.Errorf("ipa gid invalid: %w", err)
}
account := Account{
Access: access,
Role: RoleUser,
UserID: uid,
GroupID: gid,
}
session_key := make([]byte, 16)
_, err = rand.Read(session_key)
if err != nil {
return account, fmt.Errorf("ipa cannot generate session key: %w", err)
}
encryptedKey, err := rsa.EncryptPKCS1v15(rand.Reader, ipa.kraTransportKey, session_key)
if err != nil {
return account, fmt.Errorf("ipa vault secret retrieval: %w", err)
}
req, err = ipa.newRequest("vault_retrieve_internal/1", []string{ipa.vaultName},
map[string]any{"username": access,
"session_key": Base64EncodedWrapped(encryptedKey),
"wrapping_algo": "aes-128-cbc"})
if err != nil {
return Account{}, fmt.Errorf("ipa vault_retrieve_internal: %w", err)
}
data := struct {
Vault_data Base64EncodedWrapped
Nonce Base64EncodedWrapped
}{}
err = ipa.rpc(req, &data)
if err != nil {
return account, err
}
aes, err := aes.NewCipher(session_key)
if err != nil {
return account, fmt.Errorf("ipa cannot create AES cipher: %w", err)
}
cbc := cipher.NewCBCDecrypter(aes, data.Nonce)
cbc.CryptBlocks(data.Vault_data, data.Vault_data)
secretUnpaddedJson, err := pkcs7Unpad(data.Vault_data, 16)
if err != nil {
return account, fmt.Errorf("ipa cannot unpad decrypted result: %w", err)
}
secret := struct {
Data Base64Encoded
}{}
json.Unmarshal(secretUnpaddedJson, &secret)
account.Secret = string(secret.Data)
return account, nil
}
func (ipa *IpaIAMService) UpdateUserAccount(access string, props MutableProps) error {
return fmt.Errorf("not implemented")
}
func (ipa *IpaIAMService) DeleteUserAccount(access string) error {
return fmt.Errorf("not implemented")
}
func (ipa *IpaIAMService) ListUserAccounts() ([]Account, error) {
return []Account{}, fmt.Errorf("not implemented")
}
func (ipa *IpaIAMService) Shutdown() error {
return nil
}
// Implementation
const requestRetries = 3
func (ipa *IpaIAMService) login() error {
form := url.Values{}
form.Set("user", ipa.username)
form.Set("password", ipa.password)
req, err := http.NewRequest(
"POST",
fmt.Sprintf("%s/ipa/session/login_password", ipa.host),
strings.NewReader(form.Encode()))
if err != nil {
return err
}
req.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
var resp *http.Response
for i := range requestRetries {
resp, err = ipa.client.Do(req)
if err == nil {
break
}
// Check for transient network errors
if isRetryable(err) {
time.Sleep(time.Second * time.Duration(i+1))
continue
}
return fmt.Errorf("login POST to %s failed: %w", req.URL, err)
}
if err != nil {
return fmt.Errorf("login POST to %s failed after retries: %w",
req.URL, err)
}
defer resp.Body.Close()
if resp.StatusCode == 401 {
return errors.New("cannot login to FreeIPA: invalid credentials")
}
if resp.StatusCode != 200 {
return fmt.Errorf("cannot login to FreeIPA: status code %d",
resp.StatusCode)
}
return nil
}
type rpcRequest = string
type rpcResponse struct {
Result json.RawMessage
Principal string
Id int
Version string
}
func (p rpcResponse) String() string {
return string(p.Result)
}
var errRpc = errors.New("IPA RPC error")
func (ipa *IpaIAMService) rpc(req rpcRequest, value any) error {
err := ipa.login()
if err != nil {
return err
}
res, err := ipa.rpcInternal(req)
if err != nil {
return err
}
return json.Unmarshal(res.Result, value)
}
func (ipa *IpaIAMService) rpcInternal(req rpcRequest) (rpcResponse, error) {
httpReq, err := http.NewRequest("POST",
fmt.Sprintf("%s/ipa/session/json", ipa.host),
strings.NewReader(req))
if err != nil {
return rpcResponse{}, err
}
debuglogger.IAMLogf("IPA request: %v", req)
httpReq.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
httpReq.Header.Set("Content-Type", "application/json")
var httpResp *http.Response
for i := range requestRetries {
httpResp, err = ipa.client.Do(httpReq)
if err == nil {
break
}
// Check for transient network errors
if isRetryable(err) {
time.Sleep(time.Second * time.Duration(i+1))
continue
}
return rpcResponse{}, fmt.Errorf("ipa request to %s failed: %w",
httpReq.URL, err)
}
if err != nil {
return rpcResponse{},
fmt.Errorf("ipa request to %s failed after retries: %w",
httpReq.URL, err)
}
defer httpResp.Body.Close()
bytes, err := io.ReadAll(httpResp.Body)
debuglogger.IAMLogf("IPA response (%v): %v", err, string(bytes))
if err != nil {
return rpcResponse{}, err
}
result := struct {
Result struct {
Json json.RawMessage `json:"result"`
Value string `json:"value"`
Summary any `json:"summary"`
} `json:"result"`
Error json.RawMessage `json:"error"`
Id int `json:"id"`
Principal string `json:"principal"`
Version string `json:"version"`
}{}
err = json.Unmarshal(bytes, &result)
if err != nil {
return rpcResponse{}, err
}
if string(result.Error) != "null" {
return rpcResponse{}, fmt.Errorf("%s: %w", string(result.Error), errRpc)
}
return rpcResponse{
Result: result.Result.Json,
Principal: result.Principal,
Id: result.Id,
Version: result.Version,
}, nil
}
func isRetryable(err error) bool {
if err == nil {
return false
}
if errors.Is(err, io.EOF) {
return true
}
if err, ok := err.(net.Error); ok && err.Timeout() {
return true
}
if opErr, ok := err.(*net.OpError); ok {
if sysErr, ok := opErr.Err.(*syscall.Errno); ok {
if *sysErr == syscall.ECONNRESET {
return true
}
}
}
return false
}
func (ipa *IpaIAMService) newRequest(method string, args []string, dict map[string]any) (rpcRequest, error) {
id := ipa.id
ipa.id++
dict["version"] = ipa.version
jmethod, errMethod := json.Marshal(method)
jargs, errArgs := json.Marshal(args)
jdict, errDict := json.Marshal(dict)
err := errors.Join(errMethod, errArgs, errDict)
if err != nil {
return "", fmt.Errorf("ipa request invalid: %w", err)
}
request := map[string]interface{}{
"id": id,
"method": json.RawMessage(jmethod),
"params": []json.RawMessage{json.RawMessage(jargs), json.RawMessage(jdict)},
}
requestJSON, err := json.Marshal(request)
if err != nil {
return "", fmt.Errorf("failed to marshal request: %w", err)
}
return string(requestJSON), nil
}
// pkcs7Unpad validates and unpads data from the given bytes slice.
// The returned value will be 1 to n bytes smaller depending on the
// amount of padding, where n is the block size.
func pkcs7Unpad(b []byte, blocksize int) ([]byte, error) {
if blocksize <= 0 {
return nil, errors.New("invalid blocksize")
}
if len(b) == 0 {
return nil, errors.New("invalid PKCS7 data (empty or not padded)")
}
if len(b)%blocksize != 0 {
return nil, errors.New("invalid padding on input")
}
c := b[len(b)-1]
n := int(c)
if n == 0 || n > len(b) {
return nil, errors.New("invalid padding on input")
}
for i := 0; i < n; i++ {
if b[len(b)-n+i] != c {
return nil, errors.New("invalid padding on input")
}
}
return b[:len(b)-n], nil
}
/*
e.g.
"value" {
"__base64__": "aGVsbG93b3JsZAo="
}
*/
type Base64EncodedWrapped []byte
func (b *Base64EncodedWrapped) UnmarshalJSON(data []byte) error {
intermediate := struct {
Base64 string `json:"__base64__"`
}{}
err := json.Unmarshal(data, &intermediate)
if err != nil {
return err
}
*b, err = base64.StdEncoding.DecodeString(intermediate.Base64)
return err
}
func (b *Base64EncodedWrapped) MarshalJSON() ([]byte, error) {
intermediate := struct {
Base64 string `json:"__base64__"`
}{Base64: base64.StdEncoding.EncodeToString(*b)}
return json.Marshal(intermediate)
}
/*
e.g.
"value": "aGVsbG93b3JsZAo="
*/
type Base64Encoded []byte
func (b *Base64Encoded) UnmarshalJSON(data []byte) error {
var intermediate string
err := json.Unmarshal(data, &intermediate)
if err != nil {
return err
}
*b, err = base64.StdEncoding.DecodeString(intermediate)
return err
}

View File

@@ -1,293 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"fmt"
"strconv"
"strings"
"sync"
"github.com/davecgh/go-spew/spew"
"github.com/go-ldap/ldap/v3"
"github.com/versity/versitygw/debuglogger"
)
type LdapIAMService struct {
conn *ldap.Conn
queryBase string
objClasses []string
accessAtr string
secretAtr string
roleAtr string
groupIdAtr string
userIdAtr string
rootAcc Account
url string
bindDN string
pass string
mu sync.Mutex
}
var _ IAMService = &LdapIAMService{}
func NewLDAPService(rootAcc Account, url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, userIdAtr, groupIdAtr, objClasses string) (IAMService, error) {
if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" ||
secAtr == "" || roleAtr == "" || userIdAtr == "" || groupIdAtr == "" || objClasses == "" {
return nil, fmt.Errorf("required parameters list not fully provided")
}
conn, err := ldap.DialURL(url)
if err != nil {
return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
}
err = conn.Bind(bindDN, pass)
if err != nil {
return nil, fmt.Errorf("failed to bind to LDAP server %w", err)
}
return &LdapIAMService{
conn: conn,
queryBase: queryBase,
objClasses: strings.Split(objClasses, ","),
accessAtr: accAtr,
secretAtr: secAtr,
roleAtr: roleAtr,
userIdAtr: userIdAtr,
groupIdAtr: groupIdAtr,
rootAcc: rootAcc,
url: url,
bindDN: bindDN,
pass: pass,
}, nil
}
func (ld *LdapIAMService) reconnect() error {
ld.conn.Close()
conn, err := ldap.DialURL(ld.url)
if err != nil {
return fmt.Errorf("failed to reconnect to LDAP server: %w", err)
}
err = conn.Bind(ld.bindDN, ld.pass)
if err != nil {
conn.Close()
return fmt.Errorf("failed to bind to LDAP server on reconnect: %w", err)
}
ld.conn = conn
return nil
}
func (ld *LdapIAMService) execute(f func(*ldap.Conn) error) error {
ld.mu.Lock()
defer ld.mu.Unlock()
err := f(ld.conn)
if err != nil {
if e, ok := err.(*ldap.Error); ok && e.ResultCode == ldap.ErrorNetwork {
if reconnErr := ld.reconnect(); reconnErr != nil {
return reconnErr
}
return f(ld.conn)
}
}
return err
}
func (ld *LdapIAMService) CreateAccount(account Account) error {
if ld.rootAcc.Access == account.Access {
return ErrUserExists
}
userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v,%v", ld.accessAtr, account.Access, ld.queryBase), nil)
userEntry.Attribute("objectClass", ld.objClasses)
userEntry.Attribute(ld.accessAtr, []string{account.Access})
userEntry.Attribute(ld.secretAtr, []string{account.Secret})
userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
userEntry.Attribute(ld.groupIdAtr, []string{fmt.Sprint(account.GroupID)})
userEntry.Attribute(ld.userIdAtr, []string{fmt.Sprint(account.UserID)})
err := ld.execute(func(c *ldap.Conn) error {
return c.Add(userEntry)
})
if err != nil {
return fmt.Errorf("error adding an entry: %w", err)
}
return nil
}
func (ld *LdapIAMService) buildSearchFilter(access string) string {
var searchFilter strings.Builder
for _, el := range ld.objClasses {
searchFilter.WriteString(fmt.Sprintf("(objectClass=%v)", el))
}
if access != "" {
searchFilter.WriteString(fmt.Sprintf("(%v=%v)", ld.accessAtr, access))
}
return fmt.Sprintf("(&%v)", searchFilter.String())
}
func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
if access == ld.rootAcc.Access {
return ld.rootAcc, nil
}
var result *ldap.SearchResult
searchRequest := ldap.NewSearchRequest(
ld.queryBase,
ldap.ScopeWholeSubtree,
ldap.NeverDerefAliases,
0,
0,
false,
ld.buildSearchFilter(access),
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.userIdAtr, ld.groupIdAtr},
nil,
)
if debuglogger.IsIAMDebugEnabled() {
debuglogger.IAMLogf("LDAP Search Request")
debuglogger.IAMLogf(spew.Sdump(searchRequest))
}
err := ld.execute(func(c *ldap.Conn) error {
var err error
result, err = c.Search(searchRequest)
return err
})
if debuglogger.IsIAMDebugEnabled() {
debuglogger.IAMLogf("LDAP Search Result")
debuglogger.IAMLogf(spew.Sdump(result))
}
if err != nil {
return Account{}, err
}
if len(result.Entries) == 0 {
return Account{}, ErrNoSuchUser
}
entry := result.Entries[0]
groupId, err := strconv.Atoi(entry.GetAttributeValue(ld.groupIdAtr))
if err != nil {
return Account{}, fmt.Errorf("invalid entry value for group-id %q: %w",
entry.GetAttributeValue(ld.groupIdAtr), err)
}
userId, err := strconv.Atoi(entry.GetAttributeValue(ld.userIdAtr))
if err != nil {
return Account{}, fmt.Errorf("invalid entry value for user-id %q: %w",
entry.GetAttributeValue(ld.userIdAtr), err)
}
return Account{
Access: entry.GetAttributeValue(ld.accessAtr),
Secret: entry.GetAttributeValue(ld.secretAtr),
Role: Role(entry.GetAttributeValue(ld.roleAtr)),
GroupID: groupId,
UserID: userId,
}, nil
}
func (ld *LdapIAMService) UpdateUserAccount(access string, props MutableProps) error {
req := ldap.NewModifyRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
if props.Secret != nil {
req.Replace(ld.secretAtr, []string{*props.Secret})
}
if props.GroupID != nil {
req.Replace(ld.groupIdAtr, []string{fmt.Sprint(*props.GroupID)})
}
if props.UserID != nil {
req.Replace(ld.userIdAtr, []string{fmt.Sprint(*props.UserID)})
}
if props.Role != "" {
req.Replace(ld.roleAtr, []string{string(props.Role)})
}
err := ld.execute(func(c *ldap.Conn) error {
return c.Modify(req)
})
//TODO: Handle non existing user case
if err != nil {
return err
}
return nil
}
func (ld *LdapIAMService) DeleteUserAccount(access string) error {
delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
err := ld.execute(func(c *ldap.Conn) error {
return c.Del(delReq)
})
if err != nil {
return err
}
return nil
}
func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
var resp *ldap.SearchResult
searchRequest := ldap.NewSearchRequest(
ld.queryBase,
ldap.ScopeWholeSubtree,
ldap.NeverDerefAliases,
0,
0,
false,
ld.buildSearchFilter(""),
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr, ld.groupIdAtr, ld.userIdAtr},
nil,
)
err := ld.execute(func(c *ldap.Conn) error {
var err error
resp, err = c.Search(searchRequest)
return err
})
if err != nil {
return nil, err
}
result := []Account{}
for _, el := range resp.Entries {
groupId, err := strconv.Atoi(el.GetAttributeValue(ld.groupIdAtr))
if err != nil {
return nil, fmt.Errorf("invalid entry value for group-id %q: %w",
el.GetAttributeValue(ld.groupIdAtr), err)
}
userId, err := strconv.Atoi(el.GetAttributeValue(ld.userIdAtr))
if err != nil {
return nil, fmt.Errorf("invalid entry value for user-id %q: %w",
el.GetAttributeValue(ld.userIdAtr), err)
}
result = append(result, Account{
Access: el.GetAttributeValue(ld.accessAtr),
Secret: el.GetAttributeValue(ld.secretAtr),
Role: Role(el.GetAttributeValue(ld.roleAtr)),
GroupID: groupId,
UserID: userId,
})
}
return result, nil
}
// Shutdown graceful termination of service
func (ld *LdapIAMService) Shutdown() error {
ld.mu.Lock()
defer ld.mu.Unlock()
return ld.conn.Close()
}

View File

@@ -1,56 +0,0 @@
package auth
import "testing"
func TestLdapIAMService_BuildSearchFilter(t *testing.T) {
tests := []struct {
name string
objClasses []string
accessAtr string
access string
expected string
}{
{
name: "single object class with access",
objClasses: []string{"inetOrgPerson"},
accessAtr: "uid",
access: "testuser",
expected: "(&(objectClass=inetOrgPerson)(uid=testuser))",
},
{
name: "single object class without access",
objClasses: []string{"inetOrgPerson"},
accessAtr: "uid",
access: "",
expected: "(&(objectClass=inetOrgPerson))",
},
{
name: "multiple object classes with access",
objClasses: []string{"inetOrgPerson", "organizationalPerson"},
accessAtr: "cn",
access: "john.doe",
expected: "(&(objectClass=inetOrgPerson)(objectClass=organizationalPerson)(cn=john.doe))",
},
{
name: "multiple object classes without access",
objClasses: []string{"inetOrgPerson", "organizationalPerson", "person"},
accessAtr: "cn",
access: "",
expected: "(&(objectClass=inetOrgPerson)(objectClass=organizationalPerson)(objectClass=person))",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ld := &LdapIAMService{
objClasses: tt.objClasses,
accessAtr: tt.accessAtr,
}
result := ld.buildSearchFilter(tt.access)
if result != tt.expected {
t.Errorf("BuildSearchFilter() = %v, want %v", result, tt.expected)
}
})
}
}

View File

@@ -1,304 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sort"
"sync"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/versity/versitygw/debuglogger"
)
// IAMServiceS3 stores user accounts in an S3 object
// The endpoint, credentials, bucket, and region are provided
// from cli configuration.
// The object format and name is the same as the internal IAM service:
// coming from iAMConfig and iamFile in iam_internal.
type IAMServiceS3 struct {
// This mutex will help with racing updates to the IAM data
// from multiple requests to this gateway instance, but
// will not help with racing updates to multiple load balanced
// gateway instances. This is a limitation of the internal
// IAM service. All account updates should be sent to a single
// gateway instance if possible.
sync.RWMutex
access string
secret string
region string
bucket string
endpoint string
sslSkipVerify bool
rootAcc Account
client *s3.Client
}
var _ IAMService = &IAMServiceS3{}
func NewS3(rootAcc Account, access, secret, region, bucket, endpoint string, sslSkipVerify bool) (*IAMServiceS3, error) {
if access == "" {
return nil, fmt.Errorf("must provide s3 IAM service access key")
}
if secret == "" {
return nil, fmt.Errorf("must provide s3 IAM service secret key")
}
if region == "" {
return nil, fmt.Errorf("must provide s3 IAM service region")
}
if bucket == "" {
return nil, fmt.Errorf("must provide s3 IAM service bucket")
}
if endpoint == "" {
return nil, fmt.Errorf("must provide s3 IAM service endpoint")
}
i := &IAMServiceS3{
access: access,
secret: secret,
region: region,
bucket: bucket,
endpoint: endpoint,
sslSkipVerify: sslSkipVerify,
rootAcc: rootAcc,
}
cfg, err := i.getConfig()
if err != nil {
return nil, fmt.Errorf("init s3 IAM: %v", err)
}
if endpoint != "" {
i.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = &endpoint
})
return i, nil
}
i.client = s3.NewFromConfig(cfg)
return i, nil
}
func (s *IAMServiceS3) CreateAccount(account Account) error {
if s.rootAcc.Access == account.Access {
return ErrUserExists
}
s.Lock()
defer s.Unlock()
conf, err := s.getAccounts()
if err != nil {
return err
}
_, ok := conf.AccessAccounts[account.Access]
if ok {
return ErrUserExists
}
conf.AccessAccounts[account.Access] = account
return s.storeAccts(conf)
}
func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
if access == s.rootAcc.Access {
return s.rootAcc, nil
}
s.RLock()
defer s.RUnlock()
conf, err := s.getAccounts()
if err != nil {
return Account{}, err
}
acct, ok := conf.AccessAccounts[access]
if !ok {
return Account{}, ErrNoSuchUser
}
return acct, nil
}
func (s *IAMServiceS3) UpdateUserAccount(access string, props MutableProps) error {
s.Lock()
defer s.Unlock()
conf, err := s.getAccounts()
if err != nil {
return err
}
acc, ok := conf.AccessAccounts[access]
if !ok {
return ErrNoSuchUser
}
updateAcc(&acc, props)
conf.AccessAccounts[access] = acc
return s.storeAccts(conf)
}
func (s *IAMServiceS3) DeleteUserAccount(access string) error {
s.Lock()
defer s.Unlock()
conf, err := s.getAccounts()
if err != nil {
return err
}
_, ok := conf.AccessAccounts[access]
if !ok {
return fmt.Errorf("account does not exist")
}
delete(conf.AccessAccounts, access)
return s.storeAccts(conf)
}
func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
s.RLock()
defer s.RUnlock()
conf, err := s.getAccounts()
if err != nil {
return nil, err
}
keys := make([]string, 0, len(conf.AccessAccounts))
for k := range conf.AccessAccounts {
keys = append(keys, k)
}
sort.Strings(keys)
var accs []Account
for _, k := range keys {
accs = append(accs, Account{
Access: k,
Secret: conf.AccessAccounts[k].Secret,
Role: conf.AccessAccounts[k].Role,
UserID: conf.AccessAccounts[k].UserID,
GroupID: conf.AccessAccounts[k].GroupID,
})
}
return accs, nil
}
func (s *IAMServiceS3) Shutdown() error {
return nil
}
func (s *IAMServiceS3) getConfig() (aws.Config, error) {
creds := credentials.NewStaticCredentialsProvider(s.access, s.secret, "")
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
}
client := &http.Client{Transport: tr}
opts := []func(*config.LoadOptions) error{
config.WithRegion(s.region),
config.WithCredentialsProvider(creds),
config.WithHTTPClient(client),
}
if debuglogger.IsIAMDebugEnabled() {
opts = append(opts,
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
}
return config.LoadDefaultConfig(context.Background(), opts...)
}
func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
obj := iamFile
out, err := s.client.GetObject(context.Background(), &s3.GetObjectInput{
Bucket: &s.bucket,
Key: &obj,
})
if err != nil {
// if the error is object not exists,
// init empty accounts struct and return that
var nsk *types.NoSuchKey
if errors.As(err, &nsk) {
return iAMConfig{AccessAccounts: map[string]Account{}}, nil
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
if apiErr.ErrorCode() == "NotFound" {
return iAMConfig{AccessAccounts: map[string]Account{}}, nil
}
}
// all other errors, return the error
return iAMConfig{}, fmt.Errorf("get %v: %w", obj, err)
}
defer out.Body.Close()
b, err := io.ReadAll(out.Body)
if err != nil {
return iAMConfig{}, fmt.Errorf("read %v: %w", obj, err)
}
conf, err := parseIAM(b)
if err != nil {
return iAMConfig{}, fmt.Errorf("parse iam data: %w", err)
}
return conf, nil
}
func (s *IAMServiceS3) storeAccts(conf iAMConfig) error {
b, err := json.Marshal(conf)
if err != nil {
return fmt.Errorf("failed to serialize iam: %w", err)
}
obj := iamFile
uploader := manager.NewUploader(s.client)
upinfo := &s3.PutObjectInput{
Body: bytes.NewReader(b),
Bucket: &s.bucket,
Key: &obj,
}
_, err = uploader.Upload(context.Background(), upinfo)
if err != nil {
return fmt.Errorf("store accounts in %v: %w", iamFile, err)
}
return nil
}

View File

@@ -1,66 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"github.com/versity/versitygw/s3err"
)
// IAMServiceSingle manages the single tenant (root-only) IAM service
type IAMServiceSingle struct {
root Account
}
var _ IAMService = &IAMServiceSingle{}
func NewIAMServiceSingle(r Account) IAMService {
return &IAMServiceSingle{
root: r,
}
}
// CreateAccount not valid in single tenant mode
func (IAMServiceSingle) CreateAccount(account Account) error {
return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
}
// GetUserAccount returns root account, if the root access key
// is provided and "ErrAdminUserNotFound" otherwise
func (s IAMServiceSingle) GetUserAccount(access string) (Account, error) {
if access == s.root.Access {
return s.root, nil
}
return Account{}, s3err.GetAPIError(s3err.ErrAdminUserNotFound)
}
// UpdateUserAccount no accounts in single tenant mode
func (IAMServiceSingle) UpdateUserAccount(access string, props MutableProps) error {
return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
}
// DeleteUserAccount no accounts in single tenant mode
func (IAMServiceSingle) DeleteUserAccount(access string) error {
return s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
}
// ListUserAccounts no accounts in single tenant mode
func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
return []Account{}, s3err.GetAPIError(s3err.ErrAdminMethodNotSupported)
}
// Shutdown graceful termination of service
func (IAMServiceSingle) Shutdown() error {
return nil
}

View File

@@ -1,330 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"time"
vault "github.com/hashicorp/vault-client-go"
"github.com/hashicorp/vault-client-go/schema"
)
const requestTimeout = 10 * time.Second
type VaultIAMService struct {
client *vault.Client
authReqOpts []vault.RequestOption
kvReqOpts []vault.RequestOption
secretStoragePath string
rootAcc Account
creds schema.AppRoleLoginRequest
}
var _ IAMService = &VaultIAMService{}
func NewVaultIAMService(rootAcc Account, endpoint, secretStoragePath,
authMethod, mountPath, rootToken, roleID, roleSecret, serverCert,
clientCert, clientCertKey string) (IAMService, error) {
opts := []vault.ClientOption{
vault.WithAddress(endpoint),
vault.WithRequestTimeout(requestTimeout),
}
if serverCert != "" {
tls := vault.TLSConfiguration{}
tls.ServerCertificate.FromBytes = []byte(serverCert)
if clientCert != "" {
if clientCertKey == "" {
return nil, fmt.Errorf("client certificate and client certificate key should both be specified")
}
tls.ClientCertificate.FromBytes = []byte(clientCert)
tls.ClientCertificateKey.FromBytes = []byte(clientCertKey)
}
opts = append(opts, vault.WithTLS(tls))
}
client, err := vault.New(opts...)
if err != nil {
return nil, fmt.Errorf("init vault client: %w", err)
}
authReqOpts := []vault.RequestOption{}
// if auth method path is not specified, it defaults to "approle"
if authMethod != "" {
authReqOpts = append(authReqOpts, vault.WithMountPath(authMethod))
}
kvReqOpts := []vault.RequestOption{}
// if mount path is not specified, it defaults to "kv-v2"
if mountPath != "" {
kvReqOpts = append(kvReqOpts, vault.WithMountPath(mountPath))
}
creds := schema.AppRoleLoginRequest{
RoleId: roleID,
SecretId: roleSecret,
}
// Authentication
switch {
case rootToken != "":
err := client.SetToken(rootToken)
if err != nil {
return nil, fmt.Errorf("root token authentication failure: %w", err)
}
case roleID != "":
if roleSecret == "" {
return nil, fmt.Errorf("role id and role secret must both be specified")
}
resp, err := client.Auth.AppRoleLogin(context.Background(),
creds, authReqOpts...)
if err != nil {
return nil, fmt.Errorf("approle authentication failure: %w", err)
}
if err := client.SetToken(resp.Auth.ClientToken); err != nil {
return nil, fmt.Errorf("approle authentication set token failure: %w", err)
}
default:
return nil, fmt.Errorf("vault authentication requires either roleid/rolesecret or root token")
}
return &VaultIAMService{
client: client,
authReqOpts: authReqOpts,
kvReqOpts: kvReqOpts,
secretStoragePath: secretStoragePath,
rootAcc: rootAcc,
creds: creds,
}, nil
}
func (vt *VaultIAMService) reAuthIfNeeded(err error) error {
if err == nil {
return nil
}
// Vault returns 403 for expired/revoked tokens
// pass all other errors back unchanged
if !vault.IsErrorStatus(err, http.StatusForbidden) {
return err
}
resp, authErr := vt.client.Auth.AppRoleLogin(context.Background(),
vt.creds, vt.authReqOpts...)
if authErr != nil {
return fmt.Errorf("vault re-authentication failure: %w", authErr)
}
if err := vt.client.SetToken(resp.Auth.ClientToken); err != nil {
return fmt.Errorf("vault re-authentication set token failure: %w", err)
}
return nil
}
func (vt *VaultIAMService) CreateAccount(account Account) error {
if vt.rootAcc.Access == account.Access {
return ErrUserExists
}
_, err := vt.client.Secrets.KvV2Write(context.Background(),
vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
Data: map[string]any{
account.Access: account,
},
Options: map[string]any{
"cas": 0,
},
}, vt.kvReqOpts...)
if err != nil {
if strings.Contains(err.Error(), "check-and-set") {
return ErrUserExists
}
reauthErr := vt.reAuthIfNeeded(err)
if reauthErr != nil {
return reauthErr
}
// retry once after re-auth
_, err = vt.client.Secrets.KvV2Write(context.Background(),
vt.secretStoragePath+"/"+account.Access, schema.KvV2WriteRequest{
Data: map[string]any{
account.Access: account,
},
Options: map[string]any{
"cas": 0,
},
}, vt.kvReqOpts...)
if err != nil {
if strings.Contains(err.Error(), "check-and-set") {
return ErrUserExists
}
return err
}
return nil
}
return nil
}
func (vt *VaultIAMService) GetUserAccount(access string) (Account, error) {
if vt.rootAcc.Access == access {
return vt.rootAcc, nil
}
resp, err := vt.client.Secrets.KvV2Read(context.Background(),
vt.secretStoragePath+"/"+access, vt.kvReqOpts...)
if err != nil {
reauthErr := vt.reAuthIfNeeded(err)
if reauthErr != nil {
return Account{}, reauthErr
}
// retry once after re-auth
resp, err = vt.client.Secrets.KvV2Read(context.Background(),
vt.secretStoragePath+"/"+access, vt.kvReqOpts...)
if err != nil {
return Account{}, err
}
}
acc, err := parseVaultUserAccount(resp.Data.Data, access)
if err != nil {
return Account{}, err
}
return acc, nil
}
func (vt *VaultIAMService) UpdateUserAccount(access string, props MutableProps) error {
acc, err := vt.GetUserAccount(access)
if err != nil {
return err
}
updateAcc(&acc, props)
err = vt.DeleteUserAccount(access)
if err != nil {
return err
}
err = vt.CreateAccount(acc)
if err != nil {
return err
}
return nil
}
func (vt *VaultIAMService) DeleteUserAccount(access string) error {
_, err := vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(context.Background(),
vt.secretStoragePath+"/"+access, vt.kvReqOpts...)
if err != nil {
reauthErr := vt.reAuthIfNeeded(err)
if reauthErr != nil {
return reauthErr
}
// retry once after re-auth
_, err = vt.client.Secrets.KvV2DeleteMetadataAndAllVersions(context.Background(),
vt.secretStoragePath+"/"+access, vt.kvReqOpts...)
if err != nil {
return err
}
}
return nil
}
func (vt *VaultIAMService) ListUserAccounts() ([]Account, error) {
resp, err := vt.client.Secrets.KvV2List(context.Background(),
vt.secretStoragePath, vt.kvReqOpts...)
if err != nil {
reauthErr := vt.reAuthIfNeeded(err)
if reauthErr != nil {
if vault.IsErrorStatus(err, http.StatusNotFound) {
return []Account{}, nil
}
return nil, reauthErr
}
// retry once after re-auth
resp, err = vt.client.Secrets.KvV2List(context.Background(),
vt.secretStoragePath, vt.kvReqOpts...)
if err != nil {
if vault.IsErrorStatus(err, http.StatusNotFound) {
return []Account{}, nil
}
return nil, err
}
}
accs := []Account{}
for _, acss := range resp.Data.Keys {
acc, err := vt.GetUserAccount(acss)
if err != nil {
return nil, err
}
accs = append(accs, acc)
}
return accs, nil
}
// the client doesn't have explicit shutdown, as it uses http.Client
func (vt *VaultIAMService) Shutdown() error {
return nil
}
var errInvalidUser error = errors.New("invalid user account entry in secrets engine")
func parseVaultUserAccount(data map[string]any, access string) (acc Account, err error) {
usrAcc, ok := data[access].(map[string]any)
if !ok {
return acc, errInvalidUser
}
acss, ok := usrAcc["access"].(string)
if !ok {
return acc, errInvalidUser
}
secret, ok := usrAcc["secret"].(string)
if !ok {
return acc, errInvalidUser
}
role, ok := usrAcc["role"].(string)
if !ok {
return acc, errInvalidUser
}
userIdJson, ok := usrAcc["userID"].(json.Number)
if !ok {
return acc, errInvalidUser
}
userId, err := userIdJson.Int64()
if err != nil {
return acc, errInvalidUser
}
groupIdJson, ok := usrAcc["groupID"].(json.Number)
if !ok {
return acc, errInvalidUser
}
groupId, err := groupIdJson.Int64()
if err != nil {
return acc, errInvalidUser
}
return Account{
Access: acss,
Secret: secret,
Role: Role(role),
UserID: int(userId),
GroupID: int(groupId),
}, nil
}

View File

@@ -1,351 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"context"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/debuglogger"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
type BucketLockConfig struct {
Enabled bool
DefaultRetention *types.DefaultRetention
CreatedAt *time.Time
}
func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
var lockConfig types.ObjectLockConfiguration
if err := xml.Unmarshal(input, &lockConfig); err != nil {
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
if lockConfig.ObjectLockEnabled != "" && lockConfig.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
config := BucketLockConfig{
Enabled: lockConfig.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
}
if lockConfig.Rule != nil && lockConfig.Rule.DefaultRetention != nil {
retention := lockConfig.Rule.DefaultRetention
if retention.Mode != types.ObjectLockRetentionModeCompliance && retention.Mode != types.ObjectLockRetentionModeGovernance {
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
if retention.Years != nil && retention.Days != nil {
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
if retention.Days != nil && *retention.Days <= 0 {
return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
}
if retention.Years != nil && *retention.Years <= 0 {
return nil, s3err.GetAPIError(s3err.ErrObjectLockInvalidRetentionPeriod)
}
config.DefaultRetention = retention
now := time.Now()
config.CreatedAt = &now
}
return json.Marshal(config)
}
func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfiguration, error) {
var config BucketLockConfig
if err := json.Unmarshal(input, &config); err != nil {
return nil, fmt.Errorf("parse object lock config: %w", err)
}
result := &types.ObjectLockConfiguration{
Rule: &types.ObjectLockRule{
DefaultRetention: config.DefaultRetention,
},
}
if config.Enabled {
result.ObjectLockEnabled = types.ObjectLockEnabledEnabled
}
return result, nil
}
func ParseObjectLockRetentionInput(input []byte) (*s3response.PutObjectRetentionInput, error) {
var retention s3response.PutObjectRetentionInput
if err := xml.Unmarshal(input, &retention); err != nil {
debuglogger.Logf("invalid object lock retention request body: %v", err)
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
if retention.RetainUntilDate.Before(time.Now()) {
debuglogger.Logf("object lock retain until date must be in the future")
return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
}
switch retention.Mode {
case types.ObjectLockRetentionModeCompliance:
case types.ObjectLockRetentionModeGovernance:
default:
debuglogger.Logf("invalid object lock retention mode: %s", retention.Mode)
return nil, s3err.GetAPIError(s3err.ErrMalformedXML)
}
return &retention, nil
}
func ParseObjectLockRetentionInputToJSON(input *s3response.PutObjectRetentionInput) ([]byte, error) {
data, err := json.Marshal(input)
if err != nil {
debuglogger.Logf("parse object lock retention to JSON: %v", err)
return nil, fmt.Errorf("parse object lock retention: %w", err)
}
return data, nil
}
// IsObjectLockRetentionPutAllowed checks if the object lock retention PUT request
// is allowed against the current state of the object lock
func IsObjectLockRetentionPutAllowed(ctx context.Context, be backend.Backend, bucket, object, versionId, userAccess string, input *s3response.PutObjectRetentionInput, bypass bool) error {
ret, err := be.GetObjectRetention(ctx, bucket, object, versionId)
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
// if object lock configuration is not set
// allow the retention modification without any checks
return nil
}
if err != nil {
debuglogger.Logf("failed to get object retention: %v", err)
return err
}
retention, err := ParseObjectLockRetentionOutput(ret)
if err != nil {
return err
}
if retention.Mode == input.Mode {
// if retention mode is the same
// the operation is allowed
return nil
}
if retention.Mode == types.ObjectLockRetentionModeCompliance {
// COMPLIANCE mode is by definition not allowed to modify
debuglogger.Logf("object lock retention change request from 'COMPLIANCE' to 'GOVERNANCE' is not allowed")
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
if !bypass {
// if x-amz-bypass-governance-retention is not provided
// return error: object is locked
debuglogger.Logf("object lock retention mode change is not allowed and bypass governence is not forced")
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
// the last case left, when user tries to chenge
// from 'GOVERNANCE' to 'COMPLIANCE' with
// 'x-amz-bypass-governance-retention' header
// first we need to check if user has 's3:BypassGovernanceRetention'
policy, err := be.GetBucketPolicy(ctx, bucket)
if err != nil {
// if it fails to get the policy, return object is locked
debuglogger.Logf("failed to get the bucket policy: %v", err)
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
err = VerifyBucketPolicy(policy, userAccess, bucket, object, BypassGovernanceRetentionAction)
if err != nil {
// if user doesn't have "s3:BypassGovernanceRetention" permission
// return object is locked
debuglogger.Logf("the user is missing 's3:BypassGovernanceRetention' permission")
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
return nil
}
func ParseObjectLockRetentionOutput(input []byte) (*types.ObjectLockRetention, error) {
var retention types.ObjectLockRetention
if err := json.Unmarshal(input, &retention); err != nil {
debuglogger.Logf("parse object lock retention output: %v", err)
return nil, fmt.Errorf("parse object lock retention: %w", err)
}
return &retention, nil
}
func ParseObjectLegalHoldOutput(status *bool) *s3response.GetObjectLegalHoldResult {
if status == nil {
return nil
}
if *status {
return &s3response.GetObjectLegalHoldResult{
Status: types.ObjectLockLegalHoldStatusOn,
}
}
return &s3response.GetObjectLegalHoldResult{
Status: types.ObjectLockLegalHoldStatusOff,
}
}
func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []types.ObjectIdentifier, bypass, isBucketPublic bool, be backend.Backend) error {
data, err := be.GetObjectLockConfiguration(ctx, bucket)
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
return nil
}
return err
}
var bucketLockConfig BucketLockConfig
if err := json.Unmarshal(data, &bucketLockConfig); err != nil {
return fmt.Errorf("parse object lock config: %w", err)
}
if !bucketLockConfig.Enabled {
return nil
}
checkDefaultRetention := false
if bucketLockConfig.DefaultRetention != nil && bucketLockConfig.CreatedAt != nil {
expirationDate := *bucketLockConfig.CreatedAt
if bucketLockConfig.DefaultRetention.Days != nil {
expirationDate = expirationDate.AddDate(0, 0, int(*bucketLockConfig.DefaultRetention.Days))
}
if bucketLockConfig.DefaultRetention.Years != nil {
expirationDate = expirationDate.AddDate(int(*bucketLockConfig.DefaultRetention.Years), 0, 0)
}
if expirationDate.After(time.Now()) {
checkDefaultRetention = true
}
}
for _, obj := range objects {
var key, versionId string
if obj.Key != nil {
key = *obj.Key
}
if obj.VersionId != nil {
versionId = *obj.VersionId
}
checkRetention := true
retentionData, err := be.GetObjectRetention(ctx, bucket, key, versionId)
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
continue
}
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
checkRetention = false
}
if err != nil && checkRetention {
return err
}
if checkRetention {
retention, err := ParseObjectLockRetentionOutput(retentionData)
if err != nil {
return err
}
if retention.Mode != "" && retention.RetainUntilDate != nil {
if retention.RetainUntilDate.After(time.Now()) {
switch retention.Mode {
case types.ObjectLockRetentionModeGovernance:
if !bypass {
return s3err.GetAPIError(s3err.ErrObjectLocked)
} else {
policy, err := be.GetBucketPolicy(ctx, bucket)
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
if err != nil {
return err
}
if isBucketPublic {
err = VerifyPublicBucketPolicy(policy, bucket, key, BypassGovernanceRetentionAction)
} else {
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
}
if err != nil {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
}
case types.ObjectLockRetentionModeCompliance:
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
}
}
}
checkLegalHold := true
status, err := be.GetObjectLegalHold(ctx, bucket, key, versionId)
if err != nil {
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
continue
}
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
checkLegalHold = false
} else {
return err
}
}
if checkLegalHold && *status {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
if checkDefaultRetention {
switch bucketLockConfig.DefaultRetention.Mode {
case types.ObjectLockRetentionModeGovernance:
if !bypass {
return s3err.GetAPIError(s3err.ErrObjectLocked)
} else {
policy, err := be.GetBucketPolicy(ctx, bucket)
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
if err != nil {
return err
}
if isBucketPublic {
err = VerifyPublicBucketPolicy(policy, bucket, key, BypassGovernanceRetentionAction)
} else {
err = VerifyBucketPolicy(policy, userAccess, bucket, key, BypassGovernanceRetentionAction)
}
if err != nil {
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
}
case types.ObjectLockRetentionModeCompliance:
return s3err.GetAPIError(s3err.ErrObjectLocked)
}
}
}
return nil
}

View File

@@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@@ -1,4 +0,0 @@
AWS SDK for Go
Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
Copyright 2014-2015 Stripe, Inc.
Copyright 2024 Versity Software

View File

@@ -1,11 +0,0 @@
# AWS SDK Go v2
This directory contains code from the [AWS SDK Go v2](https://github.com/aws/aws-sdk-go-v2) repository, modified in accordance with the Apache 2.0 License.
## Description
The AWS SDK Go v2 is a collection of libraries and tools that enable developers to build applications that integrate with various AWS services. This directory and below contains modified code from the original repository, tailored to suit versitygw specific requirements.
## License
The code in this directory is licensed under the Apache 2.0 License. Please refer to the [LICENSE](./LICENSE) file for more information.

View File

@@ -1,61 +0,0 @@
// Package unit performs initialization and validation for unit tests
package unit
import (
"context"
"crypto/rsa"
"math/big"
"github.com/aws/aws-sdk-go-v2/aws"
)
func init() {
config = aws.Config{}
config.Region = "mock-region"
config.Credentials = StubCredentialsProvider{}
}
// StubCredentialsProvider provides a stub credential provider that returns
// static credentials that never expire.
type StubCredentialsProvider struct{}
// Retrieve satisfies the CredentialsProvider interface. Returns stub
// credential value, and never error.
func (StubCredentialsProvider) Retrieve(context.Context) (aws.Credentials, error) {
return aws.Credentials{
AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION",
Source: "unit test credentials",
}, nil
}
var config aws.Config
// Config returns a copy of the mock configuration for unit tests.
func Config() aws.Config { return config.Copy() }
// RSAPrivateKey is used for testing functionality that requires some
// sort of private key. Taken from crypto/rsa/rsa_test.go
//
// Credit to golang 1.11
var RSAPrivateKey = &rsa.PrivateKey{
PublicKey: rsa.PublicKey{
N: fromBase10("14314132931241006650998084889274020608918049032671858325988396851334124245188214251956198731333464217832226406088020736932173064754214329009979944037640912127943488972644697423190955557435910767690712778463524983667852819010259499695177313115447116110358524558307947613422897787329221478860907963827160223559690523660574329011927531289655711860504630573766609239332569210831325633840174683944553667352219670930408593321661375473885147973879086994006440025257225431977751512374815915392249179976902953721486040787792801849818254465486633791826766873076617116727073077821584676715609985777563958286637185868165868520557"),
E: 3,
},
D: fromBase10("9542755287494004433998723259516013739278699355114572217325597900889416163458809501304132487555642811888150937392013824621448709836142886006653296025093941418628992648429798282127303704957273845127141852309016655778568546006839666463451542076964744073572349705538631742281931858219480985907271975884773482372966847639853897890615456605598071088189838676728836833012254065983259638538107719766738032720239892094196108713378822882383694456030043492571063441943847195939549773271694647657549658603365629458610273821292232646334717612674519997533901052790334279661754176490593041941863932308687197618671528035670452762731"),
Primes: []*big.Int{
fromBase10("130903255182996722426771613606077755295583329135067340152947172868415809027537376306193179624298874215608270802054347609836776473930072411958753044562214537013874103802006369634761074377213995983876788718033850153719421695468704276694983032644416930879093914927146648402139231293035971427838068945045019075433"),
fromBase10("109348945610485453577574767652527472924289229538286649661240938988020367005475727988253438647560958573506159449538793540472829815903949343191091817779240101054552748665267574271163617694640513549693841337820602726596756351006149518830932261246698766355347898158548465400674856021497190430791824869615170301029"),
},
}
// Taken from crypto/rsa/rsa_test.go
//
// Credit to golang 1.11
func fromBase10(base10 string) *big.Int {
i, ok := new(big.Int).SetString(base10, 10)
if !ok {
panic("bad number: " + base10)
}
return i
}

View File

@@ -1,115 +0,0 @@
package v4
import (
"strings"
"sync"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
)
func lookupKey(service, region string) string {
var s strings.Builder
s.Grow(len(region) + len(service) + 3)
s.WriteString(region)
s.WriteRune('/')
s.WriteString(service)
return s.String()
}
type derivedKey struct {
AccessKey string
Date time.Time
Credential []byte
}
type derivedKeyCache struct {
values map[string]derivedKey
mutex sync.RWMutex
}
func newDerivedKeyCache() derivedKeyCache {
return derivedKeyCache{
values: make(map[string]derivedKey),
}
}
func (s *derivedKeyCache) Get(credentials aws.Credentials, service, region string, signingTime SigningTime) []byte {
key := lookupKey(service, region)
s.mutex.RLock()
if cred, ok := s.get(key, credentials, signingTime.Time); ok {
s.mutex.RUnlock()
return cred
}
s.mutex.RUnlock()
s.mutex.Lock()
if cred, ok := s.get(key, credentials, signingTime.Time); ok {
s.mutex.Unlock()
return cred
}
cred := deriveKey(credentials.SecretAccessKey, service, region, signingTime)
entry := derivedKey{
AccessKey: credentials.AccessKeyID,
Date: signingTime.Time,
Credential: cred,
}
s.values[key] = entry
s.mutex.Unlock()
return cred
}
func (s *derivedKeyCache) get(key string, credentials aws.Credentials, signingTime time.Time) ([]byte, bool) {
cacheEntry, ok := s.retrieveFromCache(key)
if ok && cacheEntry.AccessKey == credentials.AccessKeyID && isSameDay(signingTime, cacheEntry.Date) {
return cacheEntry.Credential, true
}
return nil, false
}
func (s *derivedKeyCache) retrieveFromCache(key string) (derivedKey, bool) {
if v, ok := s.values[key]; ok {
return v, true
}
return derivedKey{}, false
}
// SigningKeyDeriver derives a signing key from a set of credentials
type SigningKeyDeriver struct {
cache derivedKeyCache
}
// NewSigningKeyDeriver returns a new SigningKeyDeriver
func NewSigningKeyDeriver() *SigningKeyDeriver {
return &SigningKeyDeriver{
cache: newDerivedKeyCache(),
}
}
// DeriveKey returns a derived signing key from the given credentials to be used with SigV4 signing.
func (k *SigningKeyDeriver) DeriveKey(credential aws.Credentials, service, region string, signingTime SigningTime) []byte {
return k.cache.Get(credential, service, region, signingTime)
}
func deriveKey(secret, service, region string, t SigningTime) []byte {
hmacDate := HMACSHA256([]byte("AWS4"+secret), []byte(t.ShortTimeFormat()))
hmacRegion := HMACSHA256(hmacDate, []byte(region))
hmacService := HMACSHA256(hmacRegion, []byte(service))
return HMACSHA256(hmacService, []byte("aws4_request"))
}
func isSameDay(x, y time.Time) bool {
xYear, xMonth, xDay := x.Date()
yYear, yMonth, yDay := y.Date()
if xYear != yYear {
return false
}
if xMonth != yMonth {
return false
}
return xDay == yDay
}

View File

@@ -1,40 +0,0 @@
package v4
// Signature Version 4 (SigV4) Constants
const (
// EmptyStringSHA256 is the hex encoded sha256 value of an empty string
EmptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`
// UnsignedPayload indicates that the request payload body is unsigned
UnsignedPayload = "UNSIGNED-PAYLOAD"
// AmzAlgorithmKey indicates the signing algorithm
AmzAlgorithmKey = "X-Amz-Algorithm"
// AmzSecurityTokenKey indicates the security token to be used with temporary credentials
AmzSecurityTokenKey = "X-Amz-Security-Token"
// AmzDateKey is the UTC timestamp for the request in the format YYYYMMDD'T'HHMMSS'Z'
AmzDateKey = "X-Amz-Date"
// AmzCredentialKey is the access key ID and credential scope
AmzCredentialKey = "X-Amz-Credential"
// AmzSignedHeadersKey is the set of headers signed for the request
AmzSignedHeadersKey = "X-Amz-SignedHeaders"
// AmzSignatureKey is the query parameter to store the SigV4 signature
AmzSignatureKey = "X-Amz-Signature"
// TimeFormat is the time format to be used in the X-Amz-Date header or query parameter
TimeFormat = "20060102T150405Z"
// ShortTimeFormat is the shorten time format used in the credential scope
ShortTimeFormat = "20060102"
// ContentSHAKey is the SHA256 of request body
ContentSHAKey = "X-Amz-Content-Sha256"
// StreamingEventsPayload indicates that the request payload body is a signed event stream.
StreamingEventsPayload = "STREAMING-AWS4-HMAC-SHA256-EVENTS"
)

View File

@@ -1,88 +0,0 @@
package v4
import (
"strings"
)
// Rules houses a set of Rule needed for validation of a
// string value
type Rules []Rule
// Rule interface allows for more flexible rules and just simply
// checks whether or not a value adheres to that Rule
type Rule interface {
IsValid(value string) bool
}
// IsValid will iterate through all rules and see if any rules
// apply to the value and supports nested rules
func (r Rules) IsValid(value string) bool {
for _, rule := range r {
if rule.IsValid(value) {
return true
}
}
return false
}
// MapRule generic Rule for maps
type MapRule map[string]struct{}
// IsValid for the map Rule satisfies whether it exists in the map
func (m MapRule) IsValid(value string) bool {
_, ok := m[value]
return ok
}
// AllowList is a generic Rule for include listing
type AllowList struct {
Rule
}
// IsValid for AllowList checks if the value is within the AllowList
func (w AllowList) IsValid(value string) bool {
return w.Rule.IsValid(value)
}
// ExcludeList is a generic Rule for exclude listing
type ExcludeList struct {
Rule
}
// IsValid for AllowList checks if the value is within the AllowList
func (b ExcludeList) IsValid(value string) bool {
return !b.Rule.IsValid(value)
}
// Patterns is a list of strings to match against
type Patterns []string
// IsValid for Patterns checks each pattern and returns if a match has
// been found
func (p Patterns) IsValid(value string) bool {
for _, pattern := range p {
if hasPrefixFold(value, pattern) {
return true
}
}
return false
}
// InclusiveRules rules allow for rules to depend on one another
type InclusiveRules []Rule
// IsValid will return true if all rules are true
func (r InclusiveRules) IsValid(value string) bool {
for _, rule := range r {
if !rule.IsValid(value) {
return false
}
}
return true
}
// hasPrefixFold tests whether the string s begins with prefix, interpreted as UTF-8 strings,
// under Unicode case-folding.
func hasPrefixFold(s, prefix string) bool {
return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix)
}

View File

@@ -1,72 +0,0 @@
package v4
// IgnoredHeaders is a list of headers that are ignored during signing
var IgnoredHeaders = Rules{
ExcludeList{
MapRule{
"Authorization": struct{}{},
// some clients use user-agent in signed headers
// "User-Agent": struct{}{},
"X-Amzn-Trace-Id": struct{}{},
"Expect": struct{}{},
},
},
}
// RequiredSignedHeaders is a allow list for Build canonical headers.
var RequiredSignedHeaders = Rules{
AllowList{
MapRule{
"Cache-Control": struct{}{},
"Content-Disposition": struct{}{},
"Content-Encoding": struct{}{},
"Content-Language": struct{}{},
"Content-Md5": struct{}{},
"Content-Type": struct{}{},
"Expires": struct{}{},
"If-Match": struct{}{},
"If-Modified-Since": struct{}{},
"If-None-Match": struct{}{},
"If-Unmodified-Since": struct{}{},
"Range": struct{}{},
"X-Amz-Acl": struct{}{},
"X-Amz-Copy-Source": struct{}{},
"X-Amz-Copy-Source-If-Match": struct{}{},
"X-Amz-Copy-Source-If-Modified-Since": struct{}{},
"X-Amz-Copy-Source-If-None-Match": struct{}{},
"X-Amz-Copy-Source-If-Unmodified-Since": struct{}{},
"X-Amz-Copy-Source-Range": struct{}{},
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": struct{}{},
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": struct{}{},
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": struct{}{},
"X-Amz-Expected-Bucket-Owner": struct{}{},
"X-Amz-Grant-Full-control": struct{}{},
"X-Amz-Grant-Read": struct{}{},
"X-Amz-Grant-Read-Acp": struct{}{},
"X-Amz-Grant-Write": struct{}{},
"X-Amz-Grant-Write-Acp": struct{}{},
"X-Amz-Metadata-Directive": struct{}{},
"X-Amz-Mfa": struct{}{},
"X-Amz-Request-Payer": struct{}{},
"X-Amz-Server-Side-Encryption": struct{}{},
"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": struct{}{},
"X-Amz-Server-Side-Encryption-Context": struct{}{},
"X-Amz-Server-Side-Encryption-Customer-Algorithm": struct{}{},
"X-Amz-Server-Side-Encryption-Customer-Key": struct{}{},
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": struct{}{},
"X-Amz-Storage-Class": struct{}{},
"X-Amz-Website-Redirect-Location": struct{}{},
"X-Amz-Content-Sha256": struct{}{},
"X-Amz-Tagging": struct{}{},
},
},
Patterns{"X-Amz-Object-Lock-"},
Patterns{"X-Amz-Meta-"},
}
// AllowedQueryHoisting is a allowed list for Build query headers. The boolean value
// represents whether or not it is a pattern.
var AllowedQueryHoisting = InclusiveRules{
ExcludeList{RequiredSignedHeaders},
Patterns{"X-Amz-"},
}

View File

@@ -1,63 +0,0 @@
package v4
import "testing"
func TestAllowedQueryHoisting(t *testing.T) {
cases := map[string]struct {
Header string
ExpectHoist bool
}{
"object-lock": {
Header: "X-Amz-Object-Lock-Mode",
ExpectHoist: false,
},
"s3 metadata": {
Header: "X-Amz-Meta-SomeName",
ExpectHoist: false,
},
"another header": {
Header: "X-Amz-SomeOtherHeader",
ExpectHoist: true,
},
"non X-AMZ header": {
Header: "X-SomeOtherHeader",
ExpectHoist: false,
},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
if e, a := c.ExpectHoist, AllowedQueryHoisting.IsValid(c.Header); e != a {
t.Errorf("expect hoist %v, was %v", e, a)
}
})
}
}
func TestIgnoredHeaders(t *testing.T) {
cases := map[string]struct {
Header string
ExpectIgnored bool
}{
"expect": {
Header: "Expect",
ExpectIgnored: true,
},
"authorization": {
Header: "Authorization",
ExpectIgnored: true,
},
"X-AMZ header": {
Header: "X-Amz-Content-Sha256",
ExpectIgnored: false,
},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
if e, a := c.ExpectIgnored, IgnoredHeaders.IsValid(c.Header); e == a {
t.Errorf("expect ignored %v, was %v", e, a)
}
})
}
}

View File

@@ -1,13 +0,0 @@
package v4
import (
"crypto/hmac"
"crypto/sha256"
)
// HMACSHA256 computes a HMAC-SHA256 of data given the provided key.
func HMACSHA256(key []byte, data []byte) []byte {
hash := hmac.New(sha256.New, key)
hash.Write(data)
return hash.Sum(nil)
}

View File

@@ -1,75 +0,0 @@
package v4
import (
"net/http"
"strings"
)
// SanitizeHostForHeader removes default port from host and updates request.Host
func SanitizeHostForHeader(r *http.Request) {
host := getHost(r)
port := portOnly(host)
if port != "" && isDefaultPort(r.URL.Scheme, port) {
r.Host = stripPort(host)
}
}
// Returns host from request
func getHost(r *http.Request) string {
if r.Host != "" {
return r.Host
}
return r.URL.Host
}
// Hostname returns u.Host, without any port number.
//
// If Host is an IPv6 literal with a port number, Hostname returns the
// IPv6 literal without the square brackets. IPv6 literals may include
// a zone identifier.
//
// Copied from the Go 1.8 standard library (net/url)
func stripPort(hostport string) string {
colon := strings.IndexByte(hostport, ':')
if colon == -1 {
return hostport
}
if i := strings.IndexByte(hostport, ']'); i != -1 {
return strings.TrimPrefix(hostport[:i], "[")
}
return hostport[:colon]
}
// Port returns the port part of u.Host, without the leading colon.
// If u.Host doesn't contain a port, Port returns an empty string.
//
// Copied from the Go 1.8 standard library (net/url)
func portOnly(hostport string) string {
colon := strings.IndexByte(hostport, ':')
if colon == -1 {
return ""
}
if i := strings.Index(hostport, "]:"); i != -1 {
return hostport[i+len("]:"):]
}
if strings.Contains(hostport, "]") {
return ""
}
return hostport[colon+len(":"):]
}
// Returns true if the specified URI is using the standard port
// (i.e. port 80 for HTTP URIs or 443 for HTTPS URIs)
func isDefaultPort(scheme, port string) bool {
if port == "" {
return true
}
lowerCaseScheme := strings.ToLower(scheme)
if (lowerCaseScheme == "http" && port == "80") || (lowerCaseScheme == "https" && port == "443") {
return true
}
return false
}

View File

@@ -1,13 +0,0 @@
package v4
import "strings"
// BuildCredentialScope builds the Signature Version 4 (SigV4) signing scope
func BuildCredentialScope(signingTime SigningTime, region, service string) string {
return strings.Join([]string{
signingTime.ShortTimeFormat(),
region,
service,
"aws4_request",
}, "/")
}

View File

@@ -1,36 +0,0 @@
package v4
import "time"
// SigningTime provides a wrapper around a time.Time which provides cached values for SigV4 signing.
type SigningTime struct {
time.Time
timeFormat string
shortTimeFormat string
}
// NewSigningTime creates a new SigningTime given a time.Time
func NewSigningTime(t time.Time) SigningTime {
return SigningTime{
Time: t,
}
}
// TimeFormat provides a time formatted in the X-Amz-Date format.
func (m *SigningTime) TimeFormat() string {
return m.format(&m.timeFormat, TimeFormat)
}
// ShortTimeFormat provides a time formatted of 20060102.
func (m *SigningTime) ShortTimeFormat() string {
return m.format(&m.shortTimeFormat, ShortTimeFormat)
}
func (m *SigningTime) format(target *string, format string) string {
if len(*target) > 0 {
return *target
}
v := m.Time.Format(format)
*target = v
return v
}

View File

@@ -1,80 +0,0 @@
package v4
import (
"net/url"
"strings"
)
const doubleSpace = " "
// StripExcessSpaces will rewrite the passed in slice's string values to not
// contain multiple side-by-side spaces.
func StripExcessSpaces(str string) string {
var j, k, l, m, spaces int
// Trim trailing spaces
for j = len(str) - 1; j >= 0 && str[j] == ' '; j-- {
}
// Trim leading spaces
for k = 0; k < j && str[k] == ' '; k++ {
}
str = str[k : j+1]
// Strip multiple spaces.
j = strings.Index(str, doubleSpace)
if j < 0 {
return str
}
buf := []byte(str)
for k, m, l = j, j, len(buf); k < l; k++ {
if buf[k] == ' ' {
if spaces == 0 {
// First space.
buf[m] = buf[k]
m++
}
spaces++
} else {
// End of multiple spaces.
spaces = 0
buf[m] = buf[k]
m++
}
}
return string(buf[:m])
}
// GetURIPath returns the escaped URI component from the provided URL.
func GetURIPath(u *url.URL) string {
var uriPath string
if len(u.Opaque) > 0 {
const schemeSep, pathSep, queryStart = "//", "/", "?"
opaque := u.Opaque
// Cut off the query string if present.
if idx := strings.Index(opaque, queryStart); idx >= 0 {
opaque = opaque[:idx]
}
// Cutout the scheme separator if present.
if strings.Index(opaque, schemeSep) == 0 {
opaque = opaque[len(schemeSep):]
}
// capture URI path starting with first path separator.
if idx := strings.Index(opaque, pathSep); idx >= 0 {
uriPath = opaque[idx:]
}
} else {
uriPath = u.EscapedPath()
}
if len(uriPath) == 0 {
uriPath = "/"
}
return uriPath
}

View File

@@ -1,158 +0,0 @@
package v4
import (
"net/http"
"net/url"
"testing"
)
func lazyURLParse(v string) func() (*url.URL, error) {
return func() (*url.URL, error) {
return url.Parse(v)
}
}
func TestGetURIPath(t *testing.T) {
cases := map[string]struct {
getURL func() (*url.URL, error)
expect string
}{
// Cases
"with scheme": {
getURL: lazyURLParse("https://localhost:9000"),
expect: "/",
},
"no port, with scheme": {
getURL: lazyURLParse("https://localhost"),
expect: "/",
},
"without scheme": {
getURL: lazyURLParse("localhost:9000"),
expect: "/",
},
"without scheme, with path": {
getURL: lazyURLParse("localhost:9000/abc123"),
expect: "/abc123",
},
"without scheme, with separator": {
getURL: lazyURLParse("//localhost:9000"),
expect: "/",
},
"no port, without scheme, with separator": {
getURL: lazyURLParse("//localhost"),
expect: "/",
},
"without scheme, with separator, with path": {
getURL: lazyURLParse("//localhost:9000/abc123"),
expect: "/abc123",
},
"no port, without scheme, with separator, with path": {
getURL: lazyURLParse("//localhost/abc123"),
expect: "/abc123",
},
"opaque with query string": {
getURL: lazyURLParse("localhost:9000/abc123?efg=456"),
expect: "/abc123",
},
"failing test": {
getURL: func() (*url.URL, error) {
endpoint := "https://service.region.amazonaws.com"
req, _ := http.NewRequest("POST", endpoint, nil)
u := req.URL
u.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
query := u.Query()
query.Set("some-query-key", "value")
u.RawQuery = query.Encode()
return u, nil
},
expect: "/bucket/key-._~,!@#$%^&*()",
},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
u, err := c.getURL()
if err != nil {
t.Fatalf("failed to get URL, %v", err)
}
actual := GetURIPath(u)
if e, a := c.expect, actual; e != a {
t.Errorf("expect %v path, got %v", e, a)
}
})
}
}
func TestStripExcessHeaders(t *testing.T) {
vals := []string{
"",
"123",
"1 2 3",
"1 2 3 ",
" 1 2 3",
"1 2 3",
"1 23",
"1 2 3",
"1 2 ",
" 1 2 ",
"12 3",
"12 3 1",
"12 3 1",
"12 3 1abc123",
}
expected := []string{
"",
"123",
"1 2 3",
"1 2 3",
"1 2 3",
"1 2 3",
"1 23",
"1 2 3",
"1 2",
"1 2",
"12 3",
"12 3 1",
"12 3 1",
"12 3 1abc123",
}
for i := 0; i < len(vals); i++ {
r := StripExcessSpaces(vals[i])
if e, a := expected[i], r; e != a {
t.Errorf("%d, expect %v, got %v", i, e, a)
}
}
}
var stripExcessSpaceCases = []string{
`AWS4-HMAC-SHA256 Credential=AKIDFAKEIDFAKEID/20160628/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=1234567890abcdef1234567890abcdef1234567890abcdef`,
`123 321 123 321`,
` 123 321 123 321 `,
` 123 321 123 321 `,
"123",
"1 2 3",
" 1 2 3",
"1 2 3",
"1 23",
"1 2 3",
"1 2 ",
" 1 2 ",
"12 3",
"12 3 1",
"12 3 1",
"12 3 1abc123",
}
func BenchmarkStripExcessSpaces(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, v := range stripExcessSpaceCases {
StripExcessSpaces(v)
}
}
}

View File

@@ -1,139 +0,0 @@
package v4_test
import (
"context"
"fmt"
"net/http"
"testing"
"time"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/versity/versitygw/aws/internal/awstesting/unit"
v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
)
var standaloneSignCases = []struct {
OrigURI string
OrigQuery string
Region, Service, SubDomain string
ExpSig string
EscapedURI string
}{
{
OrigURI: `/logs-*/_search`,
OrigQuery: `pretty=true`,
Region: "us-west-2", Service: "es", SubDomain: "hostname-clusterkey",
EscapedURI: `/logs-%2A/_search`,
ExpSig: `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-west-2/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=79d0760751907af16f64a537c1242416dacf51204a7dd5284492d15577973b91`,
},
}
func TestStandaloneSign_CustomURIEscape(t *testing.T) {
var expectSig = `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=6601e883cc6d23871fd6c2a394c5677ea2b8c82b04a6446786d64cd74f520967`
creds, err := unit.Config().Credentials.Retrieve(context.Background())
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
signer := v4.NewSigner(func(signer *v4.SignerOptions) {
signer.DisableURIPathEscaping = true
})
host := "https://subdomain.us-east-1.es.amazonaws.com"
req, err := http.NewRequest("GET", host, nil)
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
req.URL.Path = `/log-*/_search`
req.URL.Opaque = "//subdomain.us-east-1.es.amazonaws.com/log-%2A/_search"
err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, "es", "us-east-1", time.Unix(0, 0))
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
actual := req.Header.Get("Authorization")
if e, a := expectSig, actual; e != a {
t.Errorf("expect %v, got %v", e, a)
}
}
func TestStandaloneSign(t *testing.T) {
creds, err := unit.Config().Credentials.Retrieve(context.Background())
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
signer := v4.NewSigner()
for _, c := range standaloneSignCases {
host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
c.SubDomain, c.Region, c.Service)
req, err := http.NewRequest("GET", host, nil)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
// URL.EscapedPath() will be used by the signer to get the
// escaped form of the request's URI path.
req.URL.Path = c.OrigURI
req.URL.RawQuery = c.OrigQuery
err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
actual := req.Header.Get("Authorization")
if e, a := c.ExpSig, actual; e != a {
t.Errorf("expected %v, but received %v", e, a)
}
if e, a := c.OrigURI, req.URL.Path; e != a {
t.Errorf("expected %v, but received %v", e, a)
}
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
t.Errorf("expected %v, but received %v", e, a)
}
}
}
func TestStandaloneSign_RawPath(t *testing.T) {
creds, err := unit.Config().Credentials.Retrieve(context.Background())
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
signer := v4.NewSigner()
for _, c := range standaloneSignCases {
host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
c.SubDomain, c.Region, c.Service)
req, err := http.NewRequest("GET", host, nil)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
// URL.EscapedPath() will be used by the signer to get the
// escaped form of the request's URI path.
req.URL.Path = c.OrigURI
req.URL.RawPath = c.EscapedURI
req.URL.RawQuery = c.OrigQuery
err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
actual := req.Header.Get("Authorization")
if e, a := c.ExpSig, actual; e != a {
t.Errorf("expected %v, but received %v", e, a)
}
if e, a := c.OrigURI, req.URL.Path; e != a {
t.Errorf("expected %v, but received %v", e, a)
}
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
t.Errorf("expected %v, but received %v", e, a)
}
}
}

View File

@@ -1,565 +0,0 @@
// Package v4 implements signing for AWS V4 signer
//
// Provides request signing for request that need to be signed with
// AWS V4 Signatures.
//
// # Standalone Signer
//
// Generally using the signer outside of the SDK should not require any additional
//
// The signer does this by taking advantage of the URL.EscapedPath method. If your request URI requires
//
// additional escaping you many need to use the URL.Opaque to define what the raw URI should be sent
// to the service as.
//
// The signer will first check the URL.Opaque field, and use its value if set.
// The signer does require the URL.Opaque field to be set in the form of:
//
// "//<hostname>/<path>"
//
// // e.g.
// "//example.com/some/path"
//
// The leading "//" and hostname are required or the URL.Opaque escaping will
// not work correctly.
//
// If URL.Opaque is not set the signer will fallback to the URL.EscapedPath()
// method and using the returned value.
//
// AWS v4 signature validation requires that the canonical string's URI path
// element must be the URI escaped form of the HTTP request's path.
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
//
// The Go HTTP client will perform escaping automatically on the request. Some
// of these escaping may cause signature validation errors because the HTTP
// request differs from the URI path or query that the signature was generated.
// https://golang.org/pkg/net/url/#URL.EscapedPath
//
// Because of this, it is recommended that when using the signer outside of the
// SDK that explicitly escaping the request prior to being signed is preferable,
// and will help prevent signature validation errors. This can be done by setting
// the URL.Opaque or URL.RawPath. The SDK will use URL.Opaque first and then
// call URL.EscapedPath() if Opaque is not set.
//
// Test `TestStandaloneSign` provides a complete example of using the signer
// outside of the SDK and pre-escaping the URI path.
package v4
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"hash"
"net/http"
"net/textproto"
"net/url"
"slices"
"sort"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/smithy-go/encoding/httpbinding"
"github.com/aws/smithy-go/logging"
v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
)
const (
signingAlgorithm = "AWS4-HMAC-SHA256"
authorizationHeader = "Authorization"
// Version of signing v4
Version = "SigV4"
)
// HTTPSigner is an interface to a SigV4 signer that can sign HTTP requests
type HTTPSigner interface {
SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error
}
type keyDerivator interface {
DeriveKey(credential aws.Credentials, service, region string, signingTime v4Internal.SigningTime) []byte
}
// SignerOptions is the SigV4 Signer options.
type SignerOptions struct {
// Disables the Signer's moving HTTP header key/value pairs from the HTTP
// request header to the request's query string. This is most commonly used
// with pre-signed requests preventing headers from being added to the
// request's query string.
DisableHeaderHoisting bool
// Disables the automatic escaping of the URI path of the request for the
// siganture's canonical string's path. For services that do not need additional
// escaping then use this to disable the signer escaping the path.
//
// S3 is an example of a service that does not need additional escaping.
//
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
DisableURIPathEscaping bool
// The logger to send log messages to.
Logger logging.Logger
// Enable logging of signed requests.
// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
// presigned URL.
LogSigning bool
// Disables setting the session token on the request as part of signing
// through X-Amz-Security-Token. This is needed for variations of v4 that
// present the token elsewhere.
DisableSessionToken bool
}
// Signer applies AWS v4 signing to given request. Use this to sign requests
// that need to be signed with AWS V4 Signatures.
type Signer struct {
options SignerOptions
keyDerivator keyDerivator
}
// NewSigner returns a new SigV4 Signer
func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
options := SignerOptions{}
for _, fn := range optFns {
fn(&options)
}
return &Signer{options: options, keyDerivator: v4Internal.NewSigningKeyDeriver()}
}
type httpSigner struct {
Request *http.Request
ServiceName string
Region string
Time v4Internal.SigningTime
Credentials aws.Credentials
KeyDerivator keyDerivator
IsPreSign bool
SignedHdrs []string
PayloadHash string
DisableHeaderHoisting bool
DisableURIPathEscaping bool
DisableSessionToken bool
}
func (s *httpSigner) Build() (signedRequest, error) {
req := s.Request
query := req.URL.Query()
headers := req.Header
s.setRequiredSigningFields(headers, query)
// Sort Each Query Key's Values
for key := range query {
sort.Strings(query[key])
}
v4Internal.SanitizeHostForHeader(req)
credentialScope := s.buildCredentialScope()
credentialStr := s.Credentials.AccessKeyID + "/" + credentialScope
if s.IsPreSign {
query.Set(v4Internal.AmzCredentialKey, credentialStr)
}
unsignedHeaders := headers
if s.IsPreSign && !s.DisableHeaderHoisting {
var urlValues url.Values
urlValues, unsignedHeaders = buildQuery(v4Internal.AllowedQueryHoisting, headers)
for k := range urlValues {
query[k] = urlValues[k]
}
}
host := req.URL.Host
if len(req.Host) > 0 {
host = req.Host
}
signedHeaders, signedHeadersStr, canonicalHeaderStr := s.buildCanonicalHeaders(host, v4Internal.IgnoredHeaders, unsignedHeaders, s.Request.ContentLength)
if s.IsPreSign {
query.Set(v4Internal.AmzSignedHeadersKey, signedHeadersStr)
}
var rawQuery strings.Builder
rawQuery.WriteString(strings.Replace(query.Encode(), "+", "%20", -1))
canonicalURI := v4Internal.GetURIPath(req.URL)
if !s.DisableURIPathEscaping {
canonicalURI = httpbinding.EscapePath(canonicalURI, false)
}
canonicalString := s.buildCanonicalString(
req.Method,
canonicalURI,
rawQuery.String(),
signedHeadersStr,
canonicalHeaderStr,
)
strToSign := s.buildStringToSign(credentialScope, canonicalString)
signingSignature, err := s.buildSignature(strToSign)
if err != nil {
return signedRequest{}, err
}
if s.IsPreSign {
rawQuery.WriteString("&X-Amz-Signature=")
rawQuery.WriteString(signingSignature)
} else {
headers[authorizationHeader] = append(headers[authorizationHeader][:0], buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature))
}
req.URL.RawQuery = rawQuery.String()
return signedRequest{
Request: req,
SignedHeaders: signedHeaders,
CanonicalString: canonicalString,
StringToSign: strToSign,
PreSigned: s.IsPreSign,
}, nil
}
func buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature string) string {
const credential = "Credential="
const signedHeaders = "SignedHeaders="
const signature = "Signature="
const commaSpace = ", "
var parts strings.Builder
parts.Grow(len(signingAlgorithm) + 1 +
len(credential) + len(credentialStr) + 2 +
len(signedHeaders) + len(signedHeadersStr) + 2 +
len(signature) + len(signingSignature),
)
parts.WriteString(signingAlgorithm)
parts.WriteRune(' ')
parts.WriteString(credential)
parts.WriteString(credentialStr)
parts.WriteString(commaSpace)
parts.WriteString(signedHeaders)
parts.WriteString(signedHeadersStr)
parts.WriteString(commaSpace)
parts.WriteString(signature)
parts.WriteString(signingSignature)
return parts.String()
}
// SignHTTP signs AWS v4 requests with the provided payload hash, service name, region the
// request is made to, and time the request is signed at. The signTime allows
// you to specify that a request is signed for the future, and cannot be
// used until then.
//
// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
// must be provided. Even if the request has no payload (aka body). If the
// request has no payload you should use the hex encoded SHA-256 of an empty
// string as the payloadHash value.
//
// "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
//
// Some services such as Amazon S3 accept alternative values for the payload
// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
// included in the request signature.
//
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
//
// Sign differs from Presign in that it will sign the request using HTTP
// header values. This type of signing is intended for http.Request values that
// will not be shared, or are shared in a way the header values on the request
// will not be lost.
//
// The passed in request will be modified in place.
func (s Signer) SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, signedHdrs []string, optFns ...func(options *SignerOptions)) error {
options := s.options
for _, fn := range optFns {
fn(&options)
}
signer := &httpSigner{
Request: r,
PayloadHash: payloadHash,
ServiceName: service,
Region: region,
Credentials: credentials,
Time: v4Internal.NewSigningTime(signingTime.UTC()),
DisableHeaderHoisting: options.DisableHeaderHoisting,
DisableURIPathEscaping: options.DisableURIPathEscaping,
DisableSessionToken: options.DisableSessionToken,
KeyDerivator: s.keyDerivator,
SignedHdrs: signedHdrs,
}
signedRequest, err := signer.Build()
if err != nil {
return err
}
logSigningInfo(ctx, options, &signedRequest, false)
return nil
}
// PresignHTTP signs AWS v4 requests with the payload hash, service name, region
// the request is made to, and time the request is signed at. The signTime
// allows you to specify that a request is signed for the future, and cannot
// be used until then.
//
// Returns the signed URL and the map of HTTP headers that were included in the
// signature or an error if signing the request failed. For presigned requests
// these headers and their values must be included on the HTTP request when it
// is made. This is helpful to know what header values need to be shared with
// the party the presigned request will be distributed to.
//
// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
// must be provided. Even if the request has no payload (aka body). If the
// request has no payload you should use the hex encoded SHA-256 of an empty
// string as the payloadHash value.
//
// "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
//
// Some services such as Amazon S3 accept alternative values for the payload
// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
// included in the request signature.
//
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
//
// PresignHTTP differs from SignHTTP in that it will sign the request using
// query string instead of header values. This allows you to share the
// Presigned Request's URL with third parties, or distribute it throughout your
// system with minimal dependencies.
//
// PresignHTTP will not set the expires time of the presigned request
// automatically. To specify the expire duration for a request add the
// "X-Amz-Expires" query parameter on the request with the value as the
// duration in seconds the presigned URL should be considered valid for. This
// parameter is not used by all AWS services, and is most notable used by
// Amazon S3 APIs.
//
// expires := 20 * time.Minute
// query := req.URL.Query()
// query.Set("X-Amz-Expires", strconv.FormatInt(int64(expires/time.Second), 10))
// req.URL.RawQuery = query.Encode()
//
// This method does not modify the provided request.
func (s *Signer) PresignHTTP(
ctx context.Context, credentials aws.Credentials, r *http.Request,
payloadHash string, service string, region string, signingTime time.Time,
signedHdrs []string,
optFns ...func(*SignerOptions),
) (signedURI string, signedHeaders http.Header, err error) {
options := s.options
for _, fn := range optFns {
fn(&options)
}
signer := &httpSigner{
Request: r.Clone(r.Context()),
PayloadHash: payloadHash,
ServiceName: service,
Region: region,
Credentials: credentials,
Time: v4Internal.NewSigningTime(signingTime.UTC()),
IsPreSign: true,
DisableHeaderHoisting: options.DisableHeaderHoisting,
DisableURIPathEscaping: options.DisableURIPathEscaping,
DisableSessionToken: options.DisableSessionToken,
KeyDerivator: s.keyDerivator,
SignedHdrs: signedHdrs,
}
signedRequest, err := signer.Build()
if err != nil {
return "", nil, err
}
logSigningInfo(ctx, options, &signedRequest, true)
signedHeaders = make(http.Header)
// For the signed headers we canonicalize the header keys in the returned map.
// This avoids situations where can standard library double headers like host header. For example the standard
// library will set the Host header, even if it is present in lower-case form.
for k, v := range signedRequest.SignedHeaders {
key := textproto.CanonicalMIMEHeaderKey(k)
signedHeaders[key] = append(signedHeaders[key], v...)
}
return signedRequest.Request.URL.String(), signedHeaders, nil
}
func (s *httpSigner) buildCredentialScope() string {
return v4Internal.BuildCredentialScope(s.Time, s.Region, s.ServiceName)
}
func buildQuery(r v4Internal.Rule, header http.Header) (url.Values, http.Header) {
query := url.Values{}
unsignedHeaders := http.Header{}
for k, h := range header {
if r.IsValid(k) {
query[k] = h
} else {
unsignedHeaders[k] = h
}
}
return query, unsignedHeaders
}
func (s *httpSigner) buildCanonicalHeaders(host string, rule v4Internal.Rule, header http.Header, length int64) (signed http.Header, signedHeaders, canonicalHeadersStr string) {
signed = make(http.Header)
var headers []string
const hostHeader = "host"
headers = append(headers, hostHeader)
signed[hostHeader] = append(signed[hostHeader], host)
const contentLengthHeader = "content-length"
if slices.Contains(s.SignedHdrs, contentLengthHeader) {
headers = append(headers, contentLengthHeader)
signed[contentLengthHeader] = append(signed[contentLengthHeader], strconv.FormatInt(length, 10))
}
for k, v := range header {
if !rule.IsValid(k) {
continue // ignored header
}
if strings.EqualFold(k, contentLengthHeader) {
// prevent signing already handled content-length header.
continue
}
lowerCaseKey := strings.ToLower(k)
if _, ok := signed[lowerCaseKey]; ok {
// include additional values
signed[lowerCaseKey] = append(signed[lowerCaseKey], v...)
continue
}
headers = append(headers, lowerCaseKey)
signed[lowerCaseKey] = v
}
sort.Strings(headers)
signedHeaders = strings.Join(headers, ";")
var canonicalHeaders strings.Builder
n := len(headers)
const colon = ':'
for i := 0; i < n; i++ {
if headers[i] == hostHeader {
canonicalHeaders.WriteString(hostHeader)
canonicalHeaders.WriteRune(colon)
canonicalHeaders.WriteString(v4Internal.StripExcessSpaces(host))
} else {
canonicalHeaders.WriteString(headers[i])
canonicalHeaders.WriteRune(colon)
// Trim out leading, trailing, and dedup inner spaces from signed header values.
values := signed[headers[i]]
for j, v := range values {
cleanedValue := strings.TrimSpace(v4Internal.StripExcessSpaces(v))
canonicalHeaders.WriteString(cleanedValue)
if j < len(values)-1 {
canonicalHeaders.WriteRune(',')
}
}
}
canonicalHeaders.WriteRune('\n')
}
canonicalHeadersStr = canonicalHeaders.String()
return signed, signedHeaders, canonicalHeadersStr
}
func (s *httpSigner) buildCanonicalString(method, uri, query, signedHeaders, canonicalHeaders string) string {
return strings.Join([]string{
method,
uri,
query,
canonicalHeaders,
signedHeaders,
s.PayloadHash,
}, "\n")
}
func (s *httpSigner) buildStringToSign(credentialScope, canonicalRequestString string) string {
return strings.Join([]string{
signingAlgorithm,
s.Time.TimeFormat(),
credentialScope,
hex.EncodeToString(makeHash(sha256.New(), []byte(canonicalRequestString))),
}, "\n")
}
func makeHash(hash hash.Hash, b []byte) []byte {
hash.Reset()
hash.Write(b)
return hash.Sum(nil)
}
func (s *httpSigner) buildSignature(strToSign string) (string, error) {
key := s.KeyDerivator.DeriveKey(s.Credentials, s.ServiceName, s.Region, s.Time)
return hex.EncodeToString(v4Internal.HMACSHA256(key, []byte(strToSign))), nil
}
func (s *httpSigner) setRequiredSigningFields(headers http.Header, query url.Values) {
amzDate := s.Time.TimeFormat()
if s.IsPreSign {
query.Set(v4Internal.AmzAlgorithmKey, signingAlgorithm)
sessionToken := s.Credentials.SessionToken
if !s.DisableSessionToken && len(sessionToken) > 0 {
query.Set("X-Amz-Security-Token", sessionToken)
}
query.Set(v4Internal.AmzDateKey, amzDate)
return
}
headers[v4Internal.AmzDateKey] = append(headers[v4Internal.AmzDateKey][:0], amzDate)
if !s.DisableSessionToken && len(s.Credentials.SessionToken) > 0 {
headers[v4Internal.AmzSecurityTokenKey] = append(headers[v4Internal.AmzSecurityTokenKey][:0], s.Credentials.SessionToken)
}
}
func logSigningInfo(ctx context.Context, options SignerOptions, request *signedRequest, isPresign bool) {
if !options.LogSigning {
return
}
signedURLMsg := ""
if isPresign {
signedURLMsg = fmt.Sprintf(logSignedURLMsg, request.Request.URL.String())
}
logger := logging.WithContext(ctx, options.Logger)
logger.Logf(logging.Debug, logSignInfoMsg, request.CanonicalString, request.StringToSign, signedURLMsg)
}
type signedRequest struct {
Request *http.Request
SignedHeaders http.Header
CanonicalString string
StringToSign string
PreSigned bool
}
const logSignInfoMsg = `Request Signature:
---[ CANONICAL STRING ]-----------------------------
%s
---[ STRING TO SIGN ]--------------------------------
%s%s
-----------------------------------------------------`
const logSignedURLMsg = `
---[ SIGNED URL ]------------------------------------
%s`

View File

@@ -1,358 +0,0 @@
package v4
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/google/go-cmp/cmp"
v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
)
var testCredentials = aws.Credentials{AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION"}
func buildRequest(serviceName, region, body string) (*http.Request, string) {
reader := strings.NewReader(body)
return buildRequestWithBodyReader(serviceName, region, reader)
}
func buildRequestWithBodyReader(serviceName, region string, body io.Reader) (*http.Request, string) {
var bodyLen int
type lenner interface {
Len() int
}
if lr, ok := body.(lenner); ok {
bodyLen = lr.Len()
}
endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
req, _ := http.NewRequest("POST", endpoint, body)
req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
req.Header.Set("X-Amz-Target", "prefix.Operation")
req.Header.Set("Content-Type", "application/x-amz-json-1.0")
if bodyLen > 0 {
req.ContentLength = int64(bodyLen)
}
req.Header.Set("X-Amz-Meta-Other-Header", "some-value=!@#$%^&* (+)")
req.Header.Add("X-Amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
req.Header.Add("X-amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
h := sha256.New()
_, _ = io.Copy(h, body)
payloadHash := hex.EncodeToString(h.Sum(nil))
return req, payloadHash
}
func TestPresignRequest(t *testing.T) {
req, body := buildRequest("dynamodb", "us-east-1", "{}")
query := req.URL.Query()
query.Set("X-Amz-Expires", "300")
req.URL.RawQuery = query.Encode()
signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
signer := NewSigner()
signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
expectedDate := "19700101T000000Z"
expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
expectedSig := "122f0b9e091e4ba84286097e2b3404a1f1f4c4aad479adda95b7dff0ccbe5581"
expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
expectedTarget := "prefix.Operation"
q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
if err != nil {
t.Errorf("expect no error, got %v", err)
}
if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
t.Errorf("expect %v to be empty", a)
}
if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
for _, h := range strings.Split(expectedHeaders, ";") {
v := headers.Get(h)
if len(v) == 0 {
t.Errorf("expect %v, to be present in header map", h)
}
}
}
func TestPresignBodyWithArrayRequest(t *testing.T) {
req, body := buildRequest("dynamodb", "us-east-1", "{}")
req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
query := req.URL.Query()
query.Set("X-Amz-Expires", "300")
req.URL.RawQuery = query.Encode()
signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
signer := NewSigner()
signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
if err != nil {
t.Errorf("expect no error, got %v", err)
}
expectedDate := "19700101T000000Z"
expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
expectedSig := "e3ac55addee8711b76c6d608d762cff285fe8b627a057f8b5ec9268cf82c08b1"
expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
expectedTarget := "prefix.Operation"
if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
t.Errorf("expect %v to be empty, was not", a)
}
if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
for _, h := range strings.Split(expectedHeaders, ";") {
v := headers.Get(h)
if len(v) == 0 {
t.Errorf("expect %v, to be present in header map", h)
}
}
}
func TestSignRequest(t *testing.T) {
req, body := buildRequest("dynamodb", "us-east-1", "{}")
signer := NewSigner()
signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
err := signer.SignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
expectedDate := "19700101T000000Z"
expectedSig := "AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/dynamodb/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore;x-amz-security-token;x-amz-target, Signature=a518299330494908a70222cec6899f6f32f297f8595f6df1776d998936652ad9"
q := req.Header
if e, a := expectedSig, q.Get("Authorization"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
}
func TestBuildCanonicalRequest(t *testing.T) {
req, _ := buildRequest("dynamodb", "us-east-1", "{}")
req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
ctx := &httpSigner{
ServiceName: "dynamodb",
Region: "us-east-1",
Request: req,
Time: v4Internal.NewSigningTime(time.Now()),
KeyDerivator: v4Internal.NewSigningKeyDeriver(),
}
build, err := ctx.Build()
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
expected := "https://example.org/bucket/key-._~,!@#$%^&*()?Foo=a&Foo=m&Foo=o&Foo=z"
if e, a := expected, build.Request.URL.String(); e != a {
t.Errorf("expect %v, got %v", e, a)
}
}
func TestSigner_SignHTTP_NoReplaceRequestBody(t *testing.T) {
req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
req.Body = io.NopCloser(bytes.NewReader([]byte{}))
s := NewSigner()
origBody := req.Body
err := s.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if req.Body != origBody {
t.Errorf("expect request body to not be chagned")
}
}
func TestRequestHost(t *testing.T) {
req, _ := buildRequest("dynamodb", "us-east-1", "{}")
req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
req.Host = "myhost"
query := req.URL.Query()
query.Set("X-Amz-Expires", "5")
req.URL.RawQuery = query.Encode()
ctx := &httpSigner{
ServiceName: "dynamodb",
Region: "us-east-1",
Request: req,
Time: v4Internal.NewSigningTime(time.Now()),
KeyDerivator: v4Internal.NewSigningKeyDeriver(),
}
build, err := ctx.Build()
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if !strings.Contains(build.CanonicalString, "host:"+req.Host) {
t.Errorf("canonical host header invalid")
}
}
func TestSign_buildCanonicalHeadersContentLengthPresent(t *testing.T) {
body := `{"description": "this is a test"}`
req, _ := buildRequest("dynamodb", "us-east-1", body)
req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
req.Host = "myhost"
contentLength := fmt.Sprintf("%d", len([]byte(body)))
req.Header.Add("Content-Length", contentLength)
query := req.URL.Query()
query.Set("X-Amz-Expires", "5")
req.URL.RawQuery = query.Encode()
ctx := &httpSigner{
ServiceName: "dynamodb",
Region: "us-east-1",
Request: req,
Time: v4Internal.NewSigningTime(time.Now()),
KeyDerivator: v4Internal.NewSigningKeyDeriver(),
}
_, err := ctx.Build()
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
//if !strings.Contains(build.CanonicalString, "content-length:"+contentLength+"\n") {
// t.Errorf("canonical header content-length invalid")
//}
}
func TestSign_buildCanonicalHeaders(t *testing.T) {
serviceName := "mockAPI"
region := "mock-region"
endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
req, err := http.NewRequest("POST", endpoint, nil)
if err != nil {
t.Fatalf("failed to create request, %v", err)
}
req.Header.Set("FooInnerSpace", " inner space ")
req.Header.Set("FooLeadingSpace", " leading-space")
req.Header.Add("FooMultipleSpace", "no-space")
req.Header.Add("FooMultipleSpace", "\ttab-space")
req.Header.Add("FooMultipleSpace", "trailing-space ")
req.Header.Set("FooNoSpace", "no-space")
req.Header.Set("FooTabSpace", "\ttab-space\t")
req.Header.Set("FooTrailingSpace", "trailing-space ")
req.Header.Set("FooWrappedSpace", " wrapped-space ")
ctx := &httpSigner{
ServiceName: serviceName,
Region: region,
Request: req,
Time: v4Internal.NewSigningTime(time.Date(2021, 10, 20, 12, 42, 0, 0, time.UTC)),
KeyDerivator: v4Internal.NewSigningKeyDeriver(),
}
build, err := ctx.Build()
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
expectCanonicalString := strings.Join([]string{
`POST`,
`/`,
``,
`fooinnerspace:inner space`,
`fooleadingspace:leading-space`,
`foomultiplespace:no-space,tab-space,trailing-space`,
`foonospace:no-space`,
`footabspace:tab-space`,
`footrailingspace:trailing-space`,
`foowrappedspace:wrapped-space`,
`host:mockAPI.mock-region.amazonaws.com`,
`x-amz-date:20211020T124200Z`,
``,
`fooinnerspace;fooleadingspace;foomultiplespace;foonospace;footabspace;footrailingspace;foowrappedspace;host;x-amz-date`,
``,
}, "\n")
if diff := cmp.Diff(expectCanonicalString, build.CanonicalString); diff != "" {
t.Errorf("expect match, got\n%s", diff)
}
}
func BenchmarkPresignRequest(b *testing.B) {
signer := NewSigner()
req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
query := req.URL.Query()
query.Set("X-Amz-Expires", "5")
req.URL.RawQuery = query.Encode()
for i := 0; i < b.N; i++ {
signer.PresignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
}
}
func BenchmarkSignRequest(b *testing.B) {
signer := NewSigner()
req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
for i := 0; i < b.N; i++ {
signer.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,63 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package azure
import (
"errors"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/versity/versitygw/s3err"
)
// Parses azure ResponseError into AWS APIError
func azureErrToS3Err(apiErr error) error {
var azErr *azcore.ResponseError
if !errors.As(apiErr, &azErr) {
return apiErr
}
return azErrToS3err(azErr)
}
func azErrToS3err(azErr *azcore.ResponseError) s3err.APIError {
switch azErr.ErrorCode {
case "ContainerAlreadyExists":
return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
case "InvalidResourceName", "ContainerNotFound":
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
case "BlobNotFound":
return s3err.GetAPIError(s3err.ErrNoSuchKey)
case "TagsTooLarge":
return s3err.GetAPIError(s3err.ErrInvalidTagValue)
case "Requested Range Not Satisfiable":
return s3err.GetAPIError(s3err.ErrInvalidRange)
}
return s3err.APIError{
Code: azErr.ErrorCode,
Description: azErr.RawResponse.Status,
HTTPStatusCode: azErr.StatusCode,
}
}
func parseMpError(mpErr error) error {
err := azureErrToS3Err(mpErr)
serr, ok := err.(s3err.APIError)
if !ok || serr.Code != "NoSuchKey" {
return mpErr
}
return s3err.GetAPIError(s3err.ErrNoSuchUpload)
}

View File

@@ -1,288 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package backend
import (
"bufio"
"context"
"fmt"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
"github.com/versity/versitygw/s3select"
)
//go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
type Backend interface {
fmt.Stringer
Shutdown()
// bucket operations
ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error)
HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
PutBucketAcl(_ context.Context, bucket string, data []byte) error
DeleteBucket(_ context.Context, bucket string) error
PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error
GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error)
PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
DeleteBucketPolicy(_ context.Context, bucket string) error
PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error
GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error)
DeleteBucketOwnershipControls(_ context.Context, bucket string) error
PutBucketCors(_ context.Context, bucket string, cors []byte) error
GetBucketCors(_ context.Context, bucket string) ([]byte, error)
DeleteBucketCors(_ context.Context, bucket string) error
// multipart operations
CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error)
CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (_ s3response.CompleteMultipartUploadResult, versionid string, _ error)
AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error)
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error)
// standard object operations
PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error)
HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error)
CopyObject(context.Context, s3response.CopyObjectInput) (s3response.CopyObjectOutput, error)
ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error)
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error)
DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error)
// special case object operations
RestoreObject(context.Context, *s3.RestoreObjectInput) error
SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
// bucket tagging operations
GetBucketTagging(_ context.Context, bucket string) (map[string]string, error)
PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error
DeleteBucketTagging(_ context.Context, bucket string) error
// object tagging operations
GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
DeleteObjectTagging(_ context.Context, bucket, object string) error
// object lock operations
PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error
GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error)
PutObjectRetention(_ context.Context, bucket, object, versionId string, retention []byte) error
GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error)
PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error
GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error)
// non AWS actions
ChangeBucketOwner(_ context.Context, bucket, owner string) error
ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
}
type BackendUnsupported struct{}
var _ Backend = &BackendUnsupported{}
func New() Backend {
return &BackendUnsupported{}
}
func (BackendUnsupported) Shutdown() {}
func (BackendUnsupported) String() string {
return "Unsupported"
}
func (BackendUnsupported) ListBuckets(context.Context, s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucket(_ context.Context, bucket string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketVersioning(_ context.Context, bucket string, status types.BucketVersioningStatus) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (s3response.GetBucketVersioningOutput, error) {
return s3response.GetBucketVersioningOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketOwnershipControls(_ context.Context, bucket string, ownership types.ObjectOwnership) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketOwnershipControls(_ context.Context, bucket string) (types.ObjectOwnership, error) {
return types.ObjectOwnershipBucketOwnerEnforced, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucketOwnershipControls(_ context.Context, bucket string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketCors(context.Context, string, []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketCors(_ context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucketCors(_ context.Context, bucket string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CreateMultipartUpload(context.Context, s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) {
return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyPartResult, error) {
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObject(context.Context, s3response.PutObjectInput) (s3response.PutObjectOutput, error) {
return s3response.PutObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResponse, error) {
return s3response.GetObjectAttributesResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CopyObject(context.Context, s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) {
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
return func(w *bufio.Writer) {
var getProgress s3select.GetProgress
progress := input.RequestProgress
if progress != nil && *progress.Enabled {
getProgress = func() (bytesScanned int64, bytesProcessed int64) {
return -1, -1
}
}
mh := s3select.NewMessageHandler(ctx, w, getProgress)
apiErr := s3err.GetAPIError(s3err.ErrNotImplemented)
mh.FinishWithError(apiErr.Code, apiErr.Description)
}
}
func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (s3response.ListVersionsResult, error) {
return s3response.ListVersionsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucketTagging(_ context.Context, bucket string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectRetention(_ context.Context, bucket, object, versionId string, retention []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, owner string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {
return []s3response.Bucket{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}

View File

@@ -1,572 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package backend
import (
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"hash"
"io"
"io/fs"
"math"
"net/url"
"os"
"regexp"
"strconv"
"strings"
"syscall"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
const (
// this is the media type for directories in AWS and Nextcloud
DirContentType = "application/x-directory"
DefaultContentType = "binary/octet-stream"
// this is the minimum allowed size for mp parts
MinPartSize = 5 * 1024 * 1024
)
func IsValidBucketName(name string) bool { return true }
type ByBucketName []s3response.ListAllMyBucketsEntry
func (d ByBucketName) Len() int { return len(d) }
func (d ByBucketName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d ByBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name }
type ByObjectName []types.Object
func (d ByObjectName) Len() int { return len(d) }
func (d ByObjectName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
func GetPtrFromString(str string) *string {
if str == "" {
return nil
}
return &str
}
func GetStringFromPtr(str *string) string {
if str == nil {
return ""
}
return *str
}
func GetTimePtr(t time.Time) *time.Time {
return &t
}
func TrimEtag(etag *string) *string {
if etag == nil {
return nil
}
return GetPtrFromString(strings.Trim(*etag, "\""))
}
var (
errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
errInvalidCopySourceRange = s3err.GetAPIError(s3err.ErrInvalidCopySourceRange)
errPreconditionFailed = s3err.GetAPIError(s3err.ErrPreconditionFailed)
errNotModified = s3err.GetAPIError(s3err.ErrNotModified)
)
// ParseObjectRange parses input range header and returns startoffset, length, isValid
// and error. If no endoffset specified, then length is set to the object size
// for invalid inputs, it returns no error, but isValid=false
// `InvalidRange` error is returnd, only if startoffset is greater than the object size
func ParseObjectRange(size int64, acceptRange string) (int64, int64, bool, error) {
// Return full object (invalid range, no error) if header empty
if acceptRange == "" {
return 0, size, false, nil
}
rangeKv := strings.Split(acceptRange, "=")
if len(rangeKv) != 2 {
return 0, size, false, nil
}
if rangeKv[0] != "bytes" { // unsupported unit -> ignore
return 0, size, false, nil
}
bRange := strings.Split(rangeKv[1], "-")
if len(bRange) != 2 { // malformed / multi-range
return 0, size, false, nil
}
// Parse start; empty start indicates a suffix-byte-range-spec (e.g. bytes=-100)
startOffset, err := strconv.ParseInt(bRange[0], 10, strconv.IntSize)
if startOffset > int64(math.MaxInt) || startOffset < int64(math.MinInt) {
return 0, size, false, errInvalidRange
}
if err != nil && bRange[0] != "" { // invalid numeric start (non-empty) -> ignore range
return 0, size, false, nil
}
// If end part missing (e.g. bytes=100-)
if bRange[1] == "" {
if bRange[0] == "" { // bytes=- (meaningless) -> ignore
return 0, size, false, nil
}
// start beyond or at size is unsatisfiable -> error (RequestedRangeNotSatisfiable)
if startOffset >= size {
return 0, 0, false, errInvalidRange
}
// bytes=100- => from start to end
return startOffset, size - startOffset, true, nil
}
endOffset, err := strconv.ParseInt(bRange[1], 10, strconv.IntSize)
if endOffset > int64(math.MaxInt) {
return 0, size, false, errInvalidRange
}
if err != nil { // invalid numeric end -> ignore range
return 0, size, false, nil
}
// Suffix range handling (bRange[0] == "")
if bRange[0] == "" {
// Disallow -0 (always unsatisfiable)
if endOffset == 0 {
return 0, 0, false, errInvalidRange
}
// For zero-sized objects any positive suffix is treated as invalid (ignored, no error)
if size == 0 {
return 0, size, false, nil
}
// Clamp to object size (request more bytes than exist -> entire object)
endOffset = min(endOffset, size)
return size - endOffset, endOffset, true, nil
}
// Normal range (start-end)
if startOffset > endOffset { // start > end -> ignore
return 0, size, false, nil
}
// Start beyond or at end of object -> error
if startOffset >= size {
return 0, 0, false, errInvalidRange
}
// Adjust end beyond object size (trim)
if endOffset >= size {
endOffset = size - 1
}
return startOffset, endOffset - startOffset + 1, true, nil
}
// ParseCopySourceRange parses input range header and returns startoffset, length
// and error. If no endoffset specified, then length is set to the object size
func ParseCopySourceRange(size int64, acceptRange string) (int64, int64, error) {
if acceptRange == "" {
return 0, size, nil
}
rangeKv := strings.Split(acceptRange, "=")
if len(rangeKv) != 2 {
return 0, 0, errInvalidCopySourceRange
}
if rangeKv[0] != "bytes" {
return 0, 0, errInvalidCopySourceRange
}
bRange := strings.Split(rangeKv[1], "-")
if len(bRange) != 2 {
return 0, 0, errInvalidCopySourceRange
}
startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
if err != nil {
return 0, 0, errInvalidCopySourceRange
}
if startOffset >= size {
return 0, 0, s3err.CreateExceedingRangeErr(size)
}
if bRange[1] == "" {
return startOffset, size - startOffset + 1, nil
}
endOffset, err := strconv.ParseInt(bRange[1], 10, 64)
if err != nil {
return 0, 0, errInvalidCopySourceRange
}
if endOffset < startOffset {
return 0, 0, errInvalidCopySourceRange
}
if endOffset >= size {
return 0, 0, s3err.CreateExceedingRangeErr(size)
}
return startOffset, endOffset - startOffset + 1, nil
}
// ParseCopySource parses x-amz-copy-source header and returns source bucket,
// source object, versionId, error respectively
func ParseCopySource(copySourceHeader string) (string, string, string, error) {
if copySourceHeader[0] == '/' {
copySourceHeader = copySourceHeader[1:]
}
var copySource, versionId string
i := strings.LastIndex(copySourceHeader, "?versionId=")
if i == -1 {
copySource = copySourceHeader
} else {
copySource = copySourceHeader[:i]
versionId = copySourceHeader[i+11:]
}
srcBucket, srcObject, ok := strings.Cut(copySource, "/")
if !ok {
return "", "", "", s3err.GetAPIError(s3err.ErrInvalidCopySourceBucket)
}
return srcBucket, srcObject, versionId, nil
}
// ParseObjectTags parses the url encoded input string into
// map[string]string with unescaped key/value pair
func ParseObjectTags(tagging string) (map[string]string, error) {
if tagging == "" {
return nil, nil
}
tagSet := make(map[string]string)
for tagging != "" {
var tag string
tag, tagging, _ = strings.Cut(tagging, "&")
// if 'tag' before the first appearance of '&' is empty continue
if tag == "" {
continue
}
key, value, found := strings.Cut(tag, "=")
// if key is empty, but "=" is present, return invalid url ecnoding err
if found && key == "" {
return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
}
// return invalid tag key, if the key is longer than 128
if len(key) > 128 {
return nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)
}
// return invalid tag value, if tag value is longer than 256
if len(value) > 256 {
return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
}
// query unescape tag key
key, err := url.QueryUnescape(key)
if err != nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
}
// query unescape tag value
value, err = url.QueryUnescape(value)
if err != nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
}
// check tag key to be valid
if !isValidTagComponent(key) {
return nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)
}
// check tag value to be valid
if !isValidTagComponent(value) {
return nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)
}
// duplicate keys are not allowed: return invalid url encoding err
_, ok := tagSet[key]
if ok {
return nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)
}
tagSet[key] = value
}
return tagSet, nil
}
var validTagComponent = regexp.MustCompile(`^[a-zA-Z0-9:/_.\-+ ]+$`)
// isValidTagComponent matches strings which contain letters, decimal digits,
// and special chars: '/', '_', '-', '+', '.', ' ' (space)
func isValidTagComponent(str string) bool {
if str == "" {
return true
}
return validTagComponent.Match([]byte(str))
}
func GetMultipartMD5(parts []types.CompletedPart) string {
var partsEtagBytes []byte
for _, part := range parts {
partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
}
return fmt.Sprintf("\"%s-%d\"", md5String(partsEtagBytes), len(parts))
}
func getEtagBytes(etag string) []byte {
decode, err := hex.DecodeString(strings.ReplaceAll(etag, string('"'), ""))
if err != nil {
return []byte(etag)
}
return decode
}
func md5String(data []byte) string {
sum := md5.Sum(data)
return hex.EncodeToString(sum[:])
}
type FileSectionReadCloser struct {
R io.Reader
F *os.File
}
func (f *FileSectionReadCloser) Read(p []byte) (int, error) {
return f.R.Read(p)
}
func (f *FileSectionReadCloser) Close() error {
return f.F.Close()
}
// MoveFile moves a file from source to destination.
func MoveFile(source, destination string, perm os.FileMode) error {
// We use Rename as the atomic operation for object puts. The upload is
// written to a temp file to not conflict with any other simultaneous
// uploads. The final operation is to move the temp file into place for
// the object. This ensures the object semantics of last upload completed
// wins and is not some combination of writes from simultaneous uploads.
err := os.Rename(source, destination)
if err == nil || !errors.Is(err, syscall.EXDEV) {
return err
}
// Rename can fail if the source and destination are not on the same
// filesystem. The fallback is to copy the file and then remove the source.
// We need to be careful that the desination does not exist before copying
// to prevent any other simultaneous writes to the file.
sourceFile, err := os.Open(source)
if err != nil {
return fmt.Errorf("open source: %w", err)
}
defer sourceFile.Close()
var destFile *os.File
for {
destFile, err = os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
if err != nil {
if errors.Is(err, fs.ErrExist) {
if removeErr := os.Remove(destination); removeErr != nil {
return fmt.Errorf("remove existing destination: %w", removeErr)
}
continue
}
return fmt.Errorf("create destination: %w", err)
}
break
}
defer destFile.Close()
_, err = io.Copy(destFile, sourceFile)
if err != nil {
return fmt.Errorf("copy data: %w", err)
}
err = os.Remove(source)
if err != nil {
return fmt.Errorf("remove source: %w", err)
}
return nil
}
// GenerateEtag generates a new quoted etag from the provided hash.Hash
func GenerateEtag(h hash.Hash) string {
dataSum := h.Sum(nil)
return fmt.Sprintf("\"%s\"", hex.EncodeToString(dataSum[:]))
}
// AreEtagsSame compares 2 etags by ignoring quotes
func AreEtagsSame(e1, e2 string) bool {
return strings.Trim(e1, `"`) == strings.Trim(e2, `"`)
}
func getBoolPtr(b bool) *bool {
return &b
}
type PreConditions struct {
IfMatch *string
IfNoneMatch *string
IfModSince *time.Time
IfUnmodeSince *time.Time
}
// EvaluatePreconditions takes the object ETag, the last modified time and
// evaluates the read preconditions:
// - if-match,
// - if-none-match
// - if-modified-since
// - if-unmodified-since
// if-match and if-none-match are ETag comparisions
// if-modified-since and if-unmodified-since are last modifed time comparisons
func EvaluatePreconditions(etag string, modTime time.Time, preconditions PreConditions) error {
if preconditions.IfMatch == nil && preconditions.IfNoneMatch == nil && preconditions.IfModSince == nil && preconditions.IfUnmodeSince == nil {
return nil
}
// convert all conditions to *bool to evaluate the conditions
var ifMatch, ifNoneMatch, ifModSince, ifUnmodeSince *bool
if preconditions.IfMatch != nil {
ifMatch = getBoolPtr(*preconditions.IfMatch == etag)
}
if preconditions.IfNoneMatch != nil {
ifNoneMatch = getBoolPtr(*preconditions.IfNoneMatch != etag)
}
if preconditions.IfModSince != nil {
ifModSince = getBoolPtr(preconditions.IfModSince.UTC().Before(modTime.UTC()))
}
if preconditions.IfUnmodeSince != nil {
ifUnmodeSince = getBoolPtr(preconditions.IfUnmodeSince.UTC().After(modTime.UTC()))
}
if ifMatch != nil {
// if `if-match` doesn't matches, return PreconditionFailed
if !*ifMatch {
return errPreconditionFailed
}
// if-match matches
if *ifMatch {
if ifNoneMatch != nil {
// if `if-none-match` doesn't match return NotModified
if !*ifNoneMatch {
return errNotModified
}
// if both `if-match` and `if-none-match` match, return no error
return nil
}
// if `if-match` matches but `if-modified-since` is false return NotModified
if ifModSince != nil && !*ifModSince {
return errNotModified
}
// ignore `if-unmodified-since` as `if-match` is true
return nil
}
}
if ifNoneMatch != nil {
if *ifNoneMatch {
// if `if-none-match` is true, but `if-unmodified-since` is false
// return PreconditionFailed
if ifUnmodeSince != nil && !*ifUnmodeSince {
return errPreconditionFailed
}
// ignore `if-modified-since` as `if-none-match` is true
return nil
} else {
// if `if-none-match` is false and `if-unmodified-since` is false
// return PreconditionFailed
if ifUnmodeSince != nil && !*ifUnmodeSince {
return errPreconditionFailed
}
// in all other cases when `if-none-match` is false return NotModified
return errNotModified
}
}
if ifModSince != nil && !*ifModSince {
// if both `if-modified-since` and `if-unmodified-since` are false
// return PreconditionFailed
if ifUnmodeSince != nil && !*ifUnmodeSince {
return errPreconditionFailed
}
// if only `if-modified-since` is false, return NotModified
return errNotModified
}
// if `if-unmodified-since` is false return PreconditionFailed
if ifUnmodeSince != nil && !*ifUnmodeSince {
return errPreconditionFailed
}
return nil
}
// EvaluateMatchPreconditions evaluates if-match and if-none-match preconditions
func EvaluateMatchPreconditions(etag string, ifMatch, ifNoneMatch *string) error {
if ifMatch != nil && *ifMatch != etag {
return errPreconditionFailed
}
if ifNoneMatch != nil && *ifNoneMatch == etag {
return errPreconditionFailed
}
return nil
}
type ObjectDeletePreconditions struct {
IfMatch *string
IfMatchLastModTime *time.Time
IfMatchSize *int64
}
// EvaluateObjectDeletePreconditions evaluates preconditions for DeleteObject
func EvaluateObjectDeletePreconditions(etag string, modTime time.Time, size int64, preconditions ObjectDeletePreconditions) error {
ifMatch := preconditions.IfMatch
if ifMatch != nil && *ifMatch != etag {
return errPreconditionFailed
}
ifMatchTime := preconditions.IfMatchLastModTime
if ifMatchTime != nil && ifMatchTime.Unix() != modTime.Unix() {
return errPreconditionFailed
}
ifMatchSize := preconditions.IfMatchSize
if ifMatchSize != nil && *ifMatchSize != size {
return errPreconditionFailed
}
return nil
}

View File

@@ -1,42 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package meta
import "os"
// MetadataStorer defines the interface for managing metadata.
// When object == "", the operation is on the bucket.
type MetadataStorer interface {
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
// Returns the value of the attribute, or an error if the attribute does not exist.
RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error)
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
// If attribute already exists, new attribute should replace existing.
// Returns an error if the operation fails.
StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
// Returns an error if the operation fails.
DeleteAttribute(bucket, object, attribute string) error
// ListAttributes lists all attributes for an object or a bucket.
// Returns list of attribute names, or an error if the operation fails.
ListAttributes(bucket, object string) ([]string, error)
// DeleteAttributes removes all attributes for an object or a bucket.
// Returns an error if the operation fails.
DeleteAttributes(bucket, object string) error
}

View File

@@ -1,54 +0,0 @@
// Copyright 2025 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package meta
import (
"os"
)
// NoMeta is a metadata storer that does not store metadata.
// This can be useful for read only mounts where attempting to store metadata
// would fail.
type NoMeta struct{}
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
// always returns ErrNoSuchKey
func (NoMeta) RetrieveAttribute(_ *os.File, _, _, _ string) ([]byte, error) {
return nil, ErrNoSuchKey
}
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
// always returns nil without storing the attribute
func (NoMeta) StoreAttribute(_ *os.File, _, _, _ string, _ []byte) error {
return nil
}
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
// always returns nil without deleting the attribute
func (NoMeta) DeleteAttribute(_, _, _ string) error {
return nil
}
// ListAttributes lists all attributes for an object or a bucket.
// always returns an empty list of attributes
func (NoMeta) ListAttributes(_, _ string) ([]string, error) {
return []string{}, nil
}
// DeleteAttributes removes all attributes for an object or a bucket.
// always returns nil without deleting any attributes
func (NoMeta) DeleteAttributes(bucket, object string) error {
return nil
}

View File

@@ -1,139 +0,0 @@
// Copyright 2025 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package meta
import (
"errors"
"fmt"
"os"
"path/filepath"
)
// SideCar is a metadata storer that uses sidecar files to store metadata.
type SideCar struct {
dir string
}
const (
sidecarmeta = "meta"
)
// NewSideCar creates a new SideCar metadata storer.
func NewSideCar(dir string) (SideCar, error) {
fi, err := os.Lstat(dir)
if err != nil {
return SideCar{}, fmt.Errorf("failed to stat directory: %v", err)
}
if !fi.IsDir() {
return SideCar{}, fmt.Errorf("not a directory")
}
return SideCar{dir: dir}, nil
}
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) {
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
}
attr := filepath.Join(metadir, attribute)
value, err := os.ReadFile(attr)
if errors.Is(err, os.ErrNotExist) {
return nil, ErrNoSuchKey
}
if err != nil {
return nil, fmt.Errorf("failed to read attribute: %v", err)
}
return value, nil
}
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error {
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
}
err := os.MkdirAll(metadir, 0777)
if err != nil {
return fmt.Errorf("failed to create metadata directory: %v", err)
}
attr := filepath.Join(metadir, attribute)
err = os.WriteFile(attr, value, 0666)
if err != nil {
return fmt.Errorf("failed to write attribute: %v", err)
}
return nil
}
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
}
attr := filepath.Join(metadir, attribute)
err := os.Remove(attr)
if errors.Is(err, os.ErrNotExist) {
return ErrNoSuchKey
}
if err != nil {
return fmt.Errorf("failed to remove attribute: %v", err)
}
return nil
}
// ListAttributes lists all attributes for an object or a bucket.
func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
}
ents, err := os.ReadDir(metadir)
if errors.Is(err, os.ErrNotExist) {
return []string{}, nil
}
if err != nil {
return nil, fmt.Errorf("failed to list attributes: %v", err)
}
var attrs []string
for _, ent := range ents {
attrs = append(attrs, ent.Name())
}
return attrs, nil
}
// DeleteAttributes removes all attributes for an object or a bucket.
func (s SideCar) DeleteAttributes(bucket, object string) error {
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
}
err := os.RemoveAll(metadir)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("failed to remove attributes: %v", err)
}
return nil
}

View File

@@ -1,126 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package meta
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/pkg/xattr"
"github.com/versity/versitygw/s3err"
)
const (
xattrPrefix = "user."
)
var (
// ErrNoSuchKey is returned when the key does not exist.
ErrNoSuchKey = errors.New("no such key")
)
type XattrMeta struct{}
// RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
func (x XattrMeta) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) {
if f != nil {
b, err := xattr.FGet(f, xattrPrefix+attribute)
if errors.Is(err, xattr.ENOATTR) {
return nil, ErrNoSuchKey
}
return b, err
}
b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
if errors.Is(err, xattr.ENOATTR) {
return nil, ErrNoSuchKey
}
return b, err
}
// StoreAttribute stores the value of a specific attribute for an object in a bucket.
func (x XattrMeta) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error {
if f != nil {
err := xattr.FSet(f, xattrPrefix+attribute, value)
if errors.Is(err, syscall.EROFS) {
return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return err
}
err := xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
if errors.Is(err, syscall.EROFS) {
return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return err
}
// DeleteAttribute removes the value of a specific attribute for an object in a bucket.
func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
err := xattr.Remove(filepath.Join(bucket, object), xattrPrefix+attribute)
if errors.Is(err, xattr.ENOATTR) {
return ErrNoSuchKey
}
if errors.Is(err, syscall.EROFS) {
return s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return err
}
// DeleteAttributes is not implemented for xattr since xattrs
// are automatically removed when the file is deleted.
func (x XattrMeta) DeleteAttributes(bucket, object string) error {
return nil
}
// ListAttributes lists all attributes for an object in a bucket.
func (x XattrMeta) ListAttributes(bucket, object string) ([]string, error) {
attrs, err := xattr.List(filepath.Join(bucket, object))
if err != nil {
return nil, err
}
attributes := make([]string, 0, len(attrs))
for _, attr := range attrs {
if !isUserAttr(attr) {
continue
}
attributes = append(attributes, strings.TrimPrefix(attr, xattrPrefix))
}
return attributes, nil
}
func isUserAttr(attr string) bool {
return strings.HasPrefix(attr, xattrPrefix)
}
// Test is a helper function to test if xattrs are supported.
func (x XattrMeta) Test(path string) error {
// check for platform support
if !xattr.XATTR_SUPPORTED {
return fmt.Errorf("xattrs are not supported on this platform")
}
// check if the filesystem supports xattrs
_, err := xattr.Get(path, "user.test")
if errors.Is(err, syscall.ENOTSUP) {
return fmt.Errorf("xattrs are not supported on this filesystem")
}
return nil
}

View File

@@ -1,77 +0,0 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Copyright 2024 Versity Software
// MkdirAll borrowed from stdlib to add ability to set ownership
// as directories are created
package backend
import (
"io/fs"
"os"
"github.com/versity/versitygw/s3err"
)
// MkdirAll is similar to os.MkdirAll but it will return
// ErrObjectParentIsFile when appropriate
// MkdirAll creates a directory named path,
// along with any necessary parents, and returns nil,
// or else returns an error.
// The permission bits perm (before umask) are used for all
// directories that MkdirAll creates.
// Any newly created directory is set to provided uid/gid ownership.
// If path is already a directory, MkdirAll does nothing
// and returns nil.
// Any directory created will be set to provided uid/gid ownership
// if doChown is true.
func MkdirAll(path string, uid, gid int, doChown bool, dirPerm fs.FileMode) error {
// Fast path: if we can tell whether path is a directory or file, stop with success or error.
dir, err := os.Stat(path)
if err == nil {
if dir.IsDir() {
return nil
}
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
}
// Slow path: make sure parent exists and then call Mkdir for path.
i := len(path)
for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
i--
}
j := i
for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
j--
}
if j > 1 {
// Create parent.
err = MkdirAll(path[:j-1], uid, gid, doChown, dirPerm)
if err != nil {
return err
}
}
// Parent now exists; invoke Mkdir and use its result.
err = os.Mkdir(path, dirPerm)
if err != nil {
// Handle arguments like "foo/." by
// double-checking that directory doesn't exist.
dir, err1 := os.Lstat(path)
if err1 == nil && dir.IsDir() {
return nil
}
return err
}
if doChown {
err = os.Chown(path, uid, gid)
if err != nil {
return err
}
}
return nil
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,272 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build linux
// +build linux
package posix
import (
"crypto/sha256"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strconv"
"syscall"
"time"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
"golang.org/x/sys/unix"
)
const procfddir = "/proc/self/fd"
type tmpfile struct {
f *os.File
bucket string
objname string
isOTmp bool
size int64
needsChown bool
uid int
gid int
newDirPerm fs.FileMode
}
var (
// TODO: make this configurable
defaultFilePerm uint32 = 0644
)
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, dofalloc bool, forceNoTmpFile bool) (*tmpfile, error) {
uid, gid, doChown := p.getChownIDs(acct)
if forceNoTmpFile {
return p.openMkTemp(dir, bucket, obj, size, dofalloc, uid, gid, doChown)
}
// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
// This can help reduce contention within the namespace (parent directories),
// etc. And will auto cleanup the inode on close if we never link this
// file descriptor into the namespace.
// Not all filesystems support this, so fallback to CreateTemp for when
// this is not supported.
fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
if err != nil {
if errors.Is(err, syscall.EROFS) {
return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
// O_TMPFILE not supported, try fallback
return p.openMkTemp(dir, bucket, obj, size, dofalloc, uid, gid, doChown)
}
// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
// later to link file into namespace
f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
tmp := &tmpfile{
f: f,
bucket: bucket,
objname: obj,
isOTmp: true,
size: size,
needsChown: doChown,
uid: uid,
gid: gid,
newDirPerm: p.newDirPerm,
}
// falloc is best effort, its fine if this fails
if size > 0 && dofalloc {
tmp.falloc()
}
if doChown {
err := f.Chown(uid, gid)
if err != nil {
return nil, fmt.Errorf("set temp file ownership: %w", err)
}
}
return tmp, nil
}
func (p *Posix) openMkTemp(dir, bucket, obj string, size int64, dofalloc bool, uid, gid int, doChown bool) (*tmpfile, error) {
err := backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
if err != nil {
if errors.Is(err, syscall.EROFS) {
return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return nil, fmt.Errorf("make temp dir: %w", err)
}
f, err := os.CreateTemp(dir,
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
if err != nil {
if errors.Is(err, syscall.EROFS) {
return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return nil, err
}
tmp := &tmpfile{
f: f,
bucket: bucket,
objname: obj,
size: size,
needsChown: doChown,
uid: uid,
gid: gid,
}
// falloc is best effort, its fine if this fails
if size > 0 && dofalloc {
tmp.falloc()
}
if doChown {
err := f.Chown(uid, gid)
if err != nil {
return nil, fmt.Errorf("set temp file ownership: %w", err)
}
}
return tmp, nil
}
func (tmp *tmpfile) falloc() error {
err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
if err != nil {
return fmt.Errorf("fallocate: %v", err)
}
return nil
}
func (tmp *tmpfile) link() error {
// make sure this is cleaned up in all error cases
defer tmp.f.Close()
// We use Linkat/Rename as the atomic operation for object puts. The
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
// with any other simultaneous uploads. The final operation is to move the
// temp file into place for the object. This ensures the object semantics
// of last upload completed wins and is not some combination of writes
// from simultaneous uploads.
objPath := filepath.Join(tmp.bucket, tmp.objname)
dir := filepath.Dir(objPath)
err := backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown, tmp.newDirPerm)
if err != nil {
return fmt.Errorf("make parent dir: %w", err)
}
if !tmp.isOTmp {
// O_TMPFILE not suported, use fallback
return tmp.fallbackLink()
}
procdir, err := os.Open(procfddir)
if err != nil {
return fmt.Errorf("open proc dir: %w", err)
}
defer procdir.Close()
dirf, err := os.Open(dir)
if err != nil {
return fmt.Errorf("open parent dir: %w", err)
}
defer dirf.Close()
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
if errors.Is(err, syscall.EEXIST) {
// Linkat cannot overwrite files; we will allocate a temporary file, Linkat to it and then Renameat it
// to avoid potential race condition
retries := 1
for {
tmpName := fmt.Sprintf(".%s.sgwtmp.%d", filepath.Base(objPath), time.Now().UnixNano())
err := unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
int(dirf.Fd()), tmpName, unix.AT_SYMLINK_FOLLOW)
if errors.Is(err, syscall.EEXIST) && retries < 3 {
retries += 1
continue
}
if err != nil {
return fmt.Errorf("cannot find free temporary file: %w", err)
}
err = unix.Renameat(int(dirf.Fd()), tmpName, int(dirf.Fd()), filepath.Base(objPath))
if err != nil {
return fmt.Errorf("overwriting renameat failed: %w", err)
}
break
}
} else if err != nil {
return fmt.Errorf("link tmpfile (fd %q as %q): %w",
filepath.Base(tmp.f.Name()), objPath, err)
}
err = tmp.f.Close()
if err != nil {
return fmt.Errorf("close tmpfile: %w", err)
}
return nil
}
func (tmp *tmpfile) fallbackLink() error {
tempname := tmp.f.Name()
// cleanup in case anything goes wrong, if rename succeeds then
// this will no longer exist
defer os.Remove(tempname)
// reset default file mode because CreateTemp uses 0600
tmp.f.Chmod(fs.FileMode(defaultFilePerm))
err := tmp.f.Close()
if err != nil {
return fmt.Errorf("close tmpfile: %w", err)
}
objPath := filepath.Join(tmp.bucket, tmp.objname)
err = os.Rename(tempname, objPath)
if err != nil {
// rename only works for files within the same filesystem
// if this fails fallback to copy
return backend.MoveFile(tempname, objPath, fs.FileMode(defaultFilePerm))
}
return nil
}
func (tmp *tmpfile) Write(b []byte) (int, error) {
if int64(len(b)) > tmp.size {
return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
}
n, err := tmp.f.Write(b)
tmp.size -= int64(n)
return n, err
}
func (tmp *tmpfile) cleanup() {
tmp.f.Close()
}
func (tmp *tmpfile) File() *os.File {
return tmp.f
}

View File

@@ -1,112 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build !linux
// +build !linux
package posix
import (
"crypto/sha256"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"syscall"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
)
type tmpfile struct {
f *os.File
bucket string
objname string
size int64
}
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account, _ bool, _ bool) (*tmpfile, error) {
uid, gid, doChown := p.getChownIDs(acct)
// Create a temp file for upload while in progress (see link comments below).
var err error
err = backend.MkdirAll(dir, uid, gid, doChown, p.newDirPerm)
if err != nil {
if errors.Is(err, syscall.EROFS) {
return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return nil, fmt.Errorf("make temp dir: %w", err)
}
f, err := os.CreateTemp(dir,
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
if err != nil {
if errors.Is(err, syscall.EROFS) {
return nil, s3err.GetAPIError(s3err.ErrMethodNotAllowed)
}
return nil, fmt.Errorf("create temp file: %w", err)
}
if doChown {
err := f.Chown(uid, gid)
if err != nil {
return nil, fmt.Errorf("set temp file ownership: %w", err)
}
}
return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
}
var (
// TODO: make this configurable
defaultFilePerm fs.FileMode = 0644
)
func (tmp *tmpfile) link() error {
tempname := tmp.f.Name()
// cleanup in case anything goes wrong, if rename succeeds then
// this will no longer exist
defer os.Remove(tempname)
objPath := filepath.Join(tmp.bucket, tmp.objname)
// reset default file mode because CreateTemp uses 0600
tmp.f.Chmod(defaultFilePerm)
err := tmp.f.Close()
if err != nil {
return fmt.Errorf("close tmpfile: %w", err)
}
return backend.MoveFile(tempname, objPath, defaultFilePerm)
}
func (tmp *tmpfile) Write(b []byte) (int, error) {
if int64(len(b)) > tmp.size {
return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
}
n, err := tmp.f.Write(b)
tmp.size -= int64(n)
return n, err
}
func (tmp *tmpfile) cleanup() {
tmp.f.Close()
}
func (tmp *tmpfile) File() *os.File {
return tmp.f
}

View File

@@ -1,75 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3proxy
import (
"context"
"crypto/tls"
"net/http"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/smithy-go/middleware"
)
func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
cfg, err := s.getConfig(ctx, s.access, s.secret)
if err != nil {
return nil, err
}
if s.endpoint != "" {
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = &s.endpoint
o.UsePathStyle = s.usePathStyle
// The http body stream is not seekable, so most operations cannot
// be retried. The error returned to the original client may be
// retried by the client.
o.Retryer = aws.NopRetryer{}
}), nil
}
return s3.NewFromConfig(cfg), nil
}
func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
creds := credentials.NewStaticCredentialsProvider(access, secret, "")
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
}
client := &http.Client{Transport: tr}
opts := []func(*config.LoadOptions) error{
config.WithRegion(s.awsRegion),
config.WithCredentialsProvider(creds),
config.WithHTTPClient(client),
}
if s.disableChecksum {
opts = append(opts,
config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
}
if s.debug {
opts = append(opts,
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
}
return config.LoadDefaultConfig(ctx, opts...)
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,372 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package scoutfs
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/pkg/xattr"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/backend/posix"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
// ScoutfsOpts are the options for the ScoutFS backend
type ScoutfsOpts struct {
// ChownUID sets the UID of the object to the UID of the user on PUT
ChownUID bool
// ChownGID sets the GID of the object to the GID of the user on PUT
ChownGID bool
// BucketLinks enables symlinks to directories to be treated as buckets
BucketLinks bool
//VersioningDir sets the version directory to enable object versioning
VersioningDir string
// NewDirPerm specifies the permission to set on newly created directories
NewDirPerm fs.FileMode
// GlacierMode enables glacier emulation for offline files
GlacierMode bool
// DisableNoArchive prevents setting noarchive on temporary files
DisableNoArchive bool
}
type ScoutFS struct {
*posix.Posix
rootfd *os.File
rootdir string
// glaciermode enables the following behavior:
// GET object: if file offline, return invalid object state
// HEAD object: if file offline, set obj storage class to GLACIER
// if file offline and staging, x-amz-restore: ongoing-request="true"
// if file offline and not staging, x-amz-restore: ongoing-request="false"
// if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
// note: this expiry-date is not used but provided for client glacier compatibility
// ListObjects: if file offline, set obj storage class to GLACIER
// RestoreObject: add batch stage request to file
glaciermode bool
// disableNoArchive is used to disable setting scoutam noarchive flag
// on mutlipart parts. This is enabled by default to prevent archive
// copies of temporary multipart parts.
disableNoArchive bool
}
var _ backend.Backend = &ScoutFS{}
const (
stageComplete = "ongoing-request=\"false\", expiry-date=\"Fri, 2 Dec 2050 00:00:00 GMT\""
stageInProgress = "true"
stageNotInProgress = "false"
)
const (
// ScoutFS special xattr types
systemPrefix = "scoutfs.hide."
onameAttr = systemPrefix + "objname"
flagskey = systemPrefix + "sam_flags"
stagecopykey = systemPrefix + "sam_stagereq"
)
const (
// ScoutAM Flags
// Staging - file requested stage
Staging uint64 = 1 << iota
// StageFail - all copies failed to stage
StageFail
// NoArchive - no archive copies of file should be made
NoArchive
// ExtCacheRequested means file policy requests Ext Cache
ExtCacheRequested
// ExtCacheDone means this file ext cache copy has been
// created already (and possibly pruned, so may not exist)
ExtCacheDone
)
func (s *ScoutFS) Shutdown() {
s.Posix.Shutdown()
s.rootfd.Close()
_ = s.rootdir
}
func (*ScoutFS) String() string {
return "ScoutFS Gateway"
}
func (s *ScoutFS) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
out, err := s.Posix.UploadPart(ctx, input)
if err != nil {
return nil, err
}
if !s.disableNoArchive {
sum := sha256.Sum256([]byte(*input.Key))
partPath := filepath.Join(
*input.Bucket, // bucket
posix.MetaTmpMultipartDir, // temp multipart dir
fmt.Sprintf("%x", sum), // hashed objname
*input.UploadId, // upload id
fmt.Sprintf("%v", *input.PartNumber), // part number
)
err = setNoArchive(partPath)
if err != nil {
return nil, fmt.Errorf("set noarchive: %w", err)
}
}
return out, err
}
// CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
// ioctl to not have to read and copy the part data to the final object. This
// saves a read and write cycle for all mutlipart uploads.
func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) {
return s.Posix.CompleteMultipartUploadWithCopy(ctx, input, moveData)
}
func (s *ScoutFS) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
res, err := s.Posix.HeadObject(ctx, input)
if err != nil {
return nil, err
}
if s.glaciermode {
objPath := filepath.Join(*input.Bucket, *input.Key)
stclass := types.StorageClassStandard
requestOngoing := ""
requestOngoing = stageComplete
// Check if there are any offline exents associated with this file.
// If so, we will set storage class to glacier.
st, err := statMore(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return nil, fmt.Errorf("stat more: %w", err)
}
if st.Offline_blocks != 0 {
stclass = types.StorageClassGlacier
requestOngoing = stageNotInProgress
ok, err := isStaging(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return nil, fmt.Errorf("check stage status: %w", err)
}
if ok {
requestOngoing = stageInProgress
}
}
res.Restore = &requestOngoing
res.StorageClass = stclass
}
return res, nil
}
func (s *ScoutFS) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
bucket := *input.Bucket
object := *input.Key
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return nil, fmt.Errorf("stat bucket: %w", err)
}
objPath := filepath.Join(bucket, object)
fi, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
}
if err != nil {
return nil, fmt.Errorf("stat object: %w", err)
}
if strings.HasSuffix(object, "/") && !fi.IsDir() {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if s.glaciermode {
// Check if there are any offline exents associated with this file.
// If so, we will return the InvalidObjectState error.
st, err := statMore(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return nil, fmt.Errorf("stat more: %w", err)
}
if st.Offline_blocks != 0 {
return nil, s3err.GetAPIError(s3err.ErrInvalidObjectState)
}
}
return s.Posix.GetObject(ctx, input)
}
func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) {
if s.glaciermode {
return s.Posix.ListObjectsParametrized(ctx, input, s.glacierFileToObj)
} else {
return s.Posix.ListObjects(ctx, input)
}
}
func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) {
if s.glaciermode {
return s.Posix.ListObjectsV2Parametrized(ctx, input, s.glacierFileToObj)
} else {
return s.Posix.ListObjectsV2(ctx, input)
}
}
// FileToObj function for ListObject calls that adds a Glacier storage class if the file is offline
func (s *ScoutFS) glacierFileToObj(bucket string, fetchOwner bool) backend.GetObjFunc {
posixFileToObj := s.Posix.FileToObj(bucket, fetchOwner)
return func(path string, d fs.DirEntry) (s3response.Object, error) {
res, err := posixFileToObj(path, d)
if err != nil || d.IsDir() {
return res, err
}
objPath := filepath.Join(bucket, path)
// Check if there are any offline exents associated with this file.
// If so, we will return the Glacier storage class
st, err := statMore(objPath)
if errors.Is(err, fs.ErrNotExist) {
return s3response.Object{}, backend.ErrSkipObj
}
if err != nil {
return s3response.Object{}, fmt.Errorf("stat more: %w", err)
}
if st.Offline_blocks != 0 {
res.StorageClass = types.ObjectStorageClassGlacier
}
return res, nil
}
}
// RestoreObject will set stage request on file if offline and do nothing if
// file is online
func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) error {
bucket := *input.Bucket
object := *input.Key
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return fmt.Errorf("stat bucket: %w", err)
}
err = setStaging(filepath.Join(bucket, object))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return fmt.Errorf("stage object: %w", err)
}
return nil
}
func isStaging(objname string) (bool, error) {
b, err := xattr.Get(objname, flagskey)
if err != nil && !isNoAttr(err) {
return false, err
}
var flags uint64
if !isNoAttr(err) {
err = json.Unmarshal(b, &flags)
if err != nil {
return false, err
}
}
return flags&Staging == Staging, nil
}
func setFlag(objname string, flag uint64) error {
b, err := xattr.Get(objname, flagskey)
if err != nil && !isNoAttr(err) {
return err
}
var oldflags uint64
if !isNoAttr(err) {
err = json.Unmarshal(b, &oldflags)
if err != nil {
return err
}
}
newflags := oldflags | flag
if newflags == oldflags {
// no flags change, just return
return nil
}
b, err = json.Marshal(&newflags)
if err != nil {
return err
}
return xattr.Set(objname, flagskey, b)
}
func setStaging(objname string) error {
return setFlag(objname, Staging)
}
func setNoArchive(objname string) error {
return setFlag(objname, NoArchive)
}
func isNoAttr(err error) bool {
xerr, ok := err.(*xattr.Error)
if ok && xerr.Err == xattr.ENOATTR {
return true
}
return false
}

View File

@@ -1,94 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build linux && amd64
package scoutfs
import (
"fmt"
"os"
"github.com/versity/scoutfs-go"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/backend/posix"
"github.com/versity/versitygw/debuglogger"
)
func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
metastore := meta.XattrMeta{}
p, err := posix.New(rootdir, metastore, posix.PosixOpts{
ChownUID: opts.ChownUID,
ChownGID: opts.ChownGID,
BucketLinks: opts.BucketLinks,
NewDirPerm: opts.NewDirPerm,
VersioningDir: opts.VersioningDir,
})
if err != nil {
return nil, err
}
f, err := os.Open(rootdir)
if err != nil {
return nil, fmt.Errorf("open %v: %w", rootdir, err)
}
return &ScoutFS{
Posix: p,
rootfd: f,
rootdir: rootdir,
glaciermode: opts.GlacierMode,
disableNoArchive: opts.DisableNoArchive,
}, nil
}
func moveData(from *os.File, to *os.File) error {
// May fail if the files are not 4K aligned; check for alignment
ffi, err := from.Stat()
if err != nil {
return fmt.Errorf("stat from: %v", err)
}
tfi, err := to.Stat()
if err != nil {
return fmt.Errorf("stat to: %v", err)
}
if ffi.Size()%4096 != 0 || tfi.Size()%4096 != 0 {
return os.ErrInvalid
}
err = scoutfs.MoveData(from, to)
if err != nil {
debuglogger.Logf("ScoutFs MoveData failed: %v", err)
}
return err
}
func statMore(path string) (stat, error) {
st, err := scoutfs.StatMore(path)
if err != nil {
return stat{}, err
}
var s stat
s.Meta_seq = st.Meta_seq
s.Data_seq = st.Data_seq
s.Data_version = st.Data_version
s.Online_blocks = st.Online_blocks
s.Offline_blocks = st.Offline_blocks
s.Crtime_sec = st.Crtime_sec
s.Crtime_nsec = st.Crtime_nsec
return s, nil
}

View File

@@ -1,39 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build !(linux && amd64)
package scoutfs
import (
"errors"
"fmt"
"os"
)
func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
return nil, fmt.Errorf("scoutfs only available on linux")
}
var (
errNotSupported = errors.New("not supported")
)
func moveData(_, _ *os.File) error {
return errNotSupported
}
func statMore(_ string) (stat, error) {
return stat{}, errNotSupported
}

View File

@@ -1,25 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package scoutfs
type stat struct {
Meta_seq uint64
Data_seq uint64
Data_version uint64
Online_blocks uint64
Offline_blocks uint64
Crtime_sec uint64
Crtime_nsec uint32
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,550 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package main
import (
"bytes"
"crypto/sha256"
"crypto/tls"
"encoding/hex"
"encoding/xml"
"fmt"
"io"
"net/http"
"os"
"text/tabwriter"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/smithy-go"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3response"
)
var (
adminAccess string
adminSecret string
adminRegion string
adminEndpoint string
allowInsecure bool
)
func adminCommand() *cli.Command {
return &cli.Command{
Name: "admin",
Usage: "admin CLI tool",
Description: `Admin CLI tool for interacting with admin APIs.`,
Subcommands: []*cli.Command{
{
Name: "create-user",
Usage: "Create a new user",
Action: createUser,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "access",
Usage: "access key id for the new user",
Required: true,
Aliases: []string{"a"},
},
&cli.StringFlag{
Name: "secret",
Usage: "secret access key for the new user",
Required: true,
Aliases: []string{"s"},
},
&cli.StringFlag{
Name: "role",
Usage: "role for the new user",
Required: true,
Aliases: []string{"r"},
},
&cli.IntFlag{
Name: "user-id",
Usage: "userID for the new user",
Aliases: []string{"ui"},
},
&cli.IntFlag{
Name: "group-id",
Usage: "groupID for the new user",
Aliases: []string{"gi"},
},
},
},
{
Name: "update-user",
Usage: "Updates a user account",
Action: updateUser,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "access",
Usage: "user access key id to be updated",
Required: true,
Aliases: []string{"a"},
},
&cli.StringFlag{
Name: "secret",
Usage: "secret access key for the new user",
Aliases: []string{"s"},
},
&cli.StringFlag{
Name: "role",
Usage: "the new user role",
Aliases: []string{"r"},
},
&cli.IntFlag{
Name: "user-id",
Usage: "userID for the new user",
Aliases: []string{"ui"},
},
&cli.IntFlag{
Name: "group-id",
Usage: "groupID for the new user",
Aliases: []string{"gi"},
},
},
},
{
Name: "delete-user",
Usage: "Delete a user",
Action: deleteUser,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "access",
Usage: "access key id of the user to be deleted",
Required: true,
Aliases: []string{"a"},
},
},
},
{
Name: "list-users",
Usage: "List all the gateway users",
Action: listUsers,
},
{
Name: "change-bucket-owner",
Usage: "Changes the bucket owner",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "bucket",
Usage: "the bucket name to change the owner",
Required: true,
Aliases: []string{"b"},
},
&cli.StringFlag{
Name: "owner",
Usage: "the user access key id, who should be the bucket owner",
Required: true,
Aliases: []string{"o"},
},
},
Action: changeBucketOwner,
},
{
Name: "list-buckets",
Usage: "Lists all the gateway buckets and owners.",
Action: listBuckets,
},
},
Flags: []cli.Flag{
// TODO: create a configuration file for this
&cli.StringFlag{
Name: "access",
Usage: "admin access key id",
EnvVars: []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"},
Aliases: []string{"a"},
Required: true,
Destination: &adminAccess,
},
&cli.StringFlag{
Name: "secret",
Usage: "admin secret access key",
EnvVars: []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"},
Aliases: []string{"s"},
Required: true,
Destination: &adminSecret,
},
&cli.StringFlag{
Name: "region",
Usage: "admin s3 region string",
EnvVars: []string{"ADMIN_REGION"},
Value: "us-east-1",
Destination: &adminRegion,
Aliases: []string{"r"},
},
&cli.StringFlag{
Name: "endpoint-url",
Usage: "admin apis endpoint url",
EnvVars: []string{"ADMIN_ENDPOINT_URL"},
Aliases: []string{"er"},
Required: true,
Destination: &adminEndpoint,
},
&cli.BoolFlag{
Name: "allow-insecure",
Usage: "disable tls certificate verification for the admin endpoint",
EnvVars: []string{"ADMIN_ALLOW_INSECURE"},
Aliases: []string{"ai"},
Destination: &allowInsecure,
},
},
}
}
func initHTTPClient() *http.Client {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: allowInsecure},
}
return &http.Client{Transport: tr}
}
func createUser(ctx *cli.Context) error {
access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
userID, groupID := ctx.Int("user-id"), ctx.Int("group-id")
if access == "" || secret == "" {
return fmt.Errorf("invalid input parameters for the new user access/secret keys")
}
if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
return fmt.Errorf("invalid input parameter for role: %v", role)
}
acc := auth.Account{
Access: access,
Secret: secret,
Role: auth.Role(role),
UserID: userID,
GroupID: groupID,
}
accxml, err := xml.Marshal(acc)
if err != nil {
return fmt.Errorf("failed to parse user data: %w", err)
}
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accxml))
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256(accxml)
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode >= 400 {
return parseApiError(body)
}
return nil
}
func deleteUser(ctx *cli.Context) error {
access := ctx.String("access")
if access == "" {
return fmt.Errorf("invalid input parameter for the user access key")
}
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode >= 400 {
return parseApiError(body)
}
return nil
}
func updateUser(ctx *cli.Context) error {
access, secret, userId, groupId, role := ctx.String("access"), ctx.String("secret"), ctx.Int("user-id"), ctx.Int("group-id"), auth.Role(ctx.String("role"))
props := auth.MutableProps{}
if ctx.IsSet("role") {
if !role.IsValid() {
return fmt.Errorf("invalid user role: %v", role)
}
props.Role = role
}
if ctx.IsSet("secret") {
props.Secret = &secret
}
if ctx.IsSet("user-id") {
props.UserID = &userId
}
if ctx.IsSet("group-id") {
props.GroupID = &groupId
}
propsxml, err := xml.Marshal(props)
if err != nil {
return fmt.Errorf("failed to parse user attributes: %w", err)
}
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/update-user?access=%v", adminEndpoint, access), bytes.NewBuffer(propsxml))
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256(propsxml)
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode >= 400 {
return parseApiError(body)
}
return nil
}
func listUsers(ctx *cli.Context) error {
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-users", adminEndpoint), nil)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode >= 400 {
return parseApiError(body)
}
var accs auth.ListUserAccountsResult
if err := xml.Unmarshal(body, &accs); err != nil {
return err
}
printAcctTable(accs.Accounts)
return nil
}
const (
// account table formatting
minwidth int = 2 // minimal cell width including any padding
tabwidth int = 0 // width of tab characters (equivalent number of spaces)
padding int = 2 // padding added to a cell before computing its width
padchar byte = ' ' // ASCII char used for padding
flags uint = 0 // formatting control flags
)
func printAcctTable(accs []auth.Account) {
w := new(tabwriter.Writer)
w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID")
fmt.Fprintln(w, "-------\t----\t------\t-------")
for _, acc := range accs {
fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID)
}
fmt.Fprintln(w)
w.Flush()
}
func changeBucketOwner(ctx *cli.Context) error {
bucket, owner := ctx.String("bucket"), ctx.String("owner")
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", adminEndpoint, bucket, owner), nil)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode >= 400 {
return parseApiError(body)
}
return nil
}
func printBuckets(buckets []s3response.Bucket) {
w := new(tabwriter.Writer)
w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
fmt.Fprintln(w, "Bucket\tOwner")
fmt.Fprintln(w, "-------\t----")
for _, acc := range buckets {
fmt.Fprintf(w, "%v\t%v\n", acc.Name, acc.Owner)
}
fmt.Fprintln(w)
w.Flush()
}
func listBuckets(ctx *cli.Context) error {
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", adminEndpoint), nil)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", adminRegion, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := initHTTPClient()
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode >= 400 {
return parseApiError(body)
}
var result s3response.ListBucketsResult
if err := xml.Unmarshal(body, &result); err != nil {
return err
}
printBuckets(result.Buckets)
return nil
}
func parseApiError(body []byte) error {
var apiErr smithy.GenericAPIError
err := xml.Unmarshal(body, &apiErr)
if err != nil {
apiErr.Code = "InternalServerError"
apiErr.Message = err.Error()
}
return &apiErr
}

View File

@@ -1,74 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package main
import (
"fmt"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/azure"
)
var (
azAccount, azKey, azServiceURL, azSASToken string
)
func azureCommand() *cli.Command {
return &cli.Command{
Name: "azure",
Usage: "azure blob storage backend",
Description: `direct translation from s3 objects to azure blobs`,
Action: runAzure,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "account",
Usage: "azure account name",
EnvVars: []string{"AZ_ACCOUNT_NAME"},
Aliases: []string{"a"},
Destination: &azAccount,
},
&cli.StringFlag{
Name: "access-key",
Usage: "azure account key",
EnvVars: []string{"AZ_ACCESS_KEY"},
Aliases: []string{"k"},
Destination: &azKey,
},
&cli.StringFlag{
Name: "sas-token",
Usage: "azure blob storage SAS token",
EnvVars: []string{"AZ_SAS_TOKEN"},
Aliases: []string{"st"},
Destination: &azSASToken,
},
&cli.StringFlag{
Name: "url",
Usage: "azure service URL",
EnvVars: []string{"AZ_ENDPOINT"},
Aliases: []string{"u"},
Destination: &azServiceURL,
},
},
}
}
func runAzure(ctx *cli.Context) error {
be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken)
if err != nil {
return fmt.Errorf("init azure: %w", err)
}
return runGateway(ctx.Context, be)
}

View File

@@ -1,111 +0,0 @@
package main
import (
"context"
"log"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/backend/posix"
"github.com/versity/versitygw/tests/integration"
)
const (
tdir = "tempdir"
)
var (
wg sync.WaitGroup
)
func initEnv(dir string) {
// both
debug = true
region = "us-east-1"
// server
rootUserAccess = "user"
rootUserSecret = "pass"
iamDir = dir
port = "127.0.0.1:7070"
// client
awsID = "user"
awsSecret = "pass"
endpoint = "http://127.0.0.1:7070"
}
func initPosix(ctx context.Context) {
path, err := os.Getwd()
if err != nil {
log.Fatalf("get current directory: %v", err)
}
tempdir := filepath.Join(path, tdir)
initEnv(tempdir)
err = os.RemoveAll(tempdir)
if err != nil {
log.Fatalf("remove temp directory: %v", err)
}
err = os.Mkdir(tempdir, 0755)
if err != nil {
log.Fatalf("make temp directory: %v", err)
}
be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{
NewDirPerm: 0755,
})
if err != nil {
log.Fatalf("init posix: %v", err)
}
wg.Add(1)
go func() {
err = runGateway(ctx, be)
if err != nil && err != context.Canceled {
log.Fatalf("run gateway: %v", err)
}
err := os.RemoveAll(tempdir)
if err != nil {
log.Fatalf("remove temp directory: %v", err)
}
wg.Done()
}()
// wait for server to start
time.Sleep(1 * time.Second)
}
func TestIntegration(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
initPosix(ctx)
opts := []integration.Option{
integration.WithAccess(awsID),
integration.WithSecret(awsSecret),
integration.WithRegion(region),
integration.WithEndpoint(endpoint),
}
if debug {
opts = append(opts, integration.WithDebug())
}
s := integration.NewS3Conf(opts...)
// replace below with desired test
err := integration.HeadBucket_non_existing_bucket(s)
if err != nil {
t.Error(err)
}
cancel()
wg.Wait()
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,74 +0,0 @@
// Copyright 2025 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package main
import (
"errors"
"fmt"
"plugin"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/plugins"
)
func pluginCommand() *cli.Command {
return &cli.Command{
Name: "plugin",
Usage: "load a backend from a plugin",
Description: "Runs a s3 gateway and redirects the requests to the backend defined in the plugin",
Action: runPluginBackend,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "config",
Usage: "location of the config file",
Aliases: []string{"c"},
},
},
}
}
func runPluginBackend(ctx *cli.Context) error {
if ctx.NArg() == 0 {
return fmt.Errorf("no plugin file provided to be loaded")
}
pluginPath := ctx.Args().Get(0)
config := ctx.String("config")
p, err := plugin.Open(pluginPath)
if err != nil {
return err
}
backendSymbol, err := p.Lookup("Backend")
if err != nil {
return err
}
backendPluginPtr, ok := backendSymbol.(*plugins.BackendPlugin)
if !ok {
return errors.New("plugin is not of type *plugins.BackendPlugin")
}
if backendPluginPtr == nil {
return errors.New("variable Backend is nil")
}
be, err := (*backendPluginPtr).New(config)
if err != nil {
return err
}
return runGateway(ctx.Context, be)
}

View File

@@ -1,156 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package main
import (
"fmt"
"io/fs"
"math"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/meta"
"github.com/versity/versitygw/backend/posix"
)
var (
chownuid, chowngid bool
bucketlinks bool
versioningDir string
dirPerms uint
sidecar string
nometa bool
forceNoTmpFile bool
)
func posixCommand() *cli.Command {
return &cli.Command{
Name: "posix",
Usage: "posix filesystem storage backend",
Description: `Any posix filesystem that supports extended attributes. The top level
directory for the gateway must be provided. All sub directories of the
top level directory are treated as buckets, and all files/directories
below the "bucket directory" are treated as the objects. The object
name is split on "/" separator to translate to posix storage.
For example:
top level: /mnt/fs/gwroot
bucket: mybucket
object: a/b/c/myobject
will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
Action: runPosix,
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "chuid",
Usage: "chown newly created files and directories to client account UID",
EnvVars: []string{"VGW_CHOWN_UID"},
Destination: &chownuid,
},
&cli.BoolFlag{
Name: "chgid",
Usage: "chown newly created files and directories to client account GID",
EnvVars: []string{"VGW_CHOWN_GID"},
Destination: &chowngid,
},
&cli.BoolFlag{
Name: "bucketlinks",
Usage: "allow symlinked directories at bucket level to be treated as buckets",
EnvVars: []string{"VGW_BUCKET_LINKS"},
Destination: &bucketlinks,
},
&cli.StringFlag{
Name: "versioning-dir",
Usage: "the directory path to enable bucket versioning",
EnvVars: []string{"VGW_VERSIONING_DIR"},
Destination: &versioningDir,
},
&cli.UintFlag{
Name: "dir-perms",
Usage: "default directory permissions for new directories",
EnvVars: []string{"VGW_DIR_PERMS"},
Destination: &dirPerms,
DefaultText: "0755",
Value: 0755,
},
&cli.StringFlag{
Name: "sidecar",
Usage: "use provided sidecar directory to store metadata",
EnvVars: []string{"VGW_META_SIDECAR"},
Destination: &sidecar,
},
&cli.BoolFlag{
Name: "nometa",
Usage: "disable metadata storage",
EnvVars: []string{"VGW_META_NONE"},
Destination: &nometa,
},
&cli.BoolFlag{
Name: "disableotmp",
Usage: "disable O_TMPFILE support for new objects",
EnvVars: []string{"VGW_DISABLE_OTMP"},
Destination: &forceNoTmpFile,
},
},
}
}
func runPosix(ctx *cli.Context) error {
if ctx.NArg() == 0 {
return fmt.Errorf("no directory provided for operation")
}
gwroot := (ctx.Args().Get(0))
if dirPerms > math.MaxUint32 {
return fmt.Errorf("invalid directory permissions: %d", dirPerms)
}
if nometa && sidecar != "" {
return fmt.Errorf("cannot use both nometa and sidecar metadata")
}
opts := posix.PosixOpts{
ChownUID: chownuid,
ChownGID: chowngid,
BucketLinks: bucketlinks,
VersioningDir: versioningDir,
NewDirPerm: fs.FileMode(dirPerms),
ForceNoTmpFile: forceNoTmpFile,
}
var ms meta.MetadataStorer
switch {
case sidecar != "":
sc, err := meta.NewSideCar(sidecar)
if err != nil {
return fmt.Errorf("failed to init sidecar metadata: %w", err)
}
ms = sc
opts.SideCarDir = sidecar
case nometa:
ms = meta.NoMeta{}
default:
ms = meta.XattrMeta{}
err := meta.XattrMeta{}.Test(gwroot)
if err != nil {
return fmt.Errorf("xattr check failed: %w", err)
}
}
be, err := posix.New(gwroot, ms, opts)
if err != nil {
return fmt.Errorf("failed to init posix backend: %w", err)
}
return runGateway(ctx.Context, be)
}

View File

@@ -1,121 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package main
import (
"fmt"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/s3proxy"
)
var (
s3proxyAccess string
s3proxySecret string
s3proxyEndpoint string
s3proxyRegion string
s3proxyMetaBucket string
s3proxyDisableChecksum bool
s3proxySslSkipVerify bool
s3proxyUsePathStyle bool
s3proxyDebug bool
)
func s3Command() *cli.Command {
return &cli.Command{
Name: "s3",
Usage: "s3 storage backend",
Description: `This runs the gateway like an s3 proxy redirecting requests
to an s3 storage backend service.`,
Action: runS3,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "access",
Usage: "s3 proxy server access key id",
Value: "",
Required: true,
EnvVars: []string{"VGW_S3_ACCESS_KEY"},
Destination: &s3proxyAccess,
Aliases: []string{"a"},
},
&cli.StringFlag{
Name: "secret",
Usage: "s3 proxy server secret access key",
Value: "",
Required: true,
EnvVars: []string{"VGW_S3_SECRET_KEY"},
Destination: &s3proxySecret,
Aliases: []string{"s"},
},
&cli.StringFlag{
Name: "endpoint",
Usage: "s3 service endpoint, default AWS if not specified",
Value: "",
EnvVars: []string{"VGW_S3_ENDPOINT"},
Destination: &s3proxyEndpoint,
},
&cli.StringFlag{
Name: "region",
Usage: "s3 service region, default 'us-east-1' if not specified",
Value: "us-east-1",
EnvVars: []string{"VGW_S3_REGION"},
Destination: &s3proxyRegion,
},
&cli.StringFlag{
Name: "meta-bucket",
Usage: "s3 service meta bucket to store buckets acl/policy",
EnvVars: []string{"VGW_S3_META_BUCKET"},
Destination: &s3proxyMetaBucket,
},
&cli.BoolFlag{
Name: "disable-checksum",
Usage: "disable gateway to server object checksums",
Value: false,
EnvVars: []string{"VGW_S3_DISABLE_CHECKSUM"},
Destination: &s3proxyDisableChecksum,
},
&cli.BoolFlag{
Name: "ssl-skip-verify",
Usage: "skip ssl cert verification for s3 service",
EnvVars: []string{"VGW_S3_SSL_SKIP_VERIFY"},
Value: false,
Destination: &s3proxySslSkipVerify,
},
&cli.BoolFlag{
Name: "use-path-style",
Usage: "use path style addressing for s3 proxy",
EnvVars: []string{"VGW_S3_USE_PATH_STYLE"},
Value: false,
Destination: &s3proxyUsePathStyle,
},
&cli.BoolFlag{
Name: "debug",
Usage: "output extra debug tracing",
Value: false,
EnvVars: []string{"VGW_S3_DEBUG"},
Destination: &s3proxyDebug,
},
},
}
}
func runS3(ctx *cli.Context) error {
be, err := s3proxy.New(ctx.Context, s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
s3proxyMetaBucket, s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyUsePathStyle, s3proxyDebug)
if err != nil {
return fmt.Errorf("init s3 backend: %w", err)
}
return runGateway(ctx.Context, be)
}

Some files were not shown because too many files have changed in this diff Show More