Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
17b42c369a |
@@ -1,46 +0,0 @@
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
cmd/versitygw/versitygw
|
||||
/versitygw
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
|
||||
# ignore IntelliJ directories
|
||||
.idea
|
||||
|
||||
# auto generated VERSION file
|
||||
VERSION
|
||||
|
||||
# build output
|
||||
/versitygw.spec
|
||||
/versitygw.spec.in
|
||||
*.tar
|
||||
*.tar.gz
|
||||
**/rand.data
|
||||
/profile.txt
|
||||
|
||||
dist/
|
||||
|
||||
# Release config files
|
||||
/.github
|
||||
|
||||
# Docker configuration files
|
||||
*Dockerfile
|
||||
/docker-compose.yml
|
||||
|
||||
# read files
|
||||
/LICENSE
|
||||
/NOTICE
|
||||
/CODE_OF_CONDUCT.md
|
||||
/README.md
|
||||
8
.env.dev
@@ -1,8 +0,0 @@
|
||||
POSIX_PORT=7071
|
||||
PROXY_PORT=7070
|
||||
ACCESS_KEY_ID=user
|
||||
SECRET_ACCESS_KEY=pass
|
||||
IAM_DIR=.
|
||||
SETUP_DIR=.
|
||||
AZ_ACCOUNT_NAME=devstoreaccount1
|
||||
AZ_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
|
||||
27
.github/ISSUE_TEMPLATE/bug_report.md
vendored
@@ -1,27 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: bug
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior.
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Server Version**
|
||||
output of
|
||||
```
|
||||
./versitygw -version
|
||||
uname -a
|
||||
```
|
||||
|
||||
**Additional context**
|
||||
Describe s3 client and version if applicable.
|
||||
14
.github/ISSUE_TEMPLATE/feature_request.md
vendored
@@ -1,14 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: enhancement
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
||||
14
.github/dependabot.yml
vendored
@@ -1,14 +0,0 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
groups:
|
||||
dev-dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
allow:
|
||||
# Allow both direct and indirect updates for all packages
|
||||
- dependency-type: "all"
|
||||
|
||||
49
.github/workflows/docker.yaml
vendored
@@ -1,49 +0,0 @@
|
||||
name: Publish Docker image
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
push_to_registries:
|
||||
name: Push Docker image to multiple registries
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
versity/versitygw
|
||||
ghcr.io/${{ github.repository }}
|
||||
|
||||
- name: Build and push Docker images
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
VERSION=${{ github.event.release.tag_name }}
|
||||
TIME=${{ github.event.release.published_at }}
|
||||
BUILD=${{ github.sha }}
|
||||
30
.github/workflows/functional.yml
vendored
@@ -1,30 +0,0 @@
|
||||
name: functional tests
|
||||
on: pull_request
|
||||
jobs:
|
||||
|
||||
build:
|
||||
name: RunTests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
id: go
|
||||
|
||||
- name: Get Dependencies
|
||||
run: |
|
||||
go get -v -t -d ./...
|
||||
|
||||
- name: Build and Run
|
||||
run: |
|
||||
make testbin
|
||||
./runtests.sh
|
||||
|
||||
- name: Coverage Report
|
||||
run: |
|
||||
go tool covdata percent -i=/tmp/covdata
|
||||
38
.github/workflows/go.yml
vendored
@@ -1,38 +0,0 @@
|
||||
name: general
|
||||
on: pull_request
|
||||
jobs:
|
||||
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
id: go
|
||||
|
||||
- name: Verify all files pass gofmt formatting
|
||||
run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then gofmt -s -d .; exit 1; fi
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
go get -v -t -d ./...
|
||||
|
||||
- name: Build
|
||||
run: make
|
||||
|
||||
- name: Test
|
||||
run: go test -coverprofile profile.txt -race -v -timeout 30s -tags=github ./...
|
||||
|
||||
- name: Install govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
shell: bash
|
||||
|
||||
- name: Run govulncheck
|
||||
run: govulncheck ./...
|
||||
shell: bash
|
||||
38
.github/workflows/goreleaser.yml
vendored
@@ -1,38 +0,0 @@
|
||||
name: goreleaser
|
||||
|
||||
on:
|
||||
push:
|
||||
# run only against tags
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
# packages: write
|
||||
# issues: write
|
||||
|
||||
jobs:
|
||||
goreleaser:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Fetch tags
|
||||
run: git fetch --force --tags
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: stable
|
||||
|
||||
- name: Run Releaser
|
||||
uses: goreleaser/goreleaser-action@v5
|
||||
with:
|
||||
distribution: goreleaser
|
||||
version: latest
|
||||
args: release --clean
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.TOKEN }}
|
||||
24
.github/workflows/static.yml
vendored
@@ -1,24 +0,0 @@
|
||||
name: staticcheck
|
||||
on: pull_request
|
||||
jobs:
|
||||
|
||||
build:
|
||||
name: Check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
id: go
|
||||
|
||||
- name: "staticcheck"
|
||||
uses: dominikh/staticcheck-action@v1
|
||||
with:
|
||||
version: "latest"
|
||||
73
.github/workflows/system.yml
vendored
@@ -1,73 +0,0 @@
|
||||
name: system tests
|
||||
on: pull_request
|
||||
jobs:
|
||||
build:
|
||||
name: RunTests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install ShellCheck
|
||||
run: sudo apt-get install shellcheck
|
||||
|
||||
- name: Run ShellCheck
|
||||
run: shellcheck -S warning ./tests/*.sh
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
id: go
|
||||
|
||||
- name: Get Dependencies
|
||||
run: |
|
||||
go get -v -t -d ./...
|
||||
|
||||
- name: Install BATS
|
||||
run: |
|
||||
git clone https://github.com/bats-core/bats-core.git
|
||||
cd bats-core && ./install.sh $HOME
|
||||
|
||||
- name: Install s3cmd
|
||||
run: |
|
||||
sudo apt-get install s3cmd
|
||||
|
||||
- name: Install mc
|
||||
run: |
|
||||
curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /usr/local/bin/mc
|
||||
chmod 755 /usr/local/bin/mc
|
||||
|
||||
- name: Build and run, posix backend
|
||||
run: |
|
||||
make testbin
|
||||
export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
|
||||
export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
|
||||
export AWS_REGION=us-east-1
|
||||
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
|
||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
|
||||
aws configure set aws_region $AWS_REGION --profile versity
|
||||
mkdir /tmp/gw
|
||||
export WORKSPACE=$GITHUB_WORKSPACE
|
||||
openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
|
||||
openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
|
||||
mkdir cover iam
|
||||
VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh
|
||||
|
||||
#- name: Build and run, s3 backend
|
||||
# run: |
|
||||
# make testbin
|
||||
# export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST
|
||||
# export AWS_SECRET_ACCESS_KEY=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
|
||||
# export AWS_REGION=us-east-1
|
||||
# aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity_s3
|
||||
# aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity_s3
|
||||
# aws configure set aws_region $AWS_REGION --profile versity_s3
|
||||
# export AWS_ACCESS_KEY_ID_TWO=ABCDEFGHIJKLMNOPQRST
|
||||
# export AWS_SECRET_ACCESS_KEY_TWO=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmn
|
||||
# export WORKSPACE=$GITHUB_WORKSPACE
|
||||
# VERSITYGW_TEST_ENV=./tests/.env.s3 GOCOVERDIR=/tmp/cover ./tests/run_all.sh
|
||||
|
||||
- name: Coverage report
|
||||
run: |
|
||||
go tool covdata percent -i=cover
|
||||
40
.gitignore
vendored
@@ -7,8 +7,6 @@
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
cmd/versitygw/versitygw
|
||||
/versitygw
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
@@ -21,41 +19,3 @@ cmd/versitygw/versitygw
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
|
||||
# ignore IntelliJ directories
|
||||
.idea
|
||||
|
||||
# ignore VS code directories
|
||||
.vscode
|
||||
|
||||
# auto generated VERSION file
|
||||
VERSION
|
||||
|
||||
# build output
|
||||
/versitygw.spec
|
||||
*.tar
|
||||
*.tar.gz
|
||||
**/rand.data
|
||||
/profile.txt
|
||||
|
||||
dist/
|
||||
|
||||
# secrets file for local github-actions testing
|
||||
tests/.secrets*
|
||||
|
||||
# IAM users files often created in testing
|
||||
users.json
|
||||
|
||||
# env files for testing
|
||||
**/.env*
|
||||
**/!.env.default
|
||||
|
||||
# s3cmd config files (testing)
|
||||
tests/s3cfg.local*
|
||||
tests/!s3cfg.local.default
|
||||
|
||||
# keys
|
||||
*.pem
|
||||
|
||||
# patches
|
||||
*.patch
|
||||
|
||||
126
.goreleaser.yaml
@@ -1,126 +0,0 @@
|
||||
before:
|
||||
hooks:
|
||||
- go mod tidy
|
||||
|
||||
builds:
|
||||
- goos:
|
||||
- linux
|
||||
- darwin
|
||||
- freebsd
|
||||
# windows is untested, we can start doing windows releases
|
||||
# if someone is interested in taking on testing
|
||||
# - windows
|
||||
env:
|
||||
# disable cgo to fix glibc issues: https://github.com/golang/go/issues/58550
|
||||
# once we need to enable this, we will need to do per distro releases
|
||||
- CGO_ENABLED=0
|
||||
main: ./cmd/versitygw
|
||||
binary: versitygw
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
ldflags:
|
||||
- -X=main.Build={{.Commit}} -X=main.BuildTime={{.Date}} -X=main.Version={{.Version}}
|
||||
|
||||
archives:
|
||||
- format: tar.gz
|
||||
# this name template makes the OS and Arch compatible with the results of uname.
|
||||
name_template: >-
|
||||
{{ .ProjectName }}_v{{ .Version }}_
|
||||
{{- title .Os }}_
|
||||
{{- if eq .Arch "amd64" }}x86_64
|
||||
{{- else if eq .Arch "386" }}i386
|
||||
{{- else }}{{ .Arch }}{{ end }}
|
||||
{{- if .Arm }}v{{ .Arm }}{{ end }}
|
||||
|
||||
# Set this to true if you want all files in the archive to be in a single directory.
|
||||
# If set to true and you extract the archive 'goreleaser_Linux_arm64.tar.gz',
|
||||
# you'll get a folder 'goreleaser_Linux_arm64'.
|
||||
# If set to false, all files are extracted separately.
|
||||
# You can also set it to a custom folder name (templating is supported).
|
||||
wrap_in_directory: true
|
||||
|
||||
# use zip for windows archives
|
||||
format_overrides:
|
||||
- goos: windows
|
||||
format: zip
|
||||
|
||||
# Additional files/globs you want to add to the archive.
|
||||
#
|
||||
# Default: [ 'LICENSE*', 'README*', 'CHANGELOG', 'license*', 'readme*', 'changelog']
|
||||
# Templates: allowed
|
||||
files:
|
||||
- README.md
|
||||
- LICENSE
|
||||
- NOTICE
|
||||
|
||||
checksum:
|
||||
name_template: 'checksums.txt'
|
||||
|
||||
snapshot:
|
||||
name_template: "{{ incpatch .Version }}-next"
|
||||
|
||||
changelog:
|
||||
sort: asc
|
||||
filters:
|
||||
exclude:
|
||||
- '^docs:'
|
||||
- '^test:'
|
||||
- '^Merge '
|
||||
|
||||
nfpms:
|
||||
- id: packages
|
||||
package_name: versitygw
|
||||
vendor: Versity Software
|
||||
homepage: https://github.com/versity/versitygw
|
||||
maintainer: Ben McClelland <ben.mcclelland@versity.com>
|
||||
|
||||
description: |-
|
||||
The Versity S3 Gateway.
|
||||
A high-performance tool facilitating translation between AWS S3 API
|
||||
requests and various backend storage systems, including POSIX file
|
||||
backend storage. Its stateless architecture enables deployment in
|
||||
clusters for increased throughput, distributing requests across gateways
|
||||
for optimal performance. With a focus on modularity, it supports future
|
||||
extensions for additional backend systems.
|
||||
|
||||
license: Apache 2.0
|
||||
|
||||
builds:
|
||||
- versitygw
|
||||
|
||||
formats:
|
||||
- deb
|
||||
- rpm
|
||||
|
||||
umask: 0o002
|
||||
bindir: /usr/bin
|
||||
epoch: "1"
|
||||
release: "1"
|
||||
|
||||
rpm:
|
||||
group: "System Environment/Daemons"
|
||||
# RPM specific scripts.
|
||||
scripts:
|
||||
# The pretrans script runs before all RPM package transactions / stages.
|
||||
#pretrans: ./extra/pretrans.sh
|
||||
# The posttrans script runs after all RPM package transactions / stages.
|
||||
posttrans: ./extra/posttrans.sh
|
||||
|
||||
contents:
|
||||
- src: extra/versitygw@.service
|
||||
dst: /lib/systemd/system/versitygw@.service
|
||||
|
||||
- src: extra/example.conf
|
||||
dst: /etc/versitygw.d/example.conf
|
||||
type: config
|
||||
|
||||
- dst: /etc/versitygw.d
|
||||
type: dir
|
||||
file_info:
|
||||
mode: 0700
|
||||
|
||||
|
||||
|
||||
# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
|
||||
# vim: set ts=2 sw=2 tw=0 fo=cnqoj
|
||||
@@ -1,128 +0,0 @@
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
We as members, contributors, and leaders pledge to make participation in our
|
||||
community a harassment-free experience for everyone, regardless of age, body
|
||||
size, visible or invisible disability, ethnicity, sex characteristics, gender
|
||||
identity and expression, level of experience, education, socio-economic status,
|
||||
nationality, personal appearance, race, religion, or sexual identity
|
||||
and orientation.
|
||||
|
||||
We pledge to act and interact in ways that contribute to an open, welcoming,
|
||||
diverse, inclusive, and healthy community.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to a positive environment for our
|
||||
community include:
|
||||
|
||||
* Demonstrating empathy and kindness toward other people
|
||||
* Being respectful of differing opinions, viewpoints, and experiences
|
||||
* Giving and gracefully accepting constructive feedback
|
||||
* Accepting responsibility and apologizing to those affected by our mistakes,
|
||||
and learning from the experience
|
||||
* Focusing on what is best not just for us as individuals, but for the
|
||||
overall community
|
||||
|
||||
Examples of unacceptable behavior include:
|
||||
|
||||
* The use of sexualized language or imagery, and sexual attention or
|
||||
advances of any kind
|
||||
* Trolling, insulting or derogatory comments, and personal or political attacks
|
||||
* Public or private harassment
|
||||
* Publishing others' private information, such as a physical or email
|
||||
address, without their explicit permission
|
||||
* Other conduct which could reasonably be considered inappropriate in a
|
||||
professional setting
|
||||
|
||||
## Enforcement Responsibilities
|
||||
|
||||
Community leaders are responsible for clarifying and enforcing our standards of
|
||||
acceptable behavior and will take appropriate and fair corrective action in
|
||||
response to any behavior that they deem inappropriate, threatening, offensive,
|
||||
or harmful.
|
||||
|
||||
Community leaders have the right and responsibility to remove, edit, or reject
|
||||
comments, commits, code, wiki edits, issues, and other contributions that are
|
||||
not aligned to this Code of Conduct, and will communicate reasons for moderation
|
||||
decisions when appropriate.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies within all community spaces, and also applies when
|
||||
an individual is officially representing the community in public spaces.
|
||||
Examples of representing our community include using an official e-mail address,
|
||||
posting via an official social media account, or acting as an appointed
|
||||
representative at an online or offline event.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported to the community leaders responsible for enforcement at
|
||||
versitygw@versity.com.
|
||||
All complaints will be reviewed and investigated promptly and fairly.
|
||||
|
||||
All community leaders are obligated to respect the privacy and security of the
|
||||
reporter of any incident.
|
||||
|
||||
## Enforcement Guidelines
|
||||
|
||||
Community leaders will follow these Community Impact Guidelines in determining
|
||||
the consequences for any action they deem in violation of this Code of Conduct:
|
||||
|
||||
### 1. Correction
|
||||
|
||||
**Community Impact**: Use of inappropriate language or other behavior deemed
|
||||
unprofessional or unwelcome in the community.
|
||||
|
||||
**Consequence**: A private, written warning from community leaders, providing
|
||||
clarity around the nature of the violation and an explanation of why the
|
||||
behavior was inappropriate. A public apology may be requested.
|
||||
|
||||
### 2. Warning
|
||||
|
||||
**Community Impact**: A violation through a single incident or series
|
||||
of actions.
|
||||
|
||||
**Consequence**: A warning with consequences for continued behavior. No
|
||||
interaction with the people involved, including unsolicited interaction with
|
||||
those enforcing the Code of Conduct, for a specified period of time. This
|
||||
includes avoiding interactions in community spaces as well as external channels
|
||||
like social media. Violating these terms may lead to a temporary or
|
||||
permanent ban.
|
||||
|
||||
### 3. Temporary Ban
|
||||
|
||||
**Community Impact**: A serious violation of community standards, including
|
||||
sustained inappropriate behavior.
|
||||
|
||||
**Consequence**: A temporary ban from any sort of interaction or public
|
||||
communication with the community for a specified period of time. No public or
|
||||
private interaction with the people involved, including unsolicited interaction
|
||||
with those enforcing the Code of Conduct, is allowed during this period.
|
||||
Violating these terms may lead to a permanent ban.
|
||||
|
||||
### 4. Permanent Ban
|
||||
|
||||
**Community Impact**: Demonstrating a pattern of violation of community
|
||||
standards, including sustained inappropriate behavior, harassment of an
|
||||
individual, or aggression toward or disparagement of classes of individuals.
|
||||
|
||||
**Consequence**: A permanent ban from any sort of public interaction within
|
||||
the community.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
|
||||
version 2.0, available at
|
||||
https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
|
||||
|
||||
Community Impact Guidelines were inspired by [Mozilla's code of conduct
|
||||
enforcement ladder](https://github.com/mozilla/diversity).
|
||||
|
||||
[homepage]: https://www.contributor-covenant.org
|
||||
|
||||
For answers to common questions about this code of conduct, see the FAQ at
|
||||
https://www.contributor-covenant.org/faq. Translations are available at
|
||||
https://www.contributor-covenant.org/translations.
|
||||
35
Dockerfile
@@ -1,35 +0,0 @@
|
||||
FROM golang:latest
|
||||
|
||||
# Set build arguments with default values
|
||||
ARG VERSION="none"
|
||||
ARG BUILD="none"
|
||||
ARG TIME="none"
|
||||
|
||||
# Set environment variables
|
||||
ENV VERSION=${VERSION}
|
||||
ENV BUILD=${BUILD}
|
||||
ENV TIME=${TIME}
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY ./ ./
|
||||
|
||||
WORKDIR /app/cmd/versitygw
|
||||
ENV CGO_ENABLED=0
|
||||
RUN go build -ldflags "-X=main.Build=${BUILD} -X=main.BuildTime=${TIME} -X=main.Version=${VERSION}" -o versitygw
|
||||
|
||||
FROM alpine:latest
|
||||
|
||||
# These arguments can be overriden when building the image
|
||||
ARG IAM_DIR=/tmp/vgw
|
||||
ARG SETUP_DIR=/tmp/vgw
|
||||
|
||||
RUN mkdir -p $IAM_DIR
|
||||
RUN mkdir -p $SETUP_DIR
|
||||
|
||||
COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
|
||||
|
||||
ENTRYPOINT [ "/app/versitygw" ]
|
||||
@@ -1,18 +0,0 @@
|
||||
FROM golang:latest
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY ./ ./
|
||||
COPY ./tests/certs/* /etc/pki/tls/certs/
|
||||
|
||||
ARG IAM_DIR=/tmp/vgw
|
||||
ARG SETUP_DIR=/tmp/vgw
|
||||
|
||||
RUN mkdir -p $IAM_DIR
|
||||
RUN mkdir -p $SETUP_DIR
|
||||
|
||||
RUN go get github.com/githubnemo/CompileDaemon
|
||||
RUN go install github.com/githubnemo/CompileDaemon
|
||||
@@ -1,81 +0,0 @@
|
||||
FROM --platform=linux/arm64 ubuntu:latest
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
ARG SECRETS_FILE=tests/.secrets
|
||||
ARG CONFIG_FILE=tests/.env.docker
|
||||
|
||||
ENV TZ=Etc/UTC
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
git \
|
||||
make \
|
||||
wget \
|
||||
curl \
|
||||
unzip \
|
||||
tzdata \
|
||||
s3cmd \
|
||||
jq \
|
||||
bc \
|
||||
ca-certificates && \
|
||||
update-ca-certificates && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /tmp
|
||||
|
||||
# Install AWS cli
|
||||
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
|
||||
|
||||
# Install mc
|
||||
RUN curl https://dl.min.io/client/mc/release/linux-arm64/mc \
|
||||
--create-dirs \
|
||||
-o /usr/local/minio-binaries/mc && \
|
||||
chmod -R 755 /usr/local/minio-binaries
|
||||
ENV PATH="/usr/local/minio-binaries":${PATH}
|
||||
|
||||
# Download Go 1.21 (adjust the version and platform as needed)
|
||||
RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
|
||||
|
||||
# Extract the downloaded archive
|
||||
RUN tar -xvf go1.21.7.linux-arm64.tar.gz -C /usr/local
|
||||
|
||||
# Set Go environment variables
|
||||
ENV PATH="/usr/local/go/bin:${PATH}"
|
||||
ENV GOPATH="/go"
|
||||
ENV GOBIN="$GOPATH/bin"
|
||||
|
||||
# Make the directory for Go packages
|
||||
RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
|
||||
|
||||
# Create tester user
|
||||
RUN groupadd -r tester && useradd -r -g tester tester
|
||||
RUN mkdir /home/tester && chown tester:tester /home/tester
|
||||
ENV HOME=/home/tester
|
||||
|
||||
# install bats
|
||||
RUN git clone https://github.com/bats-core/bats-core.git && \
|
||||
cd bats-core && \
|
||||
./install.sh /home/tester
|
||||
|
||||
USER tester
|
||||
COPY --chown=tester:tester . /home/tester
|
||||
|
||||
WORKDIR /home/tester
|
||||
RUN make
|
||||
|
||||
RUN . $SECRETS_FILE && \
|
||||
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE && \
|
||||
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE && \
|
||||
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE && \
|
||||
aws configure set aws_region $AWS_REGION --profile $AWS_PROFILE
|
||||
|
||||
RUN mkdir /tmp/gw
|
||||
|
||||
RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen_bits:2048 && \
|
||||
openssl req -new -x509 -key versitygw-docker.pem -out cert-docker.pem -days 365 \
|
||||
-subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
|
||||
|
||||
ENV WORKSPACE=.
|
||||
ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
|
||||
|
||||
CMD ["tests/run_all.sh"]
|
||||
89
Makefile
@@ -1,89 +0,0 @@
|
||||
# Copyright 2023 Versity Software
|
||||
# This file is licensed under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Go parameters
|
||||
GOCMD=go
|
||||
GOBUILD=$(GOCMD) build
|
||||
GOCLEAN=$(GOCMD) clean
|
||||
GOTEST=$(GOCMD) test
|
||||
|
||||
BIN=versitygw
|
||||
|
||||
VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
|
||||
BUILD := $(shell git rev-parse --short HEAD || echo release-rpm)
|
||||
TIME := `date -u '+%Y-%m-%d_%I:%M:%S%p'`
|
||||
|
||||
LDFLAGS=-ldflags "-X=main.Build=$(BUILD) -X=main.BuildTime=$(TIME) -X=main.Version=$(VERSION)"
|
||||
|
||||
all: build
|
||||
|
||||
build: $(BIN)
|
||||
|
||||
.PHONY: $(BIN)
|
||||
$(BIN):
|
||||
$(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go
|
||||
|
||||
testbin:
|
||||
$(GOBUILD) $(LDFLAGS) -o $(BIN) -cover -race cmd/$(BIN)/*.go
|
||||
|
||||
.PHONY: test
|
||||
test:
|
||||
$(GOTEST) ./...
|
||||
|
||||
.PHONY: check
|
||||
check:
|
||||
# note this requires staticcheck be in your PATH:
|
||||
# export PATH=$PATH:~/go/bin
|
||||
# go install honnef.co/go/tools/cmd/staticcheck@latest
|
||||
staticcheck ./...
|
||||
golint ./...
|
||||
gofmt -s -l .
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(GOCLEAN)
|
||||
|
||||
.PHONY: cleanall
|
||||
cleanall: clean
|
||||
rm -f $(BIN)
|
||||
rm -f versitygw-*.tar
|
||||
rm -f versitygw-*.tar.gz
|
||||
|
||||
TARFILE = $(BIN)-$(VERSION).tar
|
||||
|
||||
dist:
|
||||
echo $(VERSION) >VERSION
|
||||
git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
|
||||
rm -f VERSION
|
||||
gzip -f $(TARFILE)
|
||||
|
||||
# Creates and runs S3 gateway instance in a docker container
|
||||
.PHONY: up-posix
|
||||
up-posix:
|
||||
docker compose --env-file .env.dev up posix
|
||||
|
||||
# Creates and runs S3 gateway proxy instance in a docker container
|
||||
.PHONY: up-proxy
|
||||
up-proxy:
|
||||
docker compose --env-file .env.dev up proxy
|
||||
|
||||
# Creates and runs S3 gateway to azurite instance in a docker container
|
||||
.PHONY: up-azurite
|
||||
up-azurite:
|
||||
docker compose --env-file .env.dev up azurite azuritegw
|
||||
|
||||
# Creates and runs both S3 gateway and proxy server instances in docker containers
|
||||
.PHONY: up-app
|
||||
up-app:
|
||||
docker compose --env-file .env.dev up
|
||||
88
README.md
@@ -1,88 +0,0 @@
|
||||
# The Versity S3 Gateway:<br/>A High-Performance S3 Translation Service
|
||||
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo-white.svg">
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo.svg">
|
||||
<a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
|
||||
</picture>
|
||||
|
||||
[](https://github.com/versity/versitygw/blob/main/LICENSE)
|
||||
|
||||
### Binary release builds
|
||||
Download [latest release](https://github.com/versity/versitygw/releases)
|
||||
| Linux/amd64 | Linux/arm64 | MacOS/amd64 | MacOS/arm64 | BSD/amd64 | BSD/arm64 |
|
||||
|:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
|
||||
| ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
|
||||
|
||||
### Use Cases
|
||||
* Turn your local filesystem into an S3 server with a single command!
|
||||
* Proxy S3 requests to S3 storage
|
||||
* Simple to deploy S3 server with a single command
|
||||
* Protocol compatibility in `posix` allows common access to files via posix or S3
|
||||
* Simplified interface for adding new storage system support
|
||||
|
||||
### News
|
||||
* New performance (scale up) analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
|
||||
* New performance (scale out) Part 2 analysis article [https://github.com/versity/versitygw/wiki/Performance-Part-2](https://github.com/versity/versitygw/wiki/Performance-Part-2)
|
||||
|
||||
### Mailing List
|
||||
Keep up to date with latest gateway announcements by signing up to the [versitygw mailing list](https://www.versity.com/products/versitygw#signup).
|
||||
|
||||
### Documentation
|
||||
See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
|
||||
|
||||
### Need help?
|
||||
Ask questions in the [community discussions](https://github.com/versity/versitygw/discussions).
|
||||
<br>
|
||||
Contact [Versity Sales](https://www.versity.com/contact/) to discuss enterprise support.
|
||||
|
||||
### Overview
|
||||
Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
|
||||
|
||||
The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
|
||||
|
||||
The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage and Versity’s open source ScoutFS filesystem.
|
||||
|
||||
The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateway’s stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance.
|
||||
|
||||
The S3 HTTP(S) server and routing is implemented using the [Fiber](https://gofiber.io) web framework. This framework is actively developed with a focus on performance. S3 API compatibility leverages the official [aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) whenever possible for maximum service compatibility with AWS S3.
|
||||
|
||||
## Getting Started
|
||||
See the [Quickstart](https://github.com/versity/versitygw/wiki/Quickstart) documentation.
|
||||
|
||||
### Run the gateway with posix backend:
|
||||
|
||||
```
|
||||
mkdir /tmp/vgw
|
||||
ROOT_ACCESS_KEY="testuser" ROOT_SECRET_KEY="secret" ./versitygw --port :10000 posix /tmp/vgw
|
||||
```
|
||||
This will enable an S3 server on the current host listening on port 10000 and hosting the directory `/tmp/vgw`.
|
||||
|
||||
To get the usage output, run the following:
|
||||
|
||||
```
|
||||
./versitygw --help
|
||||
```
|
||||
|
||||
The command format is
|
||||
|
||||
```
|
||||
versitygw [global options] command [command options] [arguments...]
|
||||
```
|
||||
The [global options](https://github.com/versity/versitygw/wiki/Global-Options) are specified before the backend type and the backend options are specified after.
|
||||
|
||||
***
|
||||
|
||||
#### Versity gives you clarity and control over your archival storage, so you can allocate more resources to your core mission.
|
||||
|
||||
### Contact
|
||||

|
||||
info@versity.com <br />
|
||||
+1 844 726 8826
|
||||
|
||||
### @versitysoftware
|
||||
[](https://www.linkedin.com/company/versity/)
|
||||
[](https://twitter.com/VersitySoftware)
|
||||
[](https://www.facebook.com/versitysoftware)
|
||||
[](https://www.instagram.com/versitysoftware/)
|
||||
|
||||
BIN
assets/facebook.jpg
Normal file
|
After Width: | Height: | Size: 470 B |
BIN
assets/instagram.jpg
Normal file
|
After Width: | Height: | Size: 461 B |
BIN
assets/linkedin.jpg
Normal file
|
After Width: | Height: | Size: 391 B |
13
assets/logo-white.svg
Normal file
@@ -0,0 +1,13 @@
|
||||
<svg width="209" height="73" viewBox="0 0 209 73" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M47.427 30.0292C42.767 30.9134 40.4568 30.3528 40.2354 29.406C40.1625 29.0919 40.2887 28.6812 40.6122 28.1848C42.2518 25.6629 47.9677 22.1097 54.5119 19.5417C57.6668 18.3011 60.7686 17.3735 63.4818 16.8593C68.1376 15.976 70.4445 16.5363 70.6646 17.4827C70.7388 17.7976 70.6123 18.2078 70.2897 18.7048C68.6525 21.2235 62.9369 24.7783 56.3908 27.3497C53.2389 28.5879 50.1396 29.5145 47.4284 30.0287L47.427 30.0292Z" fill="white"/>
|
||||
<path d="M48.4866 39.9975C42.4383 40.4446 38.4817 39.0018 38.7133 37.4258C38.763 37.0777 39.0232 36.6989 39.4845 36.301C41.0086 34.9865 44.833 33.4548 49.9427 32.9982C50.1102 32.9836 50.2764 32.9697 50.4411 32.9574C56.4783 32.5107 60.4292 33.9536 60.1996 35.5285C60.1487 35.8758 59.8879 36.2549 59.4247 36.6544C57.8987 37.9699 54.0743 39.5016 48.9726 39.9577C48.8092 39.9718 48.648 39.9852 48.4871 39.997L48.4866 39.9975Z" fill="white"/>
|
||||
<path d="M44.5318 47.6969C44.5093 44.8455 42.0643 42.5515 39.0706 42.5727C37.1309 42.5866 35.4376 43.57 34.4912 45.0354L34.4904 45.0349L34.4755 45.0595C34.4417 45.1127 34.4089 45.1664 34.3773 45.2211L29.6958 52.93L10.1292 20.2803C8.6447 17.8035 5.33413 16.9425 2.73347 18.3555C0.192505 19.737 -0.72811 22.7797 0.616915 25.2285L0.615815 25.2295L0.643322 25.2748C0.666427 25.3162 0.688702 25.3579 0.712632 25.3987C0.736837 25.4396 0.763518 25.4786 0.788548 25.5179L28.8811 72.4019L28.8822 72.3956C29.031 72.6689 29.3281 72.8577 29.673 72.8577C29.9979 72.8577 30.2801 72.6909 30.4363 72.4433L30.4374 72.4467L43.8764 50.2044L43.8731 50.2028C44.2991 49.4586 44.5387 48.6041 44.5318 47.6969Z" fill="white"/>
|
||||
<path d="M69.6803 0.677278C68.2968 -1.32435 60.4824 1.26118 52.2266 6.45006C43.9703 11.641 38.3998 17.4709 39.7834 19.472C41.1669 21.4739 48.9818 18.8889 57.2376 13.6979C65.4929 8.50826 71.0644 2.67916 69.6803 0.677278Z" fill="white"/>
|
||||
<path d="M108.898 20.9902L101.884 39.4324H101.798L94.2404 20.9902H87.6453L99.6694 51.7645H103.261L115.498 20.9902H108.898Z" fill="white"/>
|
||||
<path d="M118.544 38.6442C118.626 38.1404 118.765 37.6502 118.962 37.1708C119.155 36.6966 119.415 36.2812 119.733 35.921C120.056 35.5652 120.444 35.2711 120.903 35.0454C121.363 34.8191 121.884 34.7061 122.468 34.7061C123.024 34.7061 123.523 34.8191 123.97 35.0454C124.415 35.2711 124.807 35.5704 125.141 35.9406C125.473 36.3138 125.741 36.7284 125.932 37.1912C126.127 37.6563 126.269 38.1404 126.35 38.6442H118.544ZM129.504 33.9109C128.711 32.956 127.701 32.2004 126.479 31.6458C125.252 31.0894 123.803 30.8105 122.135 30.8105C120.464 30.8105 119.009 31.0807 117.772 31.6236C116.534 32.1708 115.504 32.9168 114.681 33.8708C113.86 34.8261 113.242 35.9458 112.823 37.233C112.405 38.5185 112.197 39.9027 112.197 41.3869C112.197 42.8712 112.433 44.238 112.907 45.4838C113.38 46.7301 114.055 47.8124 114.933 48.726C115.807 49.6369 116.88 50.3485 118.147 50.8501C119.415 51.3556 120.838 51.6087 122.427 51.6087C124.6 51.6087 126.509 51.0785 128.166 50.0223C129.822 48.9665 131.014 47.4522 131.737 45.4838L126.35 44.6495C125.96 45.4181 125.454 46.0528 124.827 46.5552C124.2 47.0607 123.399 47.3139 122.427 47.3139C121.676 47.3139 121.036 47.1525 120.503 46.8397C119.975 46.5226 119.559 46.1185 119.256 45.6122C118.948 45.1158 118.722 44.5568 118.587 43.94C118.445 43.3197 118.376 42.7098 118.376 42.1055H131.863V41.4282C131.863 39.9705 131.668 38.5907 131.277 37.2935C130.887 35.9915 130.298 34.8648 129.504 33.9109Z" fill="white"/>
|
||||
<path d="M145.732 30.8105C144.645 30.8105 143.674 31.0424 142.81 31.5039C141.947 31.9694 141.223 32.6245 140.638 33.4759H140.557V31.4487H134.71V50.9689H140.557V42.1821C140.557 41.4413 140.597 40.684 140.68 39.9127C140.765 39.1471 140.959 38.4511 141.265 37.8294C141.572 37.2039 142.023 36.7028 142.621 36.3178C143.222 35.9345 144.021 35.7392 145.025 35.7392C145.915 35.7392 146.721 35.9806 147.448 36.4565L148.155 31.2864C147.765 31.1546 147.367 31.0424 146.966 30.9489C146.562 30.8562 146.152 30.8105 145.732 30.8105Z" fill="white"/>
|
||||
<path d="M162.394 40.9323C161.99 40.4929 161.537 40.1267 161.033 39.8387C160.538 39.5464 160.021 39.2923 159.492 39.0792C158.962 38.8669 158.447 38.679 157.948 38.505C157.444 38.3323 156.993 38.1483 156.591 37.9503C156.187 37.7498 155.868 37.5314 155.631 37.2935C155.395 37.0529 155.273 36.7497 155.273 36.3773C155.273 35.9001 155.477 35.5025 155.883 35.1837C156.283 34.8661 156.722 34.7056 157.195 34.7056C157.779 34.7056 158.339 34.8396 158.865 35.1063C159.396 35.3707 159.868 35.6883 160.286 36.0598L162.789 32.8403C161.955 32.1482 160.919 31.6401 159.681 31.3069C158.44 30.9754 157.28 30.8105 156.193 30.8105C155.219 30.8105 154.336 30.9706 153.539 31.2864C152.751 31.6062 152.067 32.0425 151.496 32.6002C150.925 33.1574 150.479 33.8269 150.16 34.6086C149.839 35.3929 149.68 36.2312 149.68 37.1321C149.68 37.9273 149.799 38.5985 150.034 39.1405C150.272 39.6869 150.577 40.1549 150.952 40.5525C151.327 40.9519 151.765 41.2816 152.268 41.5483C152.772 41.8141 153.269 42.0511 153.772 42.263C154.273 42.4761 154.775 42.6636 155.273 42.8381C155.776 43.0112 156.216 43.2044 156.591 43.4167C156.965 43.6281 157.271 43.8738 157.509 44.1522C157.743 44.432 157.861 44.7821 157.861 45.2032C157.861 45.8396 157.634 46.3547 157.175 46.7371C156.716 47.1194 156.167 47.3139 155.525 47.3139C154.661 47.3139 153.861 47.0868 153.124 46.6348C152.39 46.1863 151.739 45.6552 151.181 45.0479L148.511 48.3444C149.399 49.3802 150.479 50.1841 151.747 50.7531C153.011 51.3234 154.342 51.6087 155.735 51.6087C156.819 51.6087 157.841 51.4508 158.805 51.1293C159.76 50.81 160.605 50.3555 161.329 49.7591C162.051 49.1635 162.63 48.4388 163.06 47.591C163.494 46.7414 163.709 45.7735 163.709 44.6886C163.709 43.8408 163.59 43.1117 163.352 42.4992C163.118 41.8915 162.797 41.369 162.394 40.9323Z" fill="white"/>
|
||||
<path d="M169.093 22.4395C168.2 22.4395 167.45 22.7401 166.841 23.3317C166.233 23.9237 165.926 24.6332 165.926 25.458C165.926 26.3045 166.233 27.0171 166.841 27.5952C167.45 28.1777 168.2 28.4674 169.093 28.4674C169.98 28.4674 170.731 28.1777 171.339 27.5952C171.951 27.0171 172.257 26.3045 172.257 25.458C172.257 24.6332 171.951 23.9237 171.339 23.3317C170.731 22.7401 169.98 22.4395 169.093 22.4395Z" fill="white"/>
|
||||
<path d="M172.014 31.4492H166.167V50.9694H172.014V31.4492Z" fill="white"/>
|
||||
<path d="M202.482 31.4492L197.425 41.9607L192.336 31.4492H185.418H184.587H184.212V22.9521H178.364V31.4492H174.405V36.139H178.364V50.9693H184.212V36.139H188.688L194.422 47.9882L189.519 57.6672H195.949L209 31.4492H202.482Z" fill="white"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 6.4 KiB |
13
assets/logo.svg
Normal file
@@ -0,0 +1,13 @@
|
||||
<svg width="150" height="53" viewBox="0 0 150 53" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M33.903 21.4665C30.5718 22.0985 28.9204 21.6977 28.7621 21.021C28.71 20.7965 28.8003 20.5028 29.0315 20.148C30.2036 18.3452 34.2896 15.8052 38.9676 13.9695C41.2229 13.0827 43.4402 12.4196 45.3797 12.052C48.7079 11.4206 50.357 11.8211 50.5143 12.4977C50.5673 12.7228 50.4769 13.016 50.2463 13.3713C49.076 15.1718 44.9902 17.7129 40.3107 19.551C38.0576 20.4362 35.8421 21.0985 33.904 21.4661L33.903 21.4665Z" fill="#191B2A"/>
|
||||
<path d="M34.6603 28.592C30.3367 28.9116 27.5083 27.8802 27.6739 26.7536C27.7095 26.5048 27.8955 26.234 28.2252 25.9495C29.3147 25.0099 32.0485 23.915 35.7012 23.5886C35.8209 23.5781 35.9397 23.5682 36.0575 23.5594C40.3731 23.2401 43.1974 24.2715 43.0332 25.3973C42.9969 25.6456 42.8105 25.9166 42.4794 26.2021C41.3885 27.1426 38.6547 28.2375 35.0077 28.5635C34.8909 28.5736 34.7757 28.5832 34.6607 28.5916L34.6603 28.592Z" fill="#191B2A"/>
|
||||
<path d="M31.8333 34.096C31.8172 32.0577 30.0694 30.4178 27.9294 30.433C26.5428 30.4429 25.3324 31.1459 24.6558 32.1934L24.6552 32.193L24.6446 32.2106C24.6204 32.2487 24.597 32.287 24.5744 32.3262L21.2279 37.8368L7.24078 14.4974C6.17961 12.7269 3.81307 12.1114 1.95401 13.1214C0.137611 14.109 -0.520485 16.2841 0.440998 18.0346L0.440212 18.0353L0.459875 18.0677C0.476391 18.0973 0.492314 18.1271 0.509421 18.1563C0.526723 18.1855 0.545796 18.2134 0.563688 18.2415L20.6455 51.7562L20.6463 51.7517C20.7527 51.947 20.965 52.082 21.2116 52.082C21.4438 52.082 21.6455 51.9628 21.7572 51.7858L21.758 51.7882L31.3648 35.8884L31.3624 35.8873C31.667 35.3553 31.8382 34.7444 31.8333 34.096Z" fill="#191B2A"/>
|
||||
<path d="M49.8105 0.484148C48.8215 -0.9467 43.2354 0.90155 37.3338 4.61078C31.4318 8.32151 27.4498 12.489 28.4388 13.9194C29.4278 15.3505 35.0143 13.5026 40.9159 9.79187C46.8171 6.08208 50.7999 1.91518 49.8105 0.484148Z" fill="#191B2A"/>
|
||||
<path d="M77.8452 15.0049L72.8313 28.1882H72.7699L67.3671 15.0049H62.6526L71.248 37.0037H73.8157L82.5626 15.0049H77.8452Z" fill="#191B2A"/>
|
||||
<path d="M84.7403 27.625C84.7988 27.2649 84.898 26.9144 85.0391 26.5717C85.1775 26.2328 85.363 25.9358 85.5906 25.6783C85.8211 25.424 86.0986 25.2138 86.4268 25.0524C86.7556 24.8907 87.1281 24.8098 87.5454 24.8098C87.9428 24.8098 88.2996 24.8907 88.619 25.0524C88.9373 25.2138 89.2178 25.4277 89.4565 25.6923C89.6938 25.9591 89.8848 26.2555 90.0216 26.5863C90.1614 26.9188 90.2623 27.2649 90.3207 27.625H84.7403ZM92.575 24.2414C92.0085 23.5588 91.2866 23.0187 90.4128 22.6222C89.5358 22.2245 88.5001 22.0251 87.3074 22.0251C86.113 22.0251 85.073 22.2183 84.1885 22.6063C83.3034 22.9975 82.5674 23.5308 81.9787 24.2128C81.3923 24.8956 80.9502 25.6961 80.6504 26.6162C80.352 27.5351 80.2031 28.5246 80.2031 29.5856C80.2031 30.6466 80.3723 31.6237 80.7108 32.5143C81.0488 33.4052 81.5314 34.1788 82.1593 34.8319C82.7839 35.483 83.5512 35.9918 84.4566 36.3503C85.363 36.7116 86.3801 36.8926 87.516 36.8926C89.0695 36.8926 90.434 36.5135 91.6183 35.7585C92.8022 35.0038 93.6544 33.9214 94.1716 32.5143L90.3207 31.9178C90.0419 32.4673 89.6798 32.921 89.2318 33.2801C88.7838 33.6415 88.2111 33.8225 87.516 33.8225C86.9795 33.8225 86.5218 33.7071 86.1411 33.4835C85.7636 33.2568 85.4658 32.9679 85.249 32.606C85.029 32.2512 84.8677 31.8516 84.7713 31.4106C84.6695 30.9672 84.6202 30.5312 84.6202 30.0993H94.2611V29.6151C94.2611 28.5731 94.1217 27.5867 93.8425 26.6594C93.5636 25.7287 93.1428 24.9233 92.575 24.2414Z" fill="#191B2A"/>
|
||||
<path d="M104.175 22.0251C103.398 22.0251 102.704 22.1909 102.087 22.5208C101.47 22.8536 100.952 23.3219 100.534 23.9304H100.476V22.4813H96.2964V36.4352H100.476V30.154C100.476 29.6245 100.505 29.0831 100.564 28.5318C100.625 27.9845 100.763 27.4869 100.982 27.0425C101.202 26.5954 101.524 26.2372 101.952 25.962C102.381 25.688 102.952 25.5484 103.67 25.5484C104.306 25.5484 104.883 25.721 105.402 26.0611L105.908 22.3653C105.629 22.2711 105.345 22.1909 105.058 22.124C104.769 22.0578 104.476 22.0251 104.175 22.0251Z" fill="#191B2A"/>
|
||||
<path d="M116.086 29.2606C115.797 28.9465 115.474 28.6847 115.114 28.4789C114.759 28.2699 114.39 28.0883 114.012 27.9359C113.633 27.7842 113.265 27.6498 112.908 27.5255C112.548 27.402 112.225 27.2705 111.938 27.129C111.649 26.9856 111.421 26.8295 111.252 26.6594C111.083 26.4875 110.996 26.2707 110.996 26.0045C110.996 25.6634 111.142 25.3792 111.432 25.1513C111.718 24.9242 112.031 24.8095 112.37 24.8095C112.787 24.8095 113.187 24.9053 113.563 25.0959C113.944 25.285 114.28 25.512 114.579 25.7775L116.369 23.4761C115.772 22.9813 115.032 22.6182 114.147 22.38C113.26 22.143 112.43 22.0251 111.653 22.0251C110.958 22.0251 110.326 22.1396 109.756 22.3653C109.193 22.5939 108.704 22.9058 108.296 23.3044C107.887 23.7028 107.569 24.1814 107.341 24.7402C107.111 25.3008 106.998 25.9001 106.998 26.5441C106.998 27.1125 107.083 27.5923 107.251 27.9798C107.421 28.3703 107.639 28.7049 107.907 28.9891C108.175 29.2746 108.489 29.5103 108.848 29.7009C109.208 29.8909 109.563 30.0604 109.923 30.2119C110.281 30.3642 110.64 30.4983 110.996 30.623C111.355 30.7467 111.67 30.8848 111.938 31.0365C112.206 31.1877 112.424 31.3634 112.594 31.5624C112.762 31.7623 112.846 32.0127 112.846 32.3137C112.846 32.7686 112.684 33.1368 112.355 33.4101C112.027 33.6835 111.635 33.8225 111.176 33.8225C110.559 33.8225 109.987 33.6601 109.46 33.337C108.935 33.0164 108.47 32.6368 108.071 32.2027L106.162 34.5591C106.797 35.2995 107.569 35.8742 108.475 36.2809C109.379 36.6886 110.331 36.8926 111.326 36.8926C112.101 36.8926 112.832 36.7797 113.521 36.5499C114.204 36.3217 114.807 35.9967 115.325 35.5704C115.841 35.1447 116.255 34.6266 116.562 34.0205C116.872 33.4132 117.026 32.7213 117.026 31.9458C117.026 31.3397 116.941 30.8186 116.771 30.3807C116.604 29.9463 116.375 29.5728 116.086 29.2606Z" fill="#191B2A"/>
|
||||
<path d="M120.875 16.041C120.237 16.041 119.701 16.2559 119.265 16.6788C118.83 17.102 118.611 17.6092 118.611 18.1988C118.611 18.8039 118.83 19.3133 119.265 19.7266C119.701 20.1429 120.237 20.35 120.875 20.35C121.509 20.35 122.046 20.1429 122.481 19.7266C122.918 19.3133 123.137 18.8039 123.137 18.1988C123.137 17.6092 122.918 17.102 122.481 16.6788C122.046 16.2559 121.509 16.041 120.875 16.041Z" fill="#191B2A"/>
|
||||
<path d="M122.963 22.4814H118.784V36.4353H122.963V22.4814Z" fill="#191B2A"/>
|
||||
<path d="M144.743 22.4813L141.128 29.9954L137.49 22.4813H132.545H131.951H131.683V16.4072H127.503V22.4813H124.673V25.8338H127.503V36.4351H131.683V25.8338H134.882L138.982 34.3041L135.477 41.223H140.073L149.402 22.4813H144.743Z" fill="#191B2A"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 6.4 KiB |
BIN
assets/scoutam_gateway.png
Normal file
|
After Width: | Height: | Size: 66 KiB |
BIN
assets/twitter.jpg
Normal file
|
After Width: | Height: | Size: 448 B |
BIN
assets/workflow.png
Normal file
|
After Width: | Height: | Size: 69 KiB |
358
auth/acl.go
@@ -1,358 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
type ACL struct {
|
||||
ACL types.BucketCannedACL
|
||||
Owner string
|
||||
Grantees []Grantee
|
||||
}
|
||||
|
||||
type Grantee struct {
|
||||
Permission types.Permission
|
||||
Access string
|
||||
}
|
||||
|
||||
type GetBucketAclOutput struct {
|
||||
Owner *types.Owner
|
||||
AccessControlList AccessControlList
|
||||
}
|
||||
|
||||
type AccessControlList struct {
|
||||
Grants []types.Grant `xml:"Grant"`
|
||||
}
|
||||
type AccessControlPolicy struct {
|
||||
AccessControlList AccessControlList `xml:"AccessControlList"`
|
||||
Owner types.Owner
|
||||
}
|
||||
|
||||
func ParseACL(data []byte) (ACL, error) {
|
||||
if len(data) == 0 {
|
||||
return ACL{}, nil
|
||||
}
|
||||
|
||||
var acl ACL
|
||||
if err := json.Unmarshal(data, &acl); err != nil {
|
||||
return acl, fmt.Errorf("parse acl: %w", err)
|
||||
}
|
||||
return acl, nil
|
||||
}
|
||||
|
||||
func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
|
||||
var acl ACL
|
||||
if err := json.Unmarshal(data, &acl); err != nil {
|
||||
return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
|
||||
}
|
||||
|
||||
grants := []types.Grant{}
|
||||
|
||||
for _, elem := range acl.Grantees {
|
||||
acs := elem.Access
|
||||
grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
|
||||
}
|
||||
|
||||
return GetBucketAclOutput{
|
||||
Owner: &types.Owner{
|
||||
ID: &acl.Owner,
|
||||
},
|
||||
AccessControlList: AccessControlList{
|
||||
Grants: grants,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
|
||||
if input == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
if acl.Owner != *input.AccessControlPolicy.Owner.ID {
|
||||
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
// if the ACL is specified, set the ACL, else replace the grantees
|
||||
if input.ACL != "" {
|
||||
acl.ACL = input.ACL
|
||||
acl.Grantees = []Grantee{}
|
||||
} else {
|
||||
grantees := []Grantee{}
|
||||
accs := []string{}
|
||||
|
||||
if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
|
||||
fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{}
|
||||
|
||||
if input.GrantFullControl != nil && *input.GrantFullControl != "" {
|
||||
fullControlList = splitUnique(*input.GrantFullControl, ",")
|
||||
for _, str := range fullControlList {
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
|
||||
}
|
||||
}
|
||||
if input.GrantRead != nil && *input.GrantRead != "" {
|
||||
readList = splitUnique(*input.GrantRead, ",")
|
||||
for _, str := range readList {
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
|
||||
}
|
||||
}
|
||||
if input.GrantReadACP != nil && *input.GrantReadACP != "" {
|
||||
readACPList = splitUnique(*input.GrantReadACP, ",")
|
||||
for _, str := range readACPList {
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
|
||||
}
|
||||
}
|
||||
if input.GrantWrite != nil && *input.GrantWrite != "" {
|
||||
writeList = splitUnique(*input.GrantWrite, ",")
|
||||
for _, str := range writeList {
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
|
||||
}
|
||||
}
|
||||
if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
|
||||
writeACPList = splitUnique(*input.GrantWriteACP, ",")
|
||||
for _, str := range writeACPList {
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
|
||||
}
|
||||
}
|
||||
|
||||
accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
|
||||
} else {
|
||||
cache := make(map[string]bool)
|
||||
for _, grt := range input.AccessControlPolicy.Grants {
|
||||
if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
|
||||
if _, ok := cache[*grt.Grantee.ID]; !ok {
|
||||
cache[*grt.Grantee.ID] = true
|
||||
accs = append(accs, *grt.Grantee.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if the specified accounts exist
|
||||
accList, err := CheckIfAccountsExist(accs, iam)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(accList) > 0 {
|
||||
return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
|
||||
}
|
||||
|
||||
acl.Grantees = grantees
|
||||
acl.ACL = ""
|
||||
}
|
||||
|
||||
result, err := json.Marshal(acl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
|
||||
result := []string{}
|
||||
|
||||
for _, acc := range accs {
|
||||
_, err := iam.GetUserAccount(acc)
|
||||
if err != nil {
|
||||
if err == ErrNoSuchUser {
|
||||
result = append(result, acc)
|
||||
continue
|
||||
}
|
||||
if err == ErrNotSupported {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
return nil, fmt.Errorf("check user account: %w", err)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func splitUnique(s, divider string) []string {
|
||||
elements := strings.Split(s, divider)
|
||||
uniqueElements := make(map[string]bool)
|
||||
result := make([]string, 0, len(elements))
|
||||
|
||||
for _, element := range elements {
|
||||
if _, ok := uniqueElements[element]; !ok {
|
||||
result = append(result, element)
|
||||
uniqueElements[element] = true
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func verifyACL(acl ACL, access string, permission types.Permission) error {
|
||||
if acl.ACL != "" {
|
||||
if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
return nil
|
||||
} else {
|
||||
if len(acl.Grantees) == 0 {
|
||||
return nil
|
||||
}
|
||||
grantee := Grantee{Access: access, Permission: permission}
|
||||
granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
|
||||
|
||||
isFound := false
|
||||
|
||||
for _, grt := range acl.Grantees {
|
||||
if grt == grantee || grt == granteeFullCtrl {
|
||||
isFound = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if isFound {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
func MayCreateBucket(acct Account, isRoot bool) error {
|
||||
if isRoot {
|
||||
return nil
|
||||
}
|
||||
|
||||
if acct.Role == RoleUser {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
|
||||
// Owner check
|
||||
if acct.Access == acl.Owner {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Root user has access over almost everything
|
||||
if isRoot {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Admin user case
|
||||
if acct.Role == RoleAdmin {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Return access denied in all other cases
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
type AccessOptions struct {
|
||||
Acl ACL
|
||||
AclPermission types.Permission
|
||||
IsRoot bool
|
||||
Acc Account
|
||||
Bucket string
|
||||
Object string
|
||||
Action Action
|
||||
}
|
||||
|
||||
func VerifyAccess(ctx context.Context, be backend.Backend, opts AccessOptions) error {
|
||||
if opts.IsRoot {
|
||||
return nil
|
||||
}
|
||||
if opts.Acc.Role == RoleAdmin {
|
||||
return nil
|
||||
}
|
||||
if opts.Acc.Access == opts.Acl.Owner {
|
||||
return nil
|
||||
}
|
||||
|
||||
policy, policyErr := be.GetBucketPolicy(ctx, opts.Bucket)
|
||||
if policyErr != nil && !errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
|
||||
return policyErr
|
||||
}
|
||||
|
||||
// If bucket policy is not set and the ACL is default, only the owner has access
|
||||
if errors.Is(policyErr, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) && opts.Acl.ACL == "" && len(opts.Acl.Grantees) == 0 {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
if err := verifyBucketPolicy(policy, opts.Acc.Access, opts.Bucket, opts.Object, opts.Action); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verifyACL(opts.Acl, opts.Acc.Access, opts.AclPermission); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func VerifyObjectCopyAccess(ctx context.Context, be backend.Backend, copySource string, opts AccessOptions) error {
|
||||
if opts.IsRoot {
|
||||
return nil
|
||||
}
|
||||
if opts.Acc.Role == RoleAdmin {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Verify destination bucket access
|
||||
if err := VerifyAccess(ctx, be, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
// Verify source bucket access
|
||||
srcBucket, srcObject, found := strings.Cut(copySource, "/")
|
||||
if !found {
|
||||
return s3err.GetAPIError(s3err.ErrInvalidCopySource)
|
||||
}
|
||||
|
||||
// Get source bucket ACL
|
||||
srcBucketACLBytes, err := be.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: &srcBucket})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var srcBucketAcl ACL
|
||||
if err := json.Unmarshal(srcBucketACLBytes, &srcBucketAcl); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := VerifyAccess(ctx, be, AccessOptions{
|
||||
Acl: srcBucketAcl,
|
||||
AclPermission: types.PermissionRead,
|
||||
IsRoot: opts.IsRoot,
|
||||
Acc: opts.Acc,
|
||||
Bucket: srcBucket,
|
||||
Object: srcObject,
|
||||
Action: GetObjectAction,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
type BucketPolicy struct {
|
||||
Statement []BucketPolicyItem `json:"Statement"`
|
||||
}
|
||||
|
||||
func (bp *BucketPolicy) Validate(bucket string, iam IAMService) error {
|
||||
for _, statement := range bp.Statement {
|
||||
err := statement.Validate(bucket, iam)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (bp *BucketPolicy) isAllowed(principal string, action Action, resource string) bool {
|
||||
for _, statement := range bp.Statement {
|
||||
if statement.findMatch(principal, action, resource) {
|
||||
switch statement.Effect {
|
||||
case BucketPolicyAccessTypeAllow:
|
||||
return true
|
||||
case BucketPolicyAccessTypeDeny:
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
type BucketPolicyItem struct {
|
||||
Effect BucketPolicyAccessType `json:"Effect"`
|
||||
Principals Principals `json:"Principal"`
|
||||
Actions Actions `json:"Action"`
|
||||
Resources Resources `json:"Resource"`
|
||||
}
|
||||
|
||||
func (bpi *BucketPolicyItem) Validate(bucket string, iam IAMService) error {
|
||||
if err := bpi.Effect.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bpi.Principals.Validate(iam); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bpi.Resources.Validate(bucket); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
containsObjectAction := bpi.Resources.ContainsObjectPattern()
|
||||
containsBucketAction := bpi.Resources.ContainsBucketPattern()
|
||||
|
||||
for action := range bpi.Actions {
|
||||
isObjectAction := action.IsObjectAction()
|
||||
if isObjectAction && !containsObjectAction {
|
||||
return fmt.Errorf("unsupported object action '%v' on the specified resources", action)
|
||||
}
|
||||
if !isObjectAction && !containsBucketAction {
|
||||
return fmt.Errorf("unsupported bucket action '%v' on the specified resources", action)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (bpi *BucketPolicyItem) findMatch(principal string, action Action, resource string) bool {
|
||||
if bpi.Principals.Contains(principal) && bpi.Actions.FindMatch(action) && bpi.Resources.FindMatch(resource) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func getMalformedPolicyError(err error) error {
|
||||
return s3err.APIError{
|
||||
Code: "MalformedPolicy",
|
||||
Description: err.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
}
|
||||
|
||||
func ValidatePolicyDocument(policyBin []byte, bucket string, iam IAMService) error {
|
||||
var policy BucketPolicy
|
||||
if err := json.Unmarshal(policyBin, &policy); err != nil {
|
||||
return getMalformedPolicyError(err)
|
||||
}
|
||||
|
||||
if err := policy.Validate(bucket, iam); err != nil {
|
||||
return getMalformedPolicyError(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyBucketPolicy(policy []byte, access, bucket, object string, action Action) error {
|
||||
// If bucket policy is not set
|
||||
if policy == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var bucketPolicy BucketPolicy
|
||||
if err := json.Unmarshal(policy, &bucketPolicy); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resource := bucket
|
||||
if object != "" {
|
||||
resource += "/" + object
|
||||
}
|
||||
|
||||
if !bucketPolicy.isAllowed(access, action, resource) {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Action string
|
||||
|
||||
const (
|
||||
GetBucketAclAction Action = "s3:GetBucketAcl"
|
||||
CreateBucketAction Action = "s3:CreateBucket"
|
||||
PutBucketAclAction Action = "s3:PutBucketAcl"
|
||||
DeleteBucketAction Action = "s3:DeleteBucket"
|
||||
PutBucketVersioningAction Action = "s3:PutBucketVersioning"
|
||||
GetBucketVersioningAction Action = "s3:GetBucketVersioning"
|
||||
PutBucketPolicyAction Action = "s3:PutBucketPolicy"
|
||||
GetBucketPolicyAction Action = "s3:GetBucketPolicy"
|
||||
DeleteBucketPolicyAction Action = "s3:DeleteBucketPolicy"
|
||||
AbortMultipartUploadAction Action = "s3:AbortMultipartUpload"
|
||||
ListMultipartUploadPartsAction Action = "s3:ListMultipartUploadParts"
|
||||
ListBucketMultipartUploadsAction Action = "s3:ListBucketMultipartUploads"
|
||||
PutObjectAction Action = "s3:PutObject"
|
||||
GetObjectAction Action = "s3:GetObject"
|
||||
DeleteObjectAction Action = "s3:DeleteObject"
|
||||
GetObjectAclAction Action = "s3:GetObjectAcl"
|
||||
GetObjectAttributesAction Action = "s3:GetObjectAttributes"
|
||||
PutObjectAclAction Action = "s3:PutObjectAcl"
|
||||
RestoreObjectAction Action = "s3:RestoreObject"
|
||||
GetBucketTaggingAction Action = "s3:GetBucketTagging"
|
||||
PutBucketTaggingAction Action = "s3:PutBucketTagging"
|
||||
GetObjectTaggingAction Action = "s3:GetObjectTagging"
|
||||
PutObjectTaggingAction Action = "s3:PutObjectTagging"
|
||||
DeleteObjectTaggingAction Action = "s3:DeleteObjectTagging"
|
||||
ListBucketVersionsAction Action = "s3:ListBucketVersions"
|
||||
ListBucketAction Action = "s3:ListBucket"
|
||||
GetBucketObjectLockConfigurationAction Action = "s3:GetBucketObjectLockConfiguration"
|
||||
PutBucketObjectLockConfigurationAction Action = "s3:PutBucketObjectLockConfiguration"
|
||||
GetObjectLegalHoldAction Action = "s3:GetObjectLegalHold"
|
||||
PutObjectLegalHoldAction Action = "s3:PutObjectLegalHold"
|
||||
GetObjectRetentionAction Action = "s3:GetObjectRetention"
|
||||
PutObjectRetentionAction Action = "s3:PutObjectRetention"
|
||||
BypassGovernanceRetentionAction Action = "s3:BypassGovernanceRetention"
|
||||
AllActions Action = "s3:*"
|
||||
)
|
||||
|
||||
var supportedActionList = map[Action]struct{}{
|
||||
GetBucketAclAction: {},
|
||||
CreateBucketAction: {},
|
||||
PutBucketAclAction: {},
|
||||
DeleteBucketAction: {},
|
||||
PutBucketVersioningAction: {},
|
||||
GetBucketVersioningAction: {},
|
||||
PutBucketPolicyAction: {},
|
||||
GetBucketPolicyAction: {},
|
||||
DeleteBucketPolicyAction: {},
|
||||
AbortMultipartUploadAction: {},
|
||||
ListMultipartUploadPartsAction: {},
|
||||
ListBucketMultipartUploadsAction: {},
|
||||
PutObjectAction: {},
|
||||
GetObjectAction: {},
|
||||
DeleteObjectAction: {},
|
||||
GetObjectAclAction: {},
|
||||
GetObjectAttributesAction: {},
|
||||
PutObjectAclAction: {},
|
||||
RestoreObjectAction: {},
|
||||
GetBucketTaggingAction: {},
|
||||
PutBucketTaggingAction: {},
|
||||
GetObjectTaggingAction: {},
|
||||
PutObjectTaggingAction: {},
|
||||
DeleteObjectTaggingAction: {},
|
||||
ListBucketVersionsAction: {},
|
||||
ListBucketAction: {},
|
||||
PutBucketObjectLockConfigurationAction: {},
|
||||
GetObjectLegalHoldAction: {},
|
||||
PutObjectLegalHoldAction: {},
|
||||
GetObjectRetentionAction: {},
|
||||
PutObjectRetentionAction: {},
|
||||
BypassGovernanceRetentionAction: {},
|
||||
AllActions: {},
|
||||
}
|
||||
|
||||
var supportedObjectActionList = map[Action]struct{}{
|
||||
AbortMultipartUploadAction: {},
|
||||
ListMultipartUploadPartsAction: {},
|
||||
PutObjectAction: {},
|
||||
GetObjectAction: {},
|
||||
DeleteObjectAction: {},
|
||||
GetObjectAclAction: {},
|
||||
GetObjectAttributesAction: {},
|
||||
PutObjectAclAction: {},
|
||||
RestoreObjectAction: {},
|
||||
GetObjectTaggingAction: {},
|
||||
PutObjectTaggingAction: {},
|
||||
DeleteObjectTaggingAction: {},
|
||||
GetObjectLegalHoldAction: {},
|
||||
PutObjectLegalHoldAction: {},
|
||||
GetObjectRetentionAction: {},
|
||||
PutObjectRetentionAction: {},
|
||||
BypassGovernanceRetentionAction: {},
|
||||
AllActions: {},
|
||||
}
|
||||
|
||||
// Validates Action: it should either wildcard match with supported actions list or be in it
|
||||
func (a Action) IsValid() error {
|
||||
if !strings.HasPrefix(string(a), "s3:") {
|
||||
return fmt.Errorf("invalid action: %v", a)
|
||||
}
|
||||
|
||||
if a == AllActions {
|
||||
return nil
|
||||
}
|
||||
|
||||
if a[len(a)-1] == '*' {
|
||||
pattern := strings.TrimSuffix(string(a), "*")
|
||||
for act := range supportedActionList {
|
||||
if strings.HasPrefix(string(act), pattern) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("invalid wildcard usage: %v prefix is not in the supported actions list", pattern)
|
||||
}
|
||||
|
||||
_, found := supportedActionList[a]
|
||||
if !found {
|
||||
return fmt.Errorf("unsupported action: %v", a)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Checks if the action is object action
|
||||
func (a Action) IsObjectAction() bool {
|
||||
if a[len(a)-1] == '*' {
|
||||
pattern := strings.TrimSuffix(string(a), "*")
|
||||
for act := range supportedObjectActionList {
|
||||
if strings.HasPrefix(string(act), pattern) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
_, found := supportedObjectActionList[a]
|
||||
return found
|
||||
}
|
||||
|
||||
func (a Action) WildCardMatch(act Action) bool {
|
||||
if strings.HasSuffix(string(a), "*") {
|
||||
pattern := strings.TrimSuffix(string(a), "*")
|
||||
return strings.HasPrefix(string(act), pattern)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type Actions map[Action]struct{}
|
||||
|
||||
// Override UnmarshalJSON method to decode both []string and string properties
|
||||
func (a *Actions) UnmarshalJSON(data []byte) error {
|
||||
ss := []string{}
|
||||
var err error
|
||||
if err = json.Unmarshal(data, &ss); err == nil {
|
||||
if len(ss) == 0 {
|
||||
return fmt.Errorf("actions can't be empty")
|
||||
}
|
||||
*a = make(Actions)
|
||||
for _, s := range ss {
|
||||
err = a.Add(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
var s string
|
||||
if err = json.Unmarshal(data, &s); err == nil {
|
||||
if s == "" {
|
||||
return fmt.Errorf("actions can't be empty")
|
||||
}
|
||||
*a = make(Actions)
|
||||
err = a.Add(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Validates and adds a new Action to Actions map
|
||||
func (a Actions) Add(str string) error {
|
||||
action := Action(str)
|
||||
err := action.IsValid()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
a[action] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a Actions) FindMatch(action Action) bool {
|
||||
_, ok := a[AllActions]
|
||||
if ok {
|
||||
return true
|
||||
}
|
||||
// First O(1) check for non wildcard actions
|
||||
_, found := a[action]
|
||||
if found {
|
||||
return true
|
||||
}
|
||||
|
||||
for act := range a {
|
||||
if strings.HasSuffix(string(act), "*") && act.WildCardMatch(action) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import "fmt"
|
||||
|
||||
type BucketPolicyAccessType string
|
||||
|
||||
const (
|
||||
BucketPolicyAccessTypeDeny BucketPolicyAccessType = "Deny"
|
||||
BucketPolicyAccessTypeAllow BucketPolicyAccessType = "Allow"
|
||||
)
|
||||
|
||||
// Checks policy statement Effect to be valid ("Deny", "Allow")
|
||||
func (bpat BucketPolicyAccessType) Validate() error {
|
||||
switch bpat {
|
||||
case BucketPolicyAccessTypeAllow, BucketPolicyAccessTypeDeny:
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("invalid effect: %v", bpat)
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type Principals map[string]struct{}
|
||||
|
||||
func (p Principals) Add(key string) {
|
||||
p[key] = struct{}{}
|
||||
}
|
||||
|
||||
// Override UnmarshalJSON method to decode both []string and string properties
|
||||
func (p *Principals) UnmarshalJSON(data []byte) error {
|
||||
ss := []string{}
|
||||
var err error
|
||||
if err = json.Unmarshal(data, &ss); err == nil {
|
||||
if len(ss) == 0 {
|
||||
return fmt.Errorf("principals can't be empty")
|
||||
}
|
||||
*p = make(Principals)
|
||||
for _, s := range ss {
|
||||
p.Add(s)
|
||||
}
|
||||
} else {
|
||||
var s string
|
||||
if err = json.Unmarshal(data, &s); err == nil {
|
||||
if s == "" {
|
||||
return fmt.Errorf("principals can't be empty")
|
||||
}
|
||||
*p = make(Principals)
|
||||
p.Add(s)
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Converts Principals map to a slice, by omitting "*"
|
||||
func (p Principals) ToSlice() []string {
|
||||
principals := []string{}
|
||||
for p := range p {
|
||||
if p == "*" {
|
||||
continue
|
||||
}
|
||||
principals = append(principals, p)
|
||||
}
|
||||
|
||||
return principals
|
||||
}
|
||||
|
||||
// Validates Principals by checking user account access keys existence
|
||||
func (p Principals) Validate(iam IAMService) error {
|
||||
_, containsWildCard := p["*"]
|
||||
if containsWildCard {
|
||||
if len(p) == 1 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("principals should either contain * or user access keys")
|
||||
}
|
||||
|
||||
accs, err := CheckIfAccountsExist(p.ToSlice(), iam)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(accs) > 0 {
|
||||
return fmt.Errorf("user accounts don't exist: %v", accs)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p Principals) Contains(userAccess string) bool {
|
||||
// "*" means it matches for any user account
|
||||
_, ok := p["*"]
|
||||
if ok {
|
||||
return true
|
||||
}
|
||||
|
||||
_, found := p[userAccess]
|
||||
return found
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Resources map[string]struct{}
|
||||
|
||||
const ResourceArnPrefix = "arn:aws:s3:::"
|
||||
|
||||
// Override UnmarshalJSON method to decode both []string and string properties
|
||||
func (r *Resources) UnmarshalJSON(data []byte) error {
|
||||
ss := []string{}
|
||||
var err error
|
||||
if err = json.Unmarshal(data, &ss); err == nil {
|
||||
if len(ss) == 0 {
|
||||
return fmt.Errorf("resources can't be empty")
|
||||
}
|
||||
*r = make(Resources)
|
||||
for _, s := range ss {
|
||||
err = r.Add(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
var s string
|
||||
if err = json.Unmarshal(data, &s); err == nil {
|
||||
if s == "" {
|
||||
return fmt.Errorf("resources can't be empty")
|
||||
}
|
||||
*r = make(Resources)
|
||||
err = r.Add(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Adds and validates a new resource to Resources map
|
||||
func (r Resources) Add(rc string) error {
|
||||
ok, pattern := isValidResource(rc)
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid resource: %v", rc)
|
||||
}
|
||||
|
||||
_, found := r[pattern]
|
||||
if found {
|
||||
return fmt.Errorf("duplicate resource: %v", rc)
|
||||
}
|
||||
|
||||
r[pattern] = struct{}{}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Checks if the resources contain object pattern
|
||||
func (r Resources) ContainsObjectPattern() bool {
|
||||
for resource := range r {
|
||||
if resource == "*" || strings.Contains(resource, "/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Checks if the resources contain bucket pattern
|
||||
func (r Resources) ContainsBucketPattern() bool {
|
||||
for resource := range r {
|
||||
if resource == "*" || !strings.Contains(resource, "/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Bucket resources should start with bucket name: arn:aws:s3:::MyBucket/*
|
||||
func (r Resources) Validate(bucket string) error {
|
||||
for resource := range r {
|
||||
if !strings.HasPrefix(resource, bucket) {
|
||||
return fmt.Errorf("incorrect bucket name in %v", resource)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r Resources) FindMatch(resource string) bool {
|
||||
for res := range r {
|
||||
if strings.HasSuffix(res, "*") {
|
||||
pattern := strings.TrimSuffix(res, "*")
|
||||
if strings.HasPrefix(resource, pattern) {
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
if res == resource {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Checks the resource to have arn prefix and not starting with /
|
||||
func isValidResource(rc string) (isValid bool, pattern string) {
|
||||
if !strings.HasPrefix(rc, ResourceArnPrefix) {
|
||||
return false, ""
|
||||
}
|
||||
|
||||
res := strings.TrimPrefix(rc, ResourceArnPrefix)
|
||||
if res == "" {
|
||||
return false, ""
|
||||
}
|
||||
// The resource can't start with / (bucket name comes first)
|
||||
if strings.HasPrefix(res, "/") {
|
||||
return false, ""
|
||||
}
|
||||
|
||||
return true, res
|
||||
}
|
||||
111
auth/iam.go
@@ -1,111 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Role string
|
||||
|
||||
const (
|
||||
RoleUser Role = "user"
|
||||
RoleAdmin Role = "admin"
|
||||
RoleUserPlus Role = "userplus"
|
||||
)
|
||||
|
||||
// Account is a gateway IAM account
|
||||
type Account struct {
|
||||
Access string `json:"access"`
|
||||
Secret string `json:"secret"`
|
||||
Role Role `json:"role"`
|
||||
UserID int `json:"userID"`
|
||||
GroupID int `json:"groupID"`
|
||||
ProjectID int `json:"projectID"`
|
||||
}
|
||||
|
||||
// IAMService is the interface for all IAM service implementations
|
||||
//
|
||||
//go:generate moq -out ../s3api/controllers/iam_moq_test.go -pkg controllers . IAMService
|
||||
type IAMService interface {
|
||||
CreateAccount(account Account) error
|
||||
GetUserAccount(access string) (Account, error)
|
||||
DeleteUserAccount(access string) error
|
||||
ListUserAccounts() ([]Account, error)
|
||||
Shutdown() error
|
||||
}
|
||||
|
||||
var ErrNoSuchUser = errors.New("user not found")
|
||||
|
||||
type Opts struct {
|
||||
Dir string
|
||||
LDAPServerURL string
|
||||
LDAPBindDN string
|
||||
LDAPPassword string
|
||||
LDAPQueryBase string
|
||||
LDAPObjClasses string
|
||||
LDAPAccessAtr string
|
||||
LDAPSecretAtr string
|
||||
LDAPRoleAtr string
|
||||
S3Access string
|
||||
S3Secret string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3Endpoint string
|
||||
S3DisableSSlVerfiy bool
|
||||
S3Debug bool
|
||||
CacheDisable bool
|
||||
CacheTTL int
|
||||
CachePrune int
|
||||
}
|
||||
|
||||
func New(o *Opts) (IAMService, error) {
|
||||
var svc IAMService
|
||||
var err error
|
||||
|
||||
switch {
|
||||
case o.Dir != "":
|
||||
svc, err = NewInternal(o.Dir)
|
||||
fmt.Printf("initializing internal IAM with %q\n", o.Dir)
|
||||
case o.LDAPServerURL != "":
|
||||
svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
|
||||
o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
|
||||
o.LDAPObjClasses)
|
||||
fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
|
||||
case o.S3Endpoint != "":
|
||||
svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
|
||||
o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
|
||||
fmt.Printf("initializing S3 IAM with '%v/%v'\n",
|
||||
o.S3Endpoint, o.S3Bucket)
|
||||
default:
|
||||
// if no iam options selected, default to the single user mode
|
||||
fmt.Println("No IAM service configured, enabling single account mode")
|
||||
return IAMServiceSingle{}, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if o.CacheDisable {
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
return NewCache(svc,
|
||||
time.Duration(o.CacheTTL)*time.Second,
|
||||
time.Duration(o.CachePrune)*time.Second), nil
|
||||
}
|
||||
@@ -1,179 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// IAMCache is an in memory cache of the IAM accounts
|
||||
// with expiration. This helps to alleviate the load on
|
||||
// the real IAM service if the gateway is handling
|
||||
// many requests. This forwards account updates to the
|
||||
// underlying service, and returns cached results while
|
||||
// the in memory account is not expired.
|
||||
type IAMCache struct {
|
||||
service IAMService
|
||||
iamcache *icache
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
var _ IAMService = &IAMCache{}
|
||||
|
||||
type item struct {
|
||||
value Account
|
||||
exp time.Time
|
||||
}
|
||||
|
||||
type icache struct {
|
||||
sync.RWMutex
|
||||
expire time.Duration
|
||||
items map[string]item
|
||||
}
|
||||
|
||||
func (i *icache) set(k string, v Account) {
|
||||
cpy := v
|
||||
i.Lock()
|
||||
i.items[k] = item{
|
||||
exp: time.Now().Add(i.expire),
|
||||
value: cpy,
|
||||
}
|
||||
i.Unlock()
|
||||
}
|
||||
|
||||
func (i *icache) get(k string) (Account, bool) {
|
||||
i.RLock()
|
||||
v, ok := i.items[k]
|
||||
i.RUnlock()
|
||||
if !ok || !v.exp.After(time.Now()) {
|
||||
return Account{}, false
|
||||
}
|
||||
return v.value, true
|
||||
}
|
||||
|
||||
func (i *icache) Delete(k string) {
|
||||
i.Lock()
|
||||
delete(i.items, k)
|
||||
i.Unlock()
|
||||
}
|
||||
|
||||
func (i *icache) gcCache(ctx context.Context, interval time.Duration) {
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
i.Lock()
|
||||
// prune expired entries
|
||||
for k, v := range i.items {
|
||||
if now.After(v.exp) {
|
||||
delete(i.items, k)
|
||||
}
|
||||
}
|
||||
i.Unlock()
|
||||
|
||||
// sleep for the clean interval or context cancelation,
|
||||
// whichever comes first
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(interval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NewCache initializes an IAM cache for the provided service. The expireTime
|
||||
// is the duration a cache entry can be valid, and the cleanupInterval is
|
||||
// how often to scan cache and cleanup expired entries.
|
||||
func NewCache(service IAMService, expireTime, cleanupInterval time.Duration) *IAMCache {
|
||||
i := &IAMCache{
|
||||
service: service,
|
||||
iamcache: &icache{
|
||||
items: make(map[string]item),
|
||||
expire: expireTime,
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go i.iamcache.gcCache(ctx, cleanupInterval)
|
||||
i.cancel = cancel
|
||||
|
||||
return i
|
||||
}
|
||||
|
||||
// CreateAccount send create to IAM service and creates an account cache entry
|
||||
func (c *IAMCache) CreateAccount(account Account) error {
|
||||
err := c.service.CreateAccount(account)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// we need a copy of account to be able to store beyond the
|
||||
// lifetime of the request, otherwise Fiber will reuse and corrupt
|
||||
// these entries
|
||||
acct := Account{
|
||||
Access: strings.Clone(account.Access),
|
||||
Secret: strings.Clone(account.Secret),
|
||||
Role: Role(strings.Clone(string(account.Role))),
|
||||
}
|
||||
|
||||
c.iamcache.set(acct.Access, acct)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetUserAccount retrieves the cache account if it is in the cache and not
|
||||
// expired. Otherwise retrieves from underlying IAM service and caches
|
||||
// result for the expire duration.
|
||||
func (c *IAMCache) GetUserAccount(access string) (Account, error) {
|
||||
acct, found := c.iamcache.get(access)
|
||||
if found {
|
||||
return acct, nil
|
||||
}
|
||||
|
||||
a, err := c.service.GetUserAccount(access)
|
||||
if err != nil {
|
||||
return Account{}, err
|
||||
}
|
||||
|
||||
c.iamcache.set(access, a)
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// DeleteUserAccount deletes account from IAM service and cache
|
||||
func (c *IAMCache) DeleteUserAccount(access string) error {
|
||||
err := c.service.DeleteUserAccount(access)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.iamcache.Delete(access)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListUserAccounts is a passthrough to the underlying service and
|
||||
// does not make use of the cache
|
||||
func (c *IAMCache) ListUserAccounts() ([]Account, error) {
|
||||
return c.service.ListUserAccounts()
|
||||
}
|
||||
|
||||
// Shutdown graceful termination of service
|
||||
func (c *IAMCache) Shutdown() error {
|
||||
c.cancel()
|
||||
return nil
|
||||
}
|
||||
@@ -1,335 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
iamFile = "users.json"
|
||||
iamBackupFile = "users.json.backup"
|
||||
)
|
||||
|
||||
// IAMServiceInternal manages the internal IAM service
|
||||
type IAMServiceInternal struct {
|
||||
dir string
|
||||
}
|
||||
|
||||
// UpdateAcctFunc accepts the current data and returns the new data to be stored
|
||||
type UpdateAcctFunc func([]byte) ([]byte, error)
|
||||
|
||||
// iAMConfig stores all internal IAM accounts
|
||||
type iAMConfig struct {
|
||||
AccessAccounts map[string]Account `json:"accessAccounts"`
|
||||
}
|
||||
|
||||
var _ IAMService = &IAMServiceInternal{}
|
||||
|
||||
// NewInternal creates a new instance for the Internal IAM service
|
||||
func NewInternal(dir string) (*IAMServiceInternal, error) {
|
||||
i := &IAMServiceInternal{
|
||||
dir: dir,
|
||||
}
|
||||
|
||||
err := i.initIAM()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init iam: %w", err)
|
||||
}
|
||||
|
||||
return i, nil
|
||||
}
|
||||
|
||||
// CreateAccount creates a new IAM account. Returns an error if the account
|
||||
// already exists.
|
||||
func (s *IAMServiceInternal) CreateAccount(account Account) error {
|
||||
return s.storeIAM(func(data []byte) ([]byte, error) {
|
||||
conf, err := parseIAM(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get iam data: %w", err)
|
||||
}
|
||||
|
||||
_, ok := conf.AccessAccounts[account.Access]
|
||||
if ok {
|
||||
return nil, fmt.Errorf("account already exists")
|
||||
}
|
||||
conf.AccessAccounts[account.Access] = account
|
||||
|
||||
b, err := json.Marshal(conf)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to serialize iam: %w", err)
|
||||
}
|
||||
|
||||
return b, nil
|
||||
})
|
||||
}
|
||||
|
||||
// GetUserAccount retrieves account info for the requested user. Returns
|
||||
// ErrNoSuchUser if the account does not exist.
|
||||
func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
|
||||
conf, err := s.getIAM()
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("get iam data: %w", err)
|
||||
}
|
||||
|
||||
acct, ok := conf.AccessAccounts[access]
|
||||
if !ok {
|
||||
return Account{}, ErrNoSuchUser
|
||||
}
|
||||
|
||||
return acct, nil
|
||||
}
|
||||
|
||||
// DeleteUserAccount deletes the specified user account. Does not check if
|
||||
// account exists.
|
||||
func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
|
||||
return s.storeIAM(func(data []byte) ([]byte, error) {
|
||||
conf, err := parseIAM(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get iam data: %w", err)
|
||||
}
|
||||
|
||||
delete(conf.AccessAccounts, access)
|
||||
|
||||
b, err := json.Marshal(conf)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to serialize iam: %w", err)
|
||||
}
|
||||
|
||||
return b, nil
|
||||
})
|
||||
}
|
||||
|
||||
// ListUserAccounts lists all the user accounts stored.
|
||||
func (s *IAMServiceInternal) ListUserAccounts() ([]Account, error) {
|
||||
conf, err := s.getIAM()
|
||||
if err != nil {
|
||||
return []Account{}, fmt.Errorf("get iam data: %w", err)
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(conf.AccessAccounts))
|
||||
for k := range conf.AccessAccounts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var accs []Account
|
||||
for _, k := range keys {
|
||||
accs = append(accs, Account{
|
||||
Access: k,
|
||||
Secret: conf.AccessAccounts[k].Secret,
|
||||
Role: conf.AccessAccounts[k].Role,
|
||||
UserID: conf.AccessAccounts[k].UserID,
|
||||
GroupID: conf.AccessAccounts[k].GroupID,
|
||||
ProjectID: conf.AccessAccounts[k].ProjectID,
|
||||
})
|
||||
}
|
||||
|
||||
return accs, nil
|
||||
}
|
||||
|
||||
// Shutdown graceful termination of service
|
||||
func (s *IAMServiceInternal) Shutdown() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
const (
|
||||
iamMode = 0600
|
||||
)
|
||||
|
||||
func (s *IAMServiceInternal) initIAM() error {
|
||||
fname := filepath.Join(s.dir, iamFile)
|
||||
|
||||
_, err := os.ReadFile(fname)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
b, err := json.Marshal(iAMConfig{AccessAccounts: map[string]Account{}})
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal default iam: %w", err)
|
||||
}
|
||||
err = os.WriteFile(fname, b, iamMode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("write default iam: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceInternal) getIAM() (iAMConfig, error) {
|
||||
b, err := s.readIAMData()
|
||||
if err != nil {
|
||||
return iAMConfig{}, err
|
||||
}
|
||||
|
||||
return parseIAM(b)
|
||||
}
|
||||
|
||||
func parseIAM(b []byte) (iAMConfig, error) {
|
||||
var conf iAMConfig
|
||||
if err := json.Unmarshal(b, &conf); err != nil {
|
||||
return iAMConfig{}, fmt.Errorf("failed to parse the config file: %w", err)
|
||||
}
|
||||
|
||||
return conf, nil
|
||||
}
|
||||
|
||||
const (
|
||||
backoff = 100 * time.Millisecond
|
||||
maxretry = 300
|
||||
)
|
||||
|
||||
func (s *IAMServiceInternal) readIAMData() ([]byte, error) {
|
||||
// We are going to be racing with other running gateways without any
|
||||
// coordination. So we might find the file does not exist at times.
|
||||
// For this case we need to retry for a while assuming the other gateway
|
||||
// will eventually write the file. If it doesn't after the max retries,
|
||||
// then we will return the error.
|
||||
|
||||
retries := 0
|
||||
|
||||
for {
|
||||
b, err := os.ReadFile(filepath.Join(s.dir, iamFile))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// racing with someone else updating
|
||||
// keep retrying after backoff
|
||||
retries++
|
||||
if retries < maxretry {
|
||||
time.Sleep(backoff)
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("read iam file: %w", err)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return b, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *IAMServiceInternal) storeIAM(update UpdateAcctFunc) error {
|
||||
// We are going to be racing with other running gateways without any
|
||||
// coordination. So the strategy here is to read the current file data.
|
||||
// If the file doesn't exist, then we assume someone else is currently
|
||||
// updating the file. So we just need to keep retrying. We also need
|
||||
// to make sure the data is consistent within a single update. So racing
|
||||
// writes to a file would possibly leave this in some invalid state.
|
||||
// We can get atomic updates with rename. If we read the data, update
|
||||
// the data, write to a temp file, then rename the tempfile back to the
|
||||
// data file. This should always result in a complete data image.
|
||||
|
||||
// There is at least one unsolved failure mode here.
|
||||
// If a gateway removes the data file and then crashes, all other
|
||||
// gateways will retry forever thinking that the original will eventually
|
||||
// write the file.
|
||||
|
||||
retries := 0
|
||||
fname := filepath.Join(s.dir, iamFile)
|
||||
|
||||
for {
|
||||
b, err := os.ReadFile(fname)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// racing with someone else updating
|
||||
// keep retrying after backoff
|
||||
retries++
|
||||
if retries < maxretry {
|
||||
time.Sleep(backoff)
|
||||
continue
|
||||
}
|
||||
|
||||
// we have been unsuccessful trying to read the iam file
|
||||
// so this must be the case where something happened and
|
||||
// the file did not get updated successfully, and probably
|
||||
// isn't going to be. The recovery procedure would be to
|
||||
// copy the backup file into place of the original.
|
||||
return fmt.Errorf("no iam file, needs backup recovery")
|
||||
}
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("read iam file: %w", err)
|
||||
}
|
||||
|
||||
// reset retries on successful read
|
||||
retries = 0
|
||||
|
||||
err = os.Remove(fname)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// racing with someone else updating
|
||||
// keep retrying after backoff
|
||||
time.Sleep(backoff)
|
||||
continue
|
||||
}
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("remove old iam file: %w", err)
|
||||
}
|
||||
|
||||
// save copy of data
|
||||
datacopy := make([]byte, len(b))
|
||||
copy(datacopy, b)
|
||||
|
||||
// make a backup copy in case we crash before update
|
||||
// this is after remove, so there is a small window something
|
||||
// can go wrong, but the remove should barrier other gateways
|
||||
// from trying to write backup at the same time. Only one
|
||||
// gateway will successfully remove the file.
|
||||
os.WriteFile(filepath.Join(s.dir, iamBackupFile), b, iamMode)
|
||||
|
||||
b, err = update(b)
|
||||
if err != nil {
|
||||
// update failed, try to write old data back out
|
||||
os.WriteFile(fname, datacopy, iamMode)
|
||||
return fmt.Errorf("update iam data: %w", err)
|
||||
}
|
||||
|
||||
err = s.writeTempFile(b)
|
||||
if err != nil {
|
||||
// update failed, try to write old data back out
|
||||
os.WriteFile(fname, datacopy, iamMode)
|
||||
return err
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceInternal) writeTempFile(b []byte) error {
|
||||
fname := filepath.Join(s.dir, iamFile)
|
||||
|
||||
f, err := os.CreateTemp(s.dir, iamFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
_, err = f.Write(b)
|
||||
if err != nil {
|
||||
return fmt.Errorf("write temp file: %w", err)
|
||||
}
|
||||
|
||||
err = os.Rename(f.Name(), fname)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rename temp file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
133
auth/iam_ldap.go
@@ -1,133 +0,0 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/go-ldap/ldap/v3"
|
||||
)
|
||||
|
||||
type LdapIAMService struct {
|
||||
conn *ldap.Conn
|
||||
queryBase string
|
||||
objClasses []string
|
||||
accessAtr string
|
||||
secretAtr string
|
||||
roleAtr string
|
||||
}
|
||||
|
||||
var _ IAMService = &LdapIAMService{}
|
||||
|
||||
func NewLDAPService(url, bindDN, pass, queryBase, accAtr, secAtr, roleAtr, objClasses string) (IAMService, error) {
|
||||
if url == "" || bindDN == "" || pass == "" || queryBase == "" || accAtr == "" || secAtr == "" || roleAtr == "" || objClasses == "" {
|
||||
return nil, fmt.Errorf("required parameters list not fully provided")
|
||||
}
|
||||
conn, err := ldap.DialURL(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to LDAP server: %w", err)
|
||||
}
|
||||
|
||||
err = conn.Bind(bindDN, pass)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to bind to LDAP server %w", err)
|
||||
}
|
||||
return &LdapIAMService{
|
||||
conn: conn,
|
||||
queryBase: queryBase,
|
||||
objClasses: strings.Split(objClasses, ","),
|
||||
accessAtr: accAtr,
|
||||
secretAtr: secAtr,
|
||||
roleAtr: roleAtr,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ld *LdapIAMService) CreateAccount(account Account) error {
|
||||
userEntry := ldap.NewAddRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, account.Access, ld.queryBase), nil)
|
||||
userEntry.Attribute("objectClass", ld.objClasses)
|
||||
userEntry.Attribute(ld.accessAtr, []string{account.Access})
|
||||
userEntry.Attribute(ld.secretAtr, []string{account.Secret})
|
||||
userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
|
||||
|
||||
err := ld.conn.Add(userEntry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error adding an entry: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
|
||||
searchRequest := ldap.NewSearchRequest(
|
||||
ld.queryBase,
|
||||
ldap.ScopeWholeSubtree,
|
||||
ldap.NeverDerefAliases,
|
||||
0,
|
||||
0,
|
||||
false,
|
||||
fmt.Sprintf("(%v=%v)", ld.accessAtr, access),
|
||||
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
|
||||
nil,
|
||||
)
|
||||
|
||||
result, err := ld.conn.Search(searchRequest)
|
||||
if err != nil {
|
||||
return Account{}, err
|
||||
}
|
||||
|
||||
entry := result.Entries[0]
|
||||
return Account{
|
||||
Access: entry.GetAttributeValue(ld.accessAtr),
|
||||
Secret: entry.GetAttributeValue(ld.secretAtr),
|
||||
Role: Role(entry.GetAttributeValue(ld.roleAtr)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ld *LdapIAMService) DeleteUserAccount(access string) error {
|
||||
delReq := ldap.NewDelRequest(fmt.Sprintf("%v=%v, %v", ld.accessAtr, access, ld.queryBase), nil)
|
||||
|
||||
err := ld.conn.Del(delReq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
|
||||
searchFilter := ""
|
||||
for _, el := range ld.objClasses {
|
||||
searchFilter += fmt.Sprintf("(objectClass=%v)", el)
|
||||
}
|
||||
searchRequest := ldap.NewSearchRequest(
|
||||
ld.queryBase,
|
||||
ldap.ScopeWholeSubtree,
|
||||
ldap.NeverDerefAliases,
|
||||
0,
|
||||
0,
|
||||
false,
|
||||
fmt.Sprintf("(&%v)", searchFilter),
|
||||
[]string{ld.accessAtr, ld.secretAtr, ld.roleAtr},
|
||||
nil,
|
||||
)
|
||||
|
||||
resp, err := ld.conn.Search(searchRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := []Account{}
|
||||
for _, el := range resp.Entries {
|
||||
result = append(result, Account{
|
||||
Access: el.GetAttributeValue(ld.accessAtr),
|
||||
Secret: el.GetAttributeValue(ld.secretAtr),
|
||||
Role: Role(el.GetAttributeValue(ld.roleAtr)),
|
||||
})
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Shutdown graceful termination of service
|
||||
func (ld *LdapIAMService) Shutdown() error {
|
||||
return ld.conn.Close()
|
||||
}
|
||||
@@ -1,263 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
)
|
||||
|
||||
// IAMServiceS3 stores user accounts in an S3 object
|
||||
// The endpoint, credentials, bucket, and region are provided
|
||||
// from cli configuration.
|
||||
// The object format and name is the same as the internal IAM service:
|
||||
// coming from iAMConfig and iamFile in iam_internal.
|
||||
|
||||
type IAMServiceS3 struct {
|
||||
access string
|
||||
secret string
|
||||
region string
|
||||
bucket string
|
||||
endpoint string
|
||||
sslSkipVerify bool
|
||||
debug bool
|
||||
client *s3.Client
|
||||
}
|
||||
|
||||
var _ IAMService = &IAMServiceS3{}
|
||||
|
||||
func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
|
||||
if access == "" {
|
||||
return nil, fmt.Errorf("must provide s3 IAM service access key")
|
||||
}
|
||||
if secret == "" {
|
||||
return nil, fmt.Errorf("must provide s3 IAM service secret key")
|
||||
}
|
||||
if region == "" {
|
||||
return nil, fmt.Errorf("must provide s3 IAM service region")
|
||||
}
|
||||
if bucket == "" {
|
||||
return nil, fmt.Errorf("must provide s3 IAM service bucket")
|
||||
}
|
||||
if endpoint == "" {
|
||||
return nil, fmt.Errorf("must provide s3 IAM service endpoint")
|
||||
}
|
||||
|
||||
i := &IAMServiceS3{
|
||||
access: access,
|
||||
secret: secret,
|
||||
region: region,
|
||||
bucket: bucket,
|
||||
endpoint: endpoint,
|
||||
sslSkipVerify: sslSkipVerify,
|
||||
debug: debug,
|
||||
}
|
||||
|
||||
cfg, err := i.getConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init s3 IAM: %v", err)
|
||||
}
|
||||
|
||||
i.client = s3.NewFromConfig(cfg)
|
||||
return i, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) CreateAccount(account Account) error {
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, ok := conf.AccessAccounts[account.Access]
|
||||
if ok {
|
||||
return fmt.Errorf("account already exists")
|
||||
}
|
||||
conf.AccessAccounts[account.Access] = account
|
||||
|
||||
return s.storeAccts(conf)
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return Account{}, err
|
||||
}
|
||||
|
||||
acct, ok := conf.AccessAccounts[access]
|
||||
if !ok {
|
||||
return Account{}, ErrNoSuchUser
|
||||
}
|
||||
|
||||
return acct, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) DeleteUserAccount(access string) error {
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, ok := conf.AccessAccounts[access]
|
||||
if !ok {
|
||||
return fmt.Errorf("account does not exist")
|
||||
}
|
||||
delete(conf.AccessAccounts, access)
|
||||
|
||||
return s.storeAccts(conf)
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(conf.AccessAccounts))
|
||||
for k := range conf.AccessAccounts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var accs []Account
|
||||
for _, k := range keys {
|
||||
accs = append(accs, Account{
|
||||
Access: k,
|
||||
Secret: conf.AccessAccounts[k].Secret,
|
||||
Role: conf.AccessAccounts[k].Role,
|
||||
UserID: conf.AccessAccounts[k].UserID,
|
||||
GroupID: conf.AccessAccounts[k].GroupID,
|
||||
ProjectID: conf.AccessAccounts[k].ProjectID,
|
||||
})
|
||||
}
|
||||
|
||||
return accs, nil
|
||||
}
|
||||
|
||||
// ResolveEndpoint is used for on prem or non-aws endpoints
|
||||
func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
PartitionID: "aws",
|
||||
URL: s.endpoint,
|
||||
SigningRegion: s.region,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) Shutdown() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) getConfig() (aws.Config, error) {
|
||||
creds := credentials.NewStaticCredentialsProvider(s.access, s.secret, "")
|
||||
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
|
||||
}
|
||||
client := &http.Client{Transport: tr}
|
||||
|
||||
opts := []func(*config.LoadOptions) error{
|
||||
config.WithRegion(s.region),
|
||||
config.WithCredentialsProvider(creds),
|
||||
config.WithHTTPClient(client),
|
||||
}
|
||||
|
||||
if s.endpoint != "" {
|
||||
opts = append(opts,
|
||||
config.WithEndpointResolverWithOptions(s))
|
||||
}
|
||||
|
||||
if s.debug {
|
||||
opts = append(opts,
|
||||
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
|
||||
}
|
||||
|
||||
return config.LoadDefaultConfig(context.Background(), opts...)
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
|
||||
obj := iamFile
|
||||
|
||||
out, err := s.client.GetObject(context.Background(), &s3.GetObjectInput{
|
||||
Bucket: &s.bucket,
|
||||
Key: &obj,
|
||||
})
|
||||
if err != nil {
|
||||
// if the error is object not exists,
|
||||
// init empty accounts stuct and return that
|
||||
var nsk *types.NoSuchKey
|
||||
if errors.As(err, &nsk) {
|
||||
return iAMConfig{}, nil
|
||||
}
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
if apiErr.ErrorCode() == "NotFound" {
|
||||
return iAMConfig{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// all other errors, return the error
|
||||
return iAMConfig{}, fmt.Errorf("get %v: %w", obj, err)
|
||||
}
|
||||
|
||||
defer out.Body.Close()
|
||||
|
||||
b, err := io.ReadAll(out.Body)
|
||||
if err != nil {
|
||||
return iAMConfig{}, fmt.Errorf("read %v: %w", obj, err)
|
||||
}
|
||||
|
||||
conf, err := parseIAM(b)
|
||||
if err != nil {
|
||||
return iAMConfig{}, fmt.Errorf("parse iam data: %w", err)
|
||||
}
|
||||
|
||||
return conf, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) storeAccts(conf iAMConfig) error {
|
||||
b, err := json.Marshal(conf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to serialize iam: %w", err)
|
||||
}
|
||||
|
||||
obj := iamFile
|
||||
uploader := manager.NewUploader(s.client)
|
||||
upinfo := &s3.PutObjectInput{
|
||||
Body: bytes.NewReader(b),
|
||||
Bucket: &s.bucket,
|
||||
Key: &obj,
|
||||
}
|
||||
_, err = uploader.Upload(context.Background(), upinfo)
|
||||
if err != nil {
|
||||
return fmt.Errorf("store accounts in %v: %w", iamFile, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"errors"
|
||||
)
|
||||
|
||||
// IAMServiceSingle manages the single tenant (root-only) IAM service
|
||||
type IAMServiceSingle struct{}
|
||||
|
||||
var _ IAMService = &IAMServiceSingle{}
|
||||
|
||||
var ErrNotSupported = errors.New("method is not supported")
|
||||
|
||||
// CreateAccount not valid in single tenant mode
|
||||
func (IAMServiceSingle) CreateAccount(account Account) error {
|
||||
return ErrNotSupported
|
||||
}
|
||||
|
||||
// GetUserAccount no accounts in single tenant mode
|
||||
func (IAMServiceSingle) GetUserAccount(access string) (Account, error) {
|
||||
return Account{}, ErrNoSuchUser
|
||||
}
|
||||
|
||||
// DeleteUserAccount no accounts in single tenant mode
|
||||
func (IAMServiceSingle) DeleteUserAccount(access string) error {
|
||||
return ErrNotSupported
|
||||
}
|
||||
|
||||
// ListUserAccounts no accounts in single tenant mode
|
||||
func (IAMServiceSingle) ListUserAccounts() ([]Account, error) {
|
||||
return []Account{}, nil
|
||||
}
|
||||
|
||||
// Shutdown graceful termination of service
|
||||
func (IAMServiceSingle) Shutdown() error {
|
||||
return nil
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
type BucketLockConfig struct {
|
||||
Enabled bool
|
||||
DefaultRetention *types.DefaultRetention
|
||||
CreatedAt *time.Time
|
||||
}
|
||||
|
||||
func ParseBucketLockConfigurationInput(input []byte) ([]byte, error) {
|
||||
var lockConfig types.ObjectLockConfiguration
|
||||
if err := xml.Unmarshal(input, &lockConfig); err != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
config := BucketLockConfig{
|
||||
Enabled: lockConfig.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
|
||||
}
|
||||
|
||||
if lockConfig.Rule != nil && lockConfig.Rule.DefaultRetention != nil {
|
||||
retention := lockConfig.Rule.DefaultRetention
|
||||
if retention.Years != nil && retention.Days != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
config.DefaultRetention = retention
|
||||
now := time.Now()
|
||||
config.CreatedAt = &now
|
||||
}
|
||||
|
||||
return json.Marshal(config)
|
||||
}
|
||||
|
||||
func ParseBucketLockConfigurationOutput(input []byte) (*types.ObjectLockConfiguration, error) {
|
||||
var config BucketLockConfig
|
||||
if err := json.Unmarshal(input, &config); err != nil {
|
||||
return nil, fmt.Errorf("parse object lock config: %w", err)
|
||||
}
|
||||
|
||||
result := &types.ObjectLockConfiguration{
|
||||
Rule: &types.ObjectLockRule{
|
||||
DefaultRetention: config.DefaultRetention,
|
||||
},
|
||||
}
|
||||
|
||||
if config.Enabled {
|
||||
result.ObjectLockEnabled = types.ObjectLockEnabledEnabled
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func ParseObjectLockRetentionInput(input []byte) ([]byte, error) {
|
||||
var retention types.ObjectLockRetention
|
||||
if err := xml.Unmarshal(input, &retention); err != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
if retention.RetainUntilDate == nil || retention.RetainUntilDate.Before(time.Now()) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)
|
||||
}
|
||||
switch retention.Mode {
|
||||
case types.ObjectLockRetentionModeCompliance:
|
||||
case types.ObjectLockRetentionModeGovernance:
|
||||
default:
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
return json.Marshal(retention)
|
||||
}
|
||||
|
||||
func ParseObjectLockRetentionOutput(input []byte) (*types.ObjectLockRetention, error) {
|
||||
var retention types.ObjectLockRetention
|
||||
if err := json.Unmarshal(input, &retention); err != nil {
|
||||
return nil, fmt.Errorf("parse object lock retention: %w", err)
|
||||
}
|
||||
|
||||
return &retention, nil
|
||||
}
|
||||
|
||||
func ParseObjectLegalHoldOutput(status *bool) *types.ObjectLockLegalHold {
|
||||
if status == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if *status {
|
||||
return &types.ObjectLockLegalHold{
|
||||
Status: types.ObjectLockLegalHoldStatusOn,
|
||||
}
|
||||
}
|
||||
|
||||
return &types.ObjectLockLegalHold{
|
||||
Status: types.ObjectLockLegalHoldStatusOff,
|
||||
}
|
||||
}
|
||||
|
||||
func CheckObjectAccess(ctx context.Context, bucket, userAccess string, objects []string, isAdminOrRoot bool, be backend.Backend) error {
|
||||
data, err := be.GetObjectLockConfiguration(ctx, bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrObjectLockConfigurationNotFound)) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
var bucketLockConfig BucketLockConfig
|
||||
if err := json.Unmarshal(data, &bucketLockConfig); err != nil {
|
||||
return fmt.Errorf("parse object lock config: %w", err)
|
||||
}
|
||||
|
||||
if !bucketLockConfig.Enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
objExists := true
|
||||
|
||||
for _, obj := range objects {
|
||||
var checkRetention bool = true
|
||||
retentionData, err := be.GetObjectRetention(ctx, bucket, obj, "")
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchKey)) {
|
||||
objExists = false
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
|
||||
checkRetention = false
|
||||
}
|
||||
if err != nil && checkRetention {
|
||||
return err
|
||||
}
|
||||
|
||||
if checkRetention {
|
||||
retention, err := ParseObjectLockRetentionOutput(retentionData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if retention.Mode != "" && retention.RetainUntilDate != nil {
|
||||
if retention.RetainUntilDate.After(time.Now()) {
|
||||
switch retention.Mode {
|
||||
case types.ObjectLockRetentionModeGovernance:
|
||||
if !isAdminOrRoot {
|
||||
policy, err := be.GetBucketPolicy(ctx, bucket)
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchBucketPolicy)) {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = verifyBucketPolicy(policy, userAccess, bucket, obj, BypassGovernanceRetentionAction)
|
||||
if err != nil {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
}
|
||||
case types.ObjectLockRetentionModeCompliance:
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
status, err := be.GetObjectLegalHold(ctx, bucket, obj, "")
|
||||
if errors.Is(err, s3err.GetAPIError(s3err.ErrNoSuchObjectLockConfiguration)) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if *status && !isAdminOrRoot {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
}
|
||||
|
||||
if bucketLockConfig.DefaultRetention != nil && bucketLockConfig.CreatedAt != nil && objExists {
|
||||
expirationDate := *bucketLockConfig.CreatedAt
|
||||
if bucketLockConfig.DefaultRetention.Days != nil {
|
||||
expirationDate = expirationDate.AddDate(0, 0, int(*bucketLockConfig.DefaultRetention.Days))
|
||||
}
|
||||
if bucketLockConfig.DefaultRetention.Years != nil {
|
||||
expirationDate = expirationDate.AddDate(int(*bucketLockConfig.DefaultRetention.Years), 0, 0)
|
||||
}
|
||||
|
||||
if expirationDate.After(time.Now()) {
|
||||
switch bucketLockConfig.DefaultRetention.Mode {
|
||||
case types.ObjectLockRetentionModeGovernance:
|
||||
if !isAdminOrRoot {
|
||||
policy, err := be.GetBucketPolicy(ctx, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = verifyBucketPolicy(policy, userAccess, bucket, "", BypassGovernanceRetentionAction)
|
||||
if err != nil {
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
}
|
||||
case types.ObjectLockRetentionModeCompliance:
|
||||
return s3err.GetAPIError(s3err.ErrObjectLocked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
202
aws/LICENSE.txt
@@ -1,202 +0,0 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -1,4 +0,0 @@
|
||||
AWS SDK for Go
|
||||
Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
Copyright 2014-2015 Stripe, Inc.
|
||||
Copyright 2024 Versity Software
|
||||
@@ -1,11 +0,0 @@
|
||||
# AWS SDK Go v2
|
||||
|
||||
This directory contains code from the [AWS SDK Go v2](https://github.com/aws/aws-sdk-go-v2) repository, modified in accordance with the Apache 2.0 License.
|
||||
|
||||
## Description
|
||||
|
||||
The AWS SDK Go v2 is a collection of libraries and tools that enable developers to build applications that integrate with various AWS services. This directory and below contains modified code from the original repository, tailored to suit versitygw specific requirements.
|
||||
|
||||
## License
|
||||
|
||||
The code in this directory is licensed under the Apache 2.0 License. Please refer to the [LICENSE](./LICENSE) file for more information.
|
||||
@@ -1,61 +0,0 @@
|
||||
// Package unit performs initialization and validation for unit tests
|
||||
package unit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rsa"
|
||||
"math/big"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
)
|
||||
|
||||
func init() {
|
||||
config = aws.Config{}
|
||||
config.Region = "mock-region"
|
||||
config.Credentials = StubCredentialsProvider{}
|
||||
}
|
||||
|
||||
// StubCredentialsProvider provides a stub credential provider that returns
|
||||
// static credentials that never expire.
|
||||
type StubCredentialsProvider struct{}
|
||||
|
||||
// Retrieve satisfies the CredentialsProvider interface. Returns stub
|
||||
// credential value, and never error.
|
||||
func (StubCredentialsProvider) Retrieve(context.Context) (aws.Credentials, error) {
|
||||
return aws.Credentials{
|
||||
AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION",
|
||||
Source: "unit test credentials",
|
||||
}, nil
|
||||
}
|
||||
|
||||
var config aws.Config
|
||||
|
||||
// Config returns a copy of the mock configuration for unit tests.
|
||||
func Config() aws.Config { return config.Copy() }
|
||||
|
||||
// RSAPrivateKey is used for testing functionality that requires some
|
||||
// sort of private key. Taken from crypto/rsa/rsa_test.go
|
||||
//
|
||||
// Credit to golang 1.11
|
||||
var RSAPrivateKey = &rsa.PrivateKey{
|
||||
PublicKey: rsa.PublicKey{
|
||||
N: fromBase10("14314132931241006650998084889274020608918049032671858325988396851334124245188214251956198731333464217832226406088020736932173064754214329009979944037640912127943488972644697423190955557435910767690712778463524983667852819010259499695177313115447116110358524558307947613422897787329221478860907963827160223559690523660574329011927531289655711860504630573766609239332569210831325633840174683944553667352219670930408593321661375473885147973879086994006440025257225431977751512374815915392249179976902953721486040787792801849818254465486633791826766873076617116727073077821584676715609985777563958286637185868165868520557"),
|
||||
E: 3,
|
||||
},
|
||||
D: fromBase10("9542755287494004433998723259516013739278699355114572217325597900889416163458809501304132487555642811888150937392013824621448709836142886006653296025093941418628992648429798282127303704957273845127141852309016655778568546006839666463451542076964744073572349705538631742281931858219480985907271975884773482372966847639853897890615456605598071088189838676728836833012254065983259638538107719766738032720239892094196108713378822882383694456030043492571063441943847195939549773271694647657549658603365629458610273821292232646334717612674519997533901052790334279661754176490593041941863932308687197618671528035670452762731"),
|
||||
Primes: []*big.Int{
|
||||
fromBase10("130903255182996722426771613606077755295583329135067340152947172868415809027537376306193179624298874215608270802054347609836776473930072411958753044562214537013874103802006369634761074377213995983876788718033850153719421695468704276694983032644416930879093914927146648402139231293035971427838068945045019075433"),
|
||||
fromBase10("109348945610485453577574767652527472924289229538286649661240938988020367005475727988253438647560958573506159449538793540472829815903949343191091817779240101054552748665267574271163617694640513549693841337820602726596756351006149518830932261246698766355347898158548465400674856021497190430791824869615170301029"),
|
||||
},
|
||||
}
|
||||
|
||||
// Taken from crypto/rsa/rsa_test.go
|
||||
//
|
||||
// Credit to golang 1.11
|
||||
func fromBase10(base10 string) *big.Int {
|
||||
i, ok := new(big.Int).SetString(base10, 10)
|
||||
if !ok {
|
||||
panic("bad number: " + base10)
|
||||
}
|
||||
return i
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
package v4
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
)
|
||||
|
||||
func lookupKey(service, region string) string {
|
||||
var s strings.Builder
|
||||
s.Grow(len(region) + len(service) + 3)
|
||||
s.WriteString(region)
|
||||
s.WriteRune('/')
|
||||
s.WriteString(service)
|
||||
return s.String()
|
||||
}
|
||||
|
||||
type derivedKey struct {
|
||||
AccessKey string
|
||||
Date time.Time
|
||||
Credential []byte
|
||||
}
|
||||
|
||||
type derivedKeyCache struct {
|
||||
values map[string]derivedKey
|
||||
mutex sync.RWMutex
|
||||
}
|
||||
|
||||
func newDerivedKeyCache() derivedKeyCache {
|
||||
return derivedKeyCache{
|
||||
values: make(map[string]derivedKey),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *derivedKeyCache) Get(credentials aws.Credentials, service, region string, signingTime SigningTime) []byte {
|
||||
key := lookupKey(service, region)
|
||||
s.mutex.RLock()
|
||||
if cred, ok := s.get(key, credentials, signingTime.Time); ok {
|
||||
s.mutex.RUnlock()
|
||||
return cred
|
||||
}
|
||||
s.mutex.RUnlock()
|
||||
|
||||
s.mutex.Lock()
|
||||
if cred, ok := s.get(key, credentials, signingTime.Time); ok {
|
||||
s.mutex.Unlock()
|
||||
return cred
|
||||
}
|
||||
cred := deriveKey(credentials.SecretAccessKey, service, region, signingTime)
|
||||
entry := derivedKey{
|
||||
AccessKey: credentials.AccessKeyID,
|
||||
Date: signingTime.Time,
|
||||
Credential: cred,
|
||||
}
|
||||
s.values[key] = entry
|
||||
s.mutex.Unlock()
|
||||
|
||||
return cred
|
||||
}
|
||||
|
||||
func (s *derivedKeyCache) get(key string, credentials aws.Credentials, signingTime time.Time) ([]byte, bool) {
|
||||
cacheEntry, ok := s.retrieveFromCache(key)
|
||||
if ok && cacheEntry.AccessKey == credentials.AccessKeyID && isSameDay(signingTime, cacheEntry.Date) {
|
||||
return cacheEntry.Credential, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (s *derivedKeyCache) retrieveFromCache(key string) (derivedKey, bool) {
|
||||
if v, ok := s.values[key]; ok {
|
||||
return v, true
|
||||
}
|
||||
return derivedKey{}, false
|
||||
}
|
||||
|
||||
// SigningKeyDeriver derives a signing key from a set of credentials
|
||||
type SigningKeyDeriver struct {
|
||||
cache derivedKeyCache
|
||||
}
|
||||
|
||||
// NewSigningKeyDeriver returns a new SigningKeyDeriver
|
||||
func NewSigningKeyDeriver() *SigningKeyDeriver {
|
||||
return &SigningKeyDeriver{
|
||||
cache: newDerivedKeyCache(),
|
||||
}
|
||||
}
|
||||
|
||||
// DeriveKey returns a derived signing key from the given credentials to be used with SigV4 signing.
|
||||
func (k *SigningKeyDeriver) DeriveKey(credential aws.Credentials, service, region string, signingTime SigningTime) []byte {
|
||||
return k.cache.Get(credential, service, region, signingTime)
|
||||
}
|
||||
|
||||
func deriveKey(secret, service, region string, t SigningTime) []byte {
|
||||
hmacDate := HMACSHA256([]byte("AWS4"+secret), []byte(t.ShortTimeFormat()))
|
||||
hmacRegion := HMACSHA256(hmacDate, []byte(region))
|
||||
hmacService := HMACSHA256(hmacRegion, []byte(service))
|
||||
return HMACSHA256(hmacService, []byte("aws4_request"))
|
||||
}
|
||||
|
||||
func isSameDay(x, y time.Time) bool {
|
||||
xYear, xMonth, xDay := x.Date()
|
||||
yYear, yMonth, yDay := y.Date()
|
||||
|
||||
if xYear != yYear {
|
||||
return false
|
||||
}
|
||||
|
||||
if xMonth != yMonth {
|
||||
return false
|
||||
}
|
||||
|
||||
return xDay == yDay
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
package v4
|
||||
|
||||
// Signature Version 4 (SigV4) Constants
|
||||
const (
|
||||
// EmptyStringSHA256 is the hex encoded sha256 value of an empty string
|
||||
EmptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`
|
||||
|
||||
// UnsignedPayload indicates that the request payload body is unsigned
|
||||
UnsignedPayload = "UNSIGNED-PAYLOAD"
|
||||
|
||||
// AmzAlgorithmKey indicates the signing algorithm
|
||||
AmzAlgorithmKey = "X-Amz-Algorithm"
|
||||
|
||||
// AmzSecurityTokenKey indicates the security token to be used with temporary credentials
|
||||
AmzSecurityTokenKey = "X-Amz-Security-Token"
|
||||
|
||||
// AmzDateKey is the UTC timestamp for the request in the format YYYYMMDD'T'HHMMSS'Z'
|
||||
AmzDateKey = "X-Amz-Date"
|
||||
|
||||
// AmzCredentialKey is the access key ID and credential scope
|
||||
AmzCredentialKey = "X-Amz-Credential"
|
||||
|
||||
// AmzSignedHeadersKey is the set of headers signed for the request
|
||||
AmzSignedHeadersKey = "X-Amz-SignedHeaders"
|
||||
|
||||
// AmzSignatureKey is the query parameter to store the SigV4 signature
|
||||
AmzSignatureKey = "X-Amz-Signature"
|
||||
|
||||
// TimeFormat is the time format to be used in the X-Amz-Date header or query parameter
|
||||
TimeFormat = "20060102T150405Z"
|
||||
|
||||
// ShortTimeFormat is the shorten time format used in the credential scope
|
||||
ShortTimeFormat = "20060102"
|
||||
|
||||
// ContentSHAKey is the SHA256 of request body
|
||||
ContentSHAKey = "X-Amz-Content-Sha256"
|
||||
|
||||
// StreamingEventsPayload indicates that the request payload body is a signed event stream.
|
||||
StreamingEventsPayload = "STREAMING-AWS4-HMAC-SHA256-EVENTS"
|
||||
)
|
||||
@@ -1,88 +0,0 @@
|
||||
package v4
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Rules houses a set of Rule needed for validation of a
|
||||
// string value
|
||||
type Rules []Rule
|
||||
|
||||
// Rule interface allows for more flexible rules and just simply
|
||||
// checks whether or not a value adheres to that Rule
|
||||
type Rule interface {
|
||||
IsValid(value string) bool
|
||||
}
|
||||
|
||||
// IsValid will iterate through all rules and see if any rules
|
||||
// apply to the value and supports nested rules
|
||||
func (r Rules) IsValid(value string) bool {
|
||||
for _, rule := range r {
|
||||
if rule.IsValid(value) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// MapRule generic Rule for maps
|
||||
type MapRule map[string]struct{}
|
||||
|
||||
// IsValid for the map Rule satisfies whether it exists in the map
|
||||
func (m MapRule) IsValid(value string) bool {
|
||||
_, ok := m[value]
|
||||
return ok
|
||||
}
|
||||
|
||||
// AllowList is a generic Rule for include listing
|
||||
type AllowList struct {
|
||||
Rule
|
||||
}
|
||||
|
||||
// IsValid for AllowList checks if the value is within the AllowList
|
||||
func (w AllowList) IsValid(value string) bool {
|
||||
return w.Rule.IsValid(value)
|
||||
}
|
||||
|
||||
// ExcludeList is a generic Rule for exclude listing
|
||||
type ExcludeList struct {
|
||||
Rule
|
||||
}
|
||||
|
||||
// IsValid for AllowList checks if the value is within the AllowList
|
||||
func (b ExcludeList) IsValid(value string) bool {
|
||||
return !b.Rule.IsValid(value)
|
||||
}
|
||||
|
||||
// Patterns is a list of strings to match against
|
||||
type Patterns []string
|
||||
|
||||
// IsValid for Patterns checks each pattern and returns if a match has
|
||||
// been found
|
||||
func (p Patterns) IsValid(value string) bool {
|
||||
for _, pattern := range p {
|
||||
if hasPrefixFold(value, pattern) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// InclusiveRules rules allow for rules to depend on one another
|
||||
type InclusiveRules []Rule
|
||||
|
||||
// IsValid will return true if all rules are true
|
||||
func (r InclusiveRules) IsValid(value string) bool {
|
||||
for _, rule := range r {
|
||||
if !rule.IsValid(value) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// hasPrefixFold tests whether the string s begins with prefix, interpreted as UTF-8 strings,
|
||||
// under Unicode case-folding.
|
||||
func hasPrefixFold(s, prefix string) bool {
|
||||
return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix)
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package v4
|
||||
|
||||
// IgnoredHeaders is a list of headers that are ignored during signing
|
||||
var IgnoredHeaders = Rules{
|
||||
ExcludeList{
|
||||
MapRule{
|
||||
"Authorization": struct{}{},
|
||||
// some clients use user-agent in signed headers
|
||||
// "User-Agent": struct{}{},
|
||||
"X-Amzn-Trace-Id": struct{}{},
|
||||
"Expect": struct{}{},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// RequiredSignedHeaders is a allow list for Build canonical headers.
|
||||
var RequiredSignedHeaders = Rules{
|
||||
AllowList{
|
||||
MapRule{
|
||||
"Cache-Control": struct{}{},
|
||||
"Content-Disposition": struct{}{},
|
||||
"Content-Encoding": struct{}{},
|
||||
"Content-Language": struct{}{},
|
||||
"Content-Md5": struct{}{},
|
||||
"Content-Type": struct{}{},
|
||||
"Expires": struct{}{},
|
||||
"If-Match": struct{}{},
|
||||
"If-Modified-Since": struct{}{},
|
||||
"If-None-Match": struct{}{},
|
||||
"If-Unmodified-Since": struct{}{},
|
||||
"Range": struct{}{},
|
||||
"X-Amz-Acl": struct{}{},
|
||||
"X-Amz-Copy-Source": struct{}{},
|
||||
"X-Amz-Copy-Source-If-Match": struct{}{},
|
||||
"X-Amz-Copy-Source-If-Modified-Since": struct{}{},
|
||||
"X-Amz-Copy-Source-If-None-Match": struct{}{},
|
||||
"X-Amz-Copy-Source-If-Unmodified-Since": struct{}{},
|
||||
"X-Amz-Copy-Source-Range": struct{}{},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": struct{}{},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": struct{}{},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": struct{}{},
|
||||
"X-Amz-Expected-Bucket-Owner": struct{}{},
|
||||
"X-Amz-Grant-Full-control": struct{}{},
|
||||
"X-Amz-Grant-Read": struct{}{},
|
||||
"X-Amz-Grant-Read-Acp": struct{}{},
|
||||
"X-Amz-Grant-Write": struct{}{},
|
||||
"X-Amz-Grant-Write-Acp": struct{}{},
|
||||
"X-Amz-Metadata-Directive": struct{}{},
|
||||
"X-Amz-Mfa": struct{}{},
|
||||
"X-Amz-Request-Payer": struct{}{},
|
||||
"X-Amz-Server-Side-Encryption": struct{}{},
|
||||
"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": struct{}{},
|
||||
"X-Amz-Server-Side-Encryption-Context": struct{}{},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": struct{}{},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": struct{}{},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": struct{}{},
|
||||
"X-Amz-Storage-Class": struct{}{},
|
||||
"X-Amz-Website-Redirect-Location": struct{}{},
|
||||
"X-Amz-Content-Sha256": struct{}{},
|
||||
"X-Amz-Tagging": struct{}{},
|
||||
},
|
||||
},
|
||||
Patterns{"X-Amz-Object-Lock-"},
|
||||
Patterns{"X-Amz-Meta-"},
|
||||
}
|
||||
|
||||
// AllowedQueryHoisting is a allowed list for Build query headers. The boolean value
|
||||
// represents whether or not it is a pattern.
|
||||
var AllowedQueryHoisting = InclusiveRules{
|
||||
ExcludeList{RequiredSignedHeaders},
|
||||
Patterns{"X-Amz-"},
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
package v4
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestAllowedQueryHoisting(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
Header string
|
||||
ExpectHoist bool
|
||||
}{
|
||||
"object-lock": {
|
||||
Header: "X-Amz-Object-Lock-Mode",
|
||||
ExpectHoist: false,
|
||||
},
|
||||
"s3 metadata": {
|
||||
Header: "X-Amz-Meta-SomeName",
|
||||
ExpectHoist: false,
|
||||
},
|
||||
"another header": {
|
||||
Header: "X-Amz-SomeOtherHeader",
|
||||
ExpectHoist: true,
|
||||
},
|
||||
"non X-AMZ header": {
|
||||
Header: "X-SomeOtherHeader",
|
||||
ExpectHoist: false,
|
||||
},
|
||||
}
|
||||
|
||||
for name, c := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if e, a := c.ExpectHoist, AllowedQueryHoisting.IsValid(c.Header); e != a {
|
||||
t.Errorf("expect hoist %v, was %v", e, a)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIgnoredHeaders(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
Header string
|
||||
ExpectIgnored bool
|
||||
}{
|
||||
"expect": {
|
||||
Header: "Expect",
|
||||
ExpectIgnored: true,
|
||||
},
|
||||
"authorization": {
|
||||
Header: "Authorization",
|
||||
ExpectIgnored: true,
|
||||
},
|
||||
"X-AMZ header": {
|
||||
Header: "X-Amz-Content-Sha256",
|
||||
ExpectIgnored: false,
|
||||
},
|
||||
}
|
||||
|
||||
for name, c := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if e, a := c.ExpectIgnored, IgnoredHeaders.IsValid(c.Header); e == a {
|
||||
t.Errorf("expect ignored %v, was %v", e, a)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package v4
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
)
|
||||
|
||||
// HMACSHA256 computes a HMAC-SHA256 of data given the provided key.
|
||||
func HMACSHA256(key []byte, data []byte) []byte {
|
||||
hash := hmac.New(sha256.New, key)
|
||||
hash.Write(data)
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package v4
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// SanitizeHostForHeader removes default port from host and updates request.Host
|
||||
func SanitizeHostForHeader(r *http.Request) {
|
||||
host := getHost(r)
|
||||
port := portOnly(host)
|
||||
if port != "" && isDefaultPort(r.URL.Scheme, port) {
|
||||
r.Host = stripPort(host)
|
||||
}
|
||||
}
|
||||
|
||||
// Returns host from request
|
||||
func getHost(r *http.Request) string {
|
||||
if r.Host != "" {
|
||||
return r.Host
|
||||
}
|
||||
|
||||
return r.URL.Host
|
||||
}
|
||||
|
||||
// Hostname returns u.Host, without any port number.
|
||||
//
|
||||
// If Host is an IPv6 literal with a port number, Hostname returns the
|
||||
// IPv6 literal without the square brackets. IPv6 literals may include
|
||||
// a zone identifier.
|
||||
//
|
||||
// Copied from the Go 1.8 standard library (net/url)
|
||||
func stripPort(hostport string) string {
|
||||
colon := strings.IndexByte(hostport, ':')
|
||||
if colon == -1 {
|
||||
return hostport
|
||||
}
|
||||
if i := strings.IndexByte(hostport, ']'); i != -1 {
|
||||
return strings.TrimPrefix(hostport[:i], "[")
|
||||
}
|
||||
return hostport[:colon]
|
||||
}
|
||||
|
||||
// Port returns the port part of u.Host, without the leading colon.
|
||||
// If u.Host doesn't contain a port, Port returns an empty string.
|
||||
//
|
||||
// Copied from the Go 1.8 standard library (net/url)
|
||||
func portOnly(hostport string) string {
|
||||
colon := strings.IndexByte(hostport, ':')
|
||||
if colon == -1 {
|
||||
return ""
|
||||
}
|
||||
if i := strings.Index(hostport, "]:"); i != -1 {
|
||||
return hostport[i+len("]:"):]
|
||||
}
|
||||
if strings.Contains(hostport, "]") {
|
||||
return ""
|
||||
}
|
||||
return hostport[colon+len(":"):]
|
||||
}
|
||||
|
||||
// Returns true if the specified URI is using the standard port
|
||||
// (i.e. port 80 for HTTP URIs or 443 for HTTPS URIs)
|
||||
func isDefaultPort(scheme, port string) bool {
|
||||
if port == "" {
|
||||
return true
|
||||
}
|
||||
|
||||
lowerCaseScheme := strings.ToLower(scheme)
|
||||
if (lowerCaseScheme == "http" && port == "80") || (lowerCaseScheme == "https" && port == "443") {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package v4
|
||||
|
||||
import "strings"
|
||||
|
||||
// BuildCredentialScope builds the Signature Version 4 (SigV4) signing scope
|
||||
func BuildCredentialScope(signingTime SigningTime, region, service string) string {
|
||||
return strings.Join([]string{
|
||||
signingTime.ShortTimeFormat(),
|
||||
region,
|
||||
service,
|
||||
"aws4_request",
|
||||
}, "/")
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
package v4
|
||||
|
||||
import "time"
|
||||
|
||||
// SigningTime provides a wrapper around a time.Time which provides cached values for SigV4 signing.
|
||||
type SigningTime struct {
|
||||
time.Time
|
||||
timeFormat string
|
||||
shortTimeFormat string
|
||||
}
|
||||
|
||||
// NewSigningTime creates a new SigningTime given a time.Time
|
||||
func NewSigningTime(t time.Time) SigningTime {
|
||||
return SigningTime{
|
||||
Time: t,
|
||||
}
|
||||
}
|
||||
|
||||
// TimeFormat provides a time formatted in the X-Amz-Date format.
|
||||
func (m *SigningTime) TimeFormat() string {
|
||||
return m.format(&m.timeFormat, TimeFormat)
|
||||
}
|
||||
|
||||
// ShortTimeFormat provides a time formatted of 20060102.
|
||||
func (m *SigningTime) ShortTimeFormat() string {
|
||||
return m.format(&m.shortTimeFormat, ShortTimeFormat)
|
||||
}
|
||||
|
||||
func (m *SigningTime) format(target *string, format string) string {
|
||||
if len(*target) > 0 {
|
||||
return *target
|
||||
}
|
||||
v := m.Time.Format(format)
|
||||
*target = v
|
||||
return v
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
package v4
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const doubleSpace = " "
|
||||
|
||||
// StripExcessSpaces will rewrite the passed in slice's string values to not
|
||||
// contain multiple side-by-side spaces.
|
||||
func StripExcessSpaces(str string) string {
|
||||
var j, k, l, m, spaces int
|
||||
// Trim trailing spaces
|
||||
for j = len(str) - 1; j >= 0 && str[j] == ' '; j-- {
|
||||
}
|
||||
|
||||
// Trim leading spaces
|
||||
for k = 0; k < j && str[k] == ' '; k++ {
|
||||
}
|
||||
str = str[k : j+1]
|
||||
|
||||
// Strip multiple spaces.
|
||||
j = strings.Index(str, doubleSpace)
|
||||
if j < 0 {
|
||||
return str
|
||||
}
|
||||
|
||||
buf := []byte(str)
|
||||
for k, m, l = j, j, len(buf); k < l; k++ {
|
||||
if buf[k] == ' ' {
|
||||
if spaces == 0 {
|
||||
// First space.
|
||||
buf[m] = buf[k]
|
||||
m++
|
||||
}
|
||||
spaces++
|
||||
} else {
|
||||
// End of multiple spaces.
|
||||
spaces = 0
|
||||
buf[m] = buf[k]
|
||||
m++
|
||||
}
|
||||
}
|
||||
|
||||
return string(buf[:m])
|
||||
}
|
||||
|
||||
// GetURIPath returns the escaped URI component from the provided URL.
|
||||
func GetURIPath(u *url.URL) string {
|
||||
var uriPath string
|
||||
|
||||
if len(u.Opaque) > 0 {
|
||||
const schemeSep, pathSep, queryStart = "//", "/", "?"
|
||||
|
||||
opaque := u.Opaque
|
||||
// Cut off the query string if present.
|
||||
if idx := strings.Index(opaque, queryStart); idx >= 0 {
|
||||
opaque = opaque[:idx]
|
||||
}
|
||||
|
||||
// Cutout the scheme separator if present.
|
||||
if strings.Index(opaque, schemeSep) == 0 {
|
||||
opaque = opaque[len(schemeSep):]
|
||||
}
|
||||
|
||||
// capture URI path starting with first path separator.
|
||||
if idx := strings.Index(opaque, pathSep); idx >= 0 {
|
||||
uriPath = opaque[idx:]
|
||||
}
|
||||
} else {
|
||||
uriPath = u.EscapedPath()
|
||||
}
|
||||
|
||||
if len(uriPath) == 0 {
|
||||
uriPath = "/"
|
||||
}
|
||||
|
||||
return uriPath
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
package v4
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func lazyURLParse(v string) func() (*url.URL, error) {
|
||||
return func() (*url.URL, error) {
|
||||
return url.Parse(v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetURIPath(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
getURL func() (*url.URL, error)
|
||||
expect string
|
||||
}{
|
||||
// Cases
|
||||
"with scheme": {
|
||||
getURL: lazyURLParse("https://localhost:9000"),
|
||||
expect: "/",
|
||||
},
|
||||
"no port, with scheme": {
|
||||
getURL: lazyURLParse("https://localhost"),
|
||||
expect: "/",
|
||||
},
|
||||
"without scheme": {
|
||||
getURL: lazyURLParse("localhost:9000"),
|
||||
expect: "/",
|
||||
},
|
||||
"without scheme, with path": {
|
||||
getURL: lazyURLParse("localhost:9000/abc123"),
|
||||
expect: "/abc123",
|
||||
},
|
||||
"without scheme, with separator": {
|
||||
getURL: lazyURLParse("//localhost:9000"),
|
||||
expect: "/",
|
||||
},
|
||||
"no port, without scheme, with separator": {
|
||||
getURL: lazyURLParse("//localhost"),
|
||||
expect: "/",
|
||||
},
|
||||
"without scheme, with separator, with path": {
|
||||
getURL: lazyURLParse("//localhost:9000/abc123"),
|
||||
expect: "/abc123",
|
||||
},
|
||||
"no port, without scheme, with separator, with path": {
|
||||
getURL: lazyURLParse("//localhost/abc123"),
|
||||
expect: "/abc123",
|
||||
},
|
||||
"opaque with query string": {
|
||||
getURL: lazyURLParse("localhost:9000/abc123?efg=456"),
|
||||
expect: "/abc123",
|
||||
},
|
||||
"failing test": {
|
||||
getURL: func() (*url.URL, error) {
|
||||
endpoint := "https://service.region.amazonaws.com"
|
||||
req, _ := http.NewRequest("POST", endpoint, nil)
|
||||
u := req.URL
|
||||
|
||||
u.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
|
||||
|
||||
query := u.Query()
|
||||
query.Set("some-query-key", "value")
|
||||
u.RawQuery = query.Encode()
|
||||
|
||||
return u, nil
|
||||
},
|
||||
expect: "/bucket/key-._~,!@#$%^&*()",
|
||||
},
|
||||
}
|
||||
|
||||
for name, c := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
u, err := c.getURL()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get URL, %v", err)
|
||||
}
|
||||
|
||||
actual := GetURIPath(u)
|
||||
if e, a := c.expect, actual; e != a {
|
||||
t.Errorf("expect %v path, got %v", e, a)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStripExcessHeaders(t *testing.T) {
|
||||
vals := []string{
|
||||
"",
|
||||
"123",
|
||||
"1 2 3",
|
||||
"1 2 3 ",
|
||||
" 1 2 3",
|
||||
"1 2 3",
|
||||
"1 23",
|
||||
"1 2 3",
|
||||
"1 2 ",
|
||||
" 1 2 ",
|
||||
"12 3",
|
||||
"12 3 1",
|
||||
"12 3 1",
|
||||
"12 3 1abc123",
|
||||
}
|
||||
|
||||
expected := []string{
|
||||
"",
|
||||
"123",
|
||||
"1 2 3",
|
||||
"1 2 3",
|
||||
"1 2 3",
|
||||
"1 2 3",
|
||||
"1 23",
|
||||
"1 2 3",
|
||||
"1 2",
|
||||
"1 2",
|
||||
"12 3",
|
||||
"12 3 1",
|
||||
"12 3 1",
|
||||
"12 3 1abc123",
|
||||
}
|
||||
|
||||
for i := 0; i < len(vals); i++ {
|
||||
r := StripExcessSpaces(vals[i])
|
||||
if e, a := expected[i], r; e != a {
|
||||
t.Errorf("%d, expect %v, got %v", i, e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var stripExcessSpaceCases = []string{
|
||||
`AWS4-HMAC-SHA256 Credential=AKIDFAKEIDFAKEID/20160628/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=1234567890abcdef1234567890abcdef1234567890abcdef`,
|
||||
`123 321 123 321`,
|
||||
` 123 321 123 321 `,
|
||||
` 123 321 123 321 `,
|
||||
"123",
|
||||
"1 2 3",
|
||||
" 1 2 3",
|
||||
"1 2 3",
|
||||
"1 23",
|
||||
"1 2 3",
|
||||
"1 2 ",
|
||||
" 1 2 ",
|
||||
"12 3",
|
||||
"12 3 1",
|
||||
"12 3 1",
|
||||
"12 3 1abc123",
|
||||
}
|
||||
|
||||
func BenchmarkStripExcessSpaces(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
for _, v := range stripExcessSpaceCases {
|
||||
StripExcessSpaces(v)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
package v4_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/versity/versitygw/aws/internal/awstesting/unit"
|
||||
v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
|
||||
)
|
||||
|
||||
var standaloneSignCases = []struct {
|
||||
OrigURI string
|
||||
OrigQuery string
|
||||
Region, Service, SubDomain string
|
||||
ExpSig string
|
||||
EscapedURI string
|
||||
}{
|
||||
{
|
||||
OrigURI: `/logs-*/_search`,
|
||||
OrigQuery: `pretty=true`,
|
||||
Region: "us-west-2", Service: "es", SubDomain: "hostname-clusterkey",
|
||||
EscapedURI: `/logs-%2A/_search`,
|
||||
ExpSig: `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-west-2/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=79d0760751907af16f64a537c1242416dacf51204a7dd5284492d15577973b91`,
|
||||
},
|
||||
}
|
||||
|
||||
func TestStandaloneSign_CustomURIEscape(t *testing.T) {
|
||||
var expectSig = `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=6601e883cc6d23871fd6c2a394c5677ea2b8c82b04a6446786d64cd74f520967`
|
||||
|
||||
creds, err := unit.Config().Credentials.Retrieve(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
signer := v4.NewSigner(func(signer *v4.SignerOptions) {
|
||||
signer.DisableURIPathEscaping = true
|
||||
})
|
||||
|
||||
host := "https://subdomain.us-east-1.es.amazonaws.com"
|
||||
req, err := http.NewRequest("GET", host, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
|
||||
req.URL.Path = `/log-*/_search`
|
||||
req.URL.Opaque = "//subdomain.us-east-1.es.amazonaws.com/log-%2A/_search"
|
||||
|
||||
err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, "es", "us-east-1", time.Unix(0, 0))
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
|
||||
actual := req.Header.Get("Authorization")
|
||||
if e, a := expectSig, actual; e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStandaloneSign(t *testing.T) {
|
||||
creds, err := unit.Config().Credentials.Retrieve(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
signer := v4.NewSigner()
|
||||
|
||||
for _, c := range standaloneSignCases {
|
||||
host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
|
||||
c.SubDomain, c.Region, c.Service)
|
||||
|
||||
req, err := http.NewRequest("GET", host, nil)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
// URL.EscapedPath() will be used by the signer to get the
|
||||
// escaped form of the request's URI path.
|
||||
req.URL.Path = c.OrigURI
|
||||
req.URL.RawQuery = c.OrigQuery
|
||||
|
||||
err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
actual := req.Header.Get("Authorization")
|
||||
if e, a := c.ExpSig, actual; e != a {
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
if e, a := c.OrigURI, req.URL.Path; e != a {
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStandaloneSign_RawPath(t *testing.T) {
|
||||
creds, err := unit.Config().Credentials.Retrieve(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
signer := v4.NewSigner()
|
||||
|
||||
for _, c := range standaloneSignCases {
|
||||
host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
|
||||
c.SubDomain, c.Region, c.Service)
|
||||
|
||||
req, err := http.NewRequest("GET", host, nil)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
// URL.EscapedPath() will be used by the signer to get the
|
||||
// escaped form of the request's URI path.
|
||||
req.URL.Path = c.OrigURI
|
||||
req.URL.RawPath = c.EscapedURI
|
||||
req.URL.RawQuery = c.OrigQuery
|
||||
|
||||
err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
actual := req.Header.Get("Authorization")
|
||||
if e, a := c.ExpSig, actual; e != a {
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
if e, a := c.OrigURI, req.URL.Path; e != a {
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
|
||||
t.Errorf("expected %v, but recieved %v", e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,565 +0,0 @@
|
||||
// Package v4 implements signing for AWS V4 signer
|
||||
//
|
||||
// Provides request signing for request that need to be signed with
|
||||
// AWS V4 Signatures.
|
||||
//
|
||||
// # Standalone Signer
|
||||
//
|
||||
// Generally using the signer outside of the SDK should not require any additional
|
||||
//
|
||||
// The signer does this by taking advantage of the URL.EscapedPath method. If your request URI requires
|
||||
//
|
||||
// additional escaping you many need to use the URL.Opaque to define what the raw URI should be sent
|
||||
// to the service as.
|
||||
//
|
||||
// The signer will first check the URL.Opaque field, and use its value if set.
|
||||
// The signer does require the URL.Opaque field to be set in the form of:
|
||||
//
|
||||
// "//<hostname>/<path>"
|
||||
//
|
||||
// // e.g.
|
||||
// "//example.com/some/path"
|
||||
//
|
||||
// The leading "//" and hostname are required or the URL.Opaque escaping will
|
||||
// not work correctly.
|
||||
//
|
||||
// If URL.Opaque is not set the signer will fallback to the URL.EscapedPath()
|
||||
// method and using the returned value.
|
||||
//
|
||||
// AWS v4 signature validation requires that the canonical string's URI path
|
||||
// element must be the URI escaped form of the HTTP request's path.
|
||||
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
|
||||
//
|
||||
// The Go HTTP client will perform escaping automatically on the request. Some
|
||||
// of these escaping may cause signature validation errors because the HTTP
|
||||
// request differs from the URI path or query that the signature was generated.
|
||||
// https://golang.org/pkg/net/url/#URL.EscapedPath
|
||||
//
|
||||
// Because of this, it is recommended that when using the signer outside of the
|
||||
// SDK that explicitly escaping the request prior to being signed is preferable,
|
||||
// and will help prevent signature validation errors. This can be done by setting
|
||||
// the URL.Opaque or URL.RawPath. The SDK will use URL.Opaque first and then
|
||||
// call URL.EscapedPath() if Opaque is not set.
|
||||
//
|
||||
// Test `TestStandaloneSign` provides a complete example of using the signer
|
||||
// outside of the SDK and pre-escaping the URI path.
|
||||
package v4
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"hash"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/smithy-go/encoding/httpbinding"
|
||||
"github.com/aws/smithy-go/logging"
|
||||
v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
|
||||
)
|
||||
|
||||
const (
|
||||
signingAlgorithm = "AWS4-HMAC-SHA256"
|
||||
authorizationHeader = "Authorization"
|
||||
|
||||
// Version of signing v4
|
||||
Version = "SigV4"
|
||||
)
|
||||
|
||||
// HTTPSigner is an interface to a SigV4 signer that can sign HTTP requests
|
||||
type HTTPSigner interface {
|
||||
SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error
|
||||
}
|
||||
|
||||
type keyDerivator interface {
|
||||
DeriveKey(credential aws.Credentials, service, region string, signingTime v4Internal.SigningTime) []byte
|
||||
}
|
||||
|
||||
// SignerOptions is the SigV4 Signer options.
|
||||
type SignerOptions struct {
|
||||
// Disables the Signer's moving HTTP header key/value pairs from the HTTP
|
||||
// request header to the request's query string. This is most commonly used
|
||||
// with pre-signed requests preventing headers from being added to the
|
||||
// request's query string.
|
||||
DisableHeaderHoisting bool
|
||||
|
||||
// Disables the automatic escaping of the URI path of the request for the
|
||||
// siganture's canonical string's path. For services that do not need additional
|
||||
// escaping then use this to disable the signer escaping the path.
|
||||
//
|
||||
// S3 is an example of a service that does not need additional escaping.
|
||||
//
|
||||
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
|
||||
DisableURIPathEscaping bool
|
||||
|
||||
// The logger to send log messages to.
|
||||
Logger logging.Logger
|
||||
|
||||
// Enable logging of signed requests.
|
||||
// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
|
||||
// presigned URL.
|
||||
LogSigning bool
|
||||
|
||||
// Disables setting the session token on the request as part of signing
|
||||
// through X-Amz-Security-Token. This is needed for variations of v4 that
|
||||
// present the token elsewhere.
|
||||
DisableSessionToken bool
|
||||
}
|
||||
|
||||
// Signer applies AWS v4 signing to given request. Use this to sign requests
|
||||
// that need to be signed with AWS V4 Signatures.
|
||||
type Signer struct {
|
||||
options SignerOptions
|
||||
keyDerivator keyDerivator
|
||||
}
|
||||
|
||||
// NewSigner returns a new SigV4 Signer
|
||||
func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
|
||||
options := SignerOptions{}
|
||||
|
||||
for _, fn := range optFns {
|
||||
fn(&options)
|
||||
}
|
||||
|
||||
return &Signer{options: options, keyDerivator: v4Internal.NewSigningKeyDeriver()}
|
||||
}
|
||||
|
||||
type httpSigner struct {
|
||||
Request *http.Request
|
||||
ServiceName string
|
||||
Region string
|
||||
Time v4Internal.SigningTime
|
||||
Credentials aws.Credentials
|
||||
KeyDerivator keyDerivator
|
||||
IsPreSign bool
|
||||
SignedHdrs []string
|
||||
|
||||
PayloadHash string
|
||||
|
||||
DisableHeaderHoisting bool
|
||||
DisableURIPathEscaping bool
|
||||
DisableSessionToken bool
|
||||
}
|
||||
|
||||
func (s *httpSigner) Build() (signedRequest, error) {
|
||||
req := s.Request
|
||||
|
||||
query := req.URL.Query()
|
||||
headers := req.Header
|
||||
|
||||
s.setRequiredSigningFields(headers, query)
|
||||
|
||||
// Sort Each Query Key's Values
|
||||
for key := range query {
|
||||
sort.Strings(query[key])
|
||||
}
|
||||
|
||||
v4Internal.SanitizeHostForHeader(req)
|
||||
|
||||
credentialScope := s.buildCredentialScope()
|
||||
credentialStr := s.Credentials.AccessKeyID + "/" + credentialScope
|
||||
if s.IsPreSign {
|
||||
query.Set(v4Internal.AmzCredentialKey, credentialStr)
|
||||
}
|
||||
|
||||
unsignedHeaders := headers
|
||||
if s.IsPreSign && !s.DisableHeaderHoisting {
|
||||
var urlValues url.Values
|
||||
urlValues, unsignedHeaders = buildQuery(v4Internal.AllowedQueryHoisting, headers)
|
||||
for k := range urlValues {
|
||||
query[k] = urlValues[k]
|
||||
}
|
||||
}
|
||||
|
||||
host := req.URL.Host
|
||||
if len(req.Host) > 0 {
|
||||
host = req.Host
|
||||
}
|
||||
|
||||
signedHeaders, signedHeadersStr, canonicalHeaderStr := s.buildCanonicalHeaders(host, v4Internal.IgnoredHeaders, unsignedHeaders, s.Request.ContentLength)
|
||||
|
||||
if s.IsPreSign {
|
||||
query.Set(v4Internal.AmzSignedHeadersKey, signedHeadersStr)
|
||||
}
|
||||
|
||||
var rawQuery strings.Builder
|
||||
rawQuery.WriteString(strings.Replace(query.Encode(), "+", "%20", -1))
|
||||
|
||||
canonicalURI := v4Internal.GetURIPath(req.URL)
|
||||
if !s.DisableURIPathEscaping {
|
||||
canonicalURI = httpbinding.EscapePath(canonicalURI, false)
|
||||
}
|
||||
|
||||
canonicalString := s.buildCanonicalString(
|
||||
req.Method,
|
||||
canonicalURI,
|
||||
rawQuery.String(),
|
||||
signedHeadersStr,
|
||||
canonicalHeaderStr,
|
||||
)
|
||||
|
||||
strToSign := s.buildStringToSign(credentialScope, canonicalString)
|
||||
signingSignature, err := s.buildSignature(strToSign)
|
||||
if err != nil {
|
||||
return signedRequest{}, err
|
||||
}
|
||||
|
||||
if s.IsPreSign {
|
||||
rawQuery.WriteString("&X-Amz-Signature=")
|
||||
rawQuery.WriteString(signingSignature)
|
||||
} else {
|
||||
headers[authorizationHeader] = append(headers[authorizationHeader][:0], buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature))
|
||||
}
|
||||
|
||||
req.URL.RawQuery = rawQuery.String()
|
||||
|
||||
return signedRequest{
|
||||
Request: req,
|
||||
SignedHeaders: signedHeaders,
|
||||
CanonicalString: canonicalString,
|
||||
StringToSign: strToSign,
|
||||
PreSigned: s.IsPreSign,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature string) string {
|
||||
const credential = "Credential="
|
||||
const signedHeaders = "SignedHeaders="
|
||||
const signature = "Signature="
|
||||
const commaSpace = ", "
|
||||
|
||||
var parts strings.Builder
|
||||
parts.Grow(len(signingAlgorithm) + 1 +
|
||||
len(credential) + len(credentialStr) + 2 +
|
||||
len(signedHeaders) + len(signedHeadersStr) + 2 +
|
||||
len(signature) + len(signingSignature),
|
||||
)
|
||||
parts.WriteString(signingAlgorithm)
|
||||
parts.WriteRune(' ')
|
||||
parts.WriteString(credential)
|
||||
parts.WriteString(credentialStr)
|
||||
parts.WriteString(commaSpace)
|
||||
parts.WriteString(signedHeaders)
|
||||
parts.WriteString(signedHeadersStr)
|
||||
parts.WriteString(commaSpace)
|
||||
parts.WriteString(signature)
|
||||
parts.WriteString(signingSignature)
|
||||
return parts.String()
|
||||
}
|
||||
|
||||
// SignHTTP signs AWS v4 requests with the provided payload hash, service name, region the
|
||||
// request is made to, and time the request is signed at. The signTime allows
|
||||
// you to specify that a request is signed for the future, and cannot be
|
||||
// used until then.
|
||||
//
|
||||
// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
|
||||
// must be provided. Even if the request has no payload (aka body). If the
|
||||
// request has no payload you should use the hex encoded SHA-256 of an empty
|
||||
// string as the payloadHash value.
|
||||
//
|
||||
// "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
//
|
||||
// Some services such as Amazon S3 accept alternative values for the payload
|
||||
// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
|
||||
// included in the request signature.
|
||||
//
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
|
||||
//
|
||||
// Sign differs from Presign in that it will sign the request using HTTP
|
||||
// header values. This type of signing is intended for http.Request values that
|
||||
// will not be shared, or are shared in a way the header values on the request
|
||||
// will not be lost.
|
||||
//
|
||||
// The passed in request will be modified in place.
|
||||
func (s Signer) SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, signedHdrs []string, optFns ...func(options *SignerOptions)) error {
|
||||
options := s.options
|
||||
|
||||
for _, fn := range optFns {
|
||||
fn(&options)
|
||||
}
|
||||
|
||||
signer := &httpSigner{
|
||||
Request: r,
|
||||
PayloadHash: payloadHash,
|
||||
ServiceName: service,
|
||||
Region: region,
|
||||
Credentials: credentials,
|
||||
Time: v4Internal.NewSigningTime(signingTime.UTC()),
|
||||
DisableHeaderHoisting: options.DisableHeaderHoisting,
|
||||
DisableURIPathEscaping: options.DisableURIPathEscaping,
|
||||
DisableSessionToken: options.DisableSessionToken,
|
||||
KeyDerivator: s.keyDerivator,
|
||||
SignedHdrs: signedHdrs,
|
||||
}
|
||||
|
||||
signedRequest, err := signer.Build()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logSigningInfo(ctx, options, &signedRequest, false)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PresignHTTP signs AWS v4 requests with the payload hash, service name, region
|
||||
// the request is made to, and time the request is signed at. The signTime
|
||||
// allows you to specify that a request is signed for the future, and cannot
|
||||
// be used until then.
|
||||
//
|
||||
// Returns the signed URL and the map of HTTP headers that were included in the
|
||||
// signature or an error if signing the request failed. For presigned requests
|
||||
// these headers and their values must be included on the HTTP request when it
|
||||
// is made. This is helpful to know what header values need to be shared with
|
||||
// the party the presigned request will be distributed to.
|
||||
//
|
||||
// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
|
||||
// must be provided. Even if the request has no payload (aka body). If the
|
||||
// request has no payload you should use the hex encoded SHA-256 of an empty
|
||||
// string as the payloadHash value.
|
||||
//
|
||||
// "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
//
|
||||
// Some services such as Amazon S3 accept alternative values for the payload
|
||||
// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
|
||||
// included in the request signature.
|
||||
//
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
|
||||
//
|
||||
// PresignHTTP differs from SignHTTP in that it will sign the request using
|
||||
// query string instead of header values. This allows you to share the
|
||||
// Presigned Request's URL with third parties, or distribute it throughout your
|
||||
// system with minimal dependencies.
|
||||
//
|
||||
// PresignHTTP will not set the expires time of the presigned request
|
||||
// automatically. To specify the expire duration for a request add the
|
||||
// "X-Amz-Expires" query parameter on the request with the value as the
|
||||
// duration in seconds the presigned URL should be considered valid for. This
|
||||
// parameter is not used by all AWS services, and is most notable used by
|
||||
// Amazon S3 APIs.
|
||||
//
|
||||
// expires := 20 * time.Minute
|
||||
// query := req.URL.Query()
|
||||
// query.Set("X-Amz-Expires", strconv.FormatInt(int64(expires/time.Second), 10))
|
||||
// req.URL.RawQuery = query.Encode()
|
||||
//
|
||||
// This method does not modify the provided request.
|
||||
func (s *Signer) PresignHTTP(
|
||||
ctx context.Context, credentials aws.Credentials, r *http.Request,
|
||||
payloadHash string, service string, region string, signingTime time.Time,
|
||||
signedHdrs []string,
|
||||
optFns ...func(*SignerOptions),
|
||||
) (signedURI string, signedHeaders http.Header, err error) {
|
||||
options := s.options
|
||||
|
||||
for _, fn := range optFns {
|
||||
fn(&options)
|
||||
}
|
||||
|
||||
signer := &httpSigner{
|
||||
Request: r.Clone(r.Context()),
|
||||
PayloadHash: payloadHash,
|
||||
ServiceName: service,
|
||||
Region: region,
|
||||
Credentials: credentials,
|
||||
Time: v4Internal.NewSigningTime(signingTime.UTC()),
|
||||
IsPreSign: true,
|
||||
DisableHeaderHoisting: options.DisableHeaderHoisting,
|
||||
DisableURIPathEscaping: options.DisableURIPathEscaping,
|
||||
DisableSessionToken: options.DisableSessionToken,
|
||||
KeyDerivator: s.keyDerivator,
|
||||
SignedHdrs: signedHdrs,
|
||||
}
|
||||
|
||||
signedRequest, err := signer.Build()
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
logSigningInfo(ctx, options, &signedRequest, true)
|
||||
|
||||
signedHeaders = make(http.Header)
|
||||
|
||||
// For the signed headers we canonicalize the header keys in the returned map.
|
||||
// This avoids situations where can standard library double headers like host header. For example the standard
|
||||
// library will set the Host header, even if it is present in lower-case form.
|
||||
for k, v := range signedRequest.SignedHeaders {
|
||||
key := textproto.CanonicalMIMEHeaderKey(k)
|
||||
signedHeaders[key] = append(signedHeaders[key], v...)
|
||||
}
|
||||
|
||||
return signedRequest.Request.URL.String(), signedHeaders, nil
|
||||
}
|
||||
|
||||
func (s *httpSigner) buildCredentialScope() string {
|
||||
return v4Internal.BuildCredentialScope(s.Time, s.Region, s.ServiceName)
|
||||
}
|
||||
|
||||
func buildQuery(r v4Internal.Rule, header http.Header) (url.Values, http.Header) {
|
||||
query := url.Values{}
|
||||
unsignedHeaders := http.Header{}
|
||||
for k, h := range header {
|
||||
if r.IsValid(k) {
|
||||
query[k] = h
|
||||
} else {
|
||||
unsignedHeaders[k] = h
|
||||
}
|
||||
}
|
||||
|
||||
return query, unsignedHeaders
|
||||
}
|
||||
|
||||
func (s *httpSigner) buildCanonicalHeaders(host string, rule v4Internal.Rule, header http.Header, length int64) (signed http.Header, signedHeaders, canonicalHeadersStr string) {
|
||||
signed = make(http.Header)
|
||||
|
||||
var headers []string
|
||||
const hostHeader = "host"
|
||||
headers = append(headers, hostHeader)
|
||||
signed[hostHeader] = append(signed[hostHeader], host)
|
||||
|
||||
const contentLengthHeader = "content-length"
|
||||
if slices.Contains(s.SignedHdrs, contentLengthHeader) {
|
||||
headers = append(headers, contentLengthHeader)
|
||||
signed[contentLengthHeader] = append(signed[contentLengthHeader], strconv.FormatInt(length, 10))
|
||||
}
|
||||
|
||||
for k, v := range header {
|
||||
if !rule.IsValid(k) {
|
||||
continue // ignored header
|
||||
}
|
||||
if strings.EqualFold(k, contentLengthHeader) {
|
||||
// prevent signing already handled content-length header.
|
||||
continue
|
||||
}
|
||||
|
||||
lowerCaseKey := strings.ToLower(k)
|
||||
if _, ok := signed[lowerCaseKey]; ok {
|
||||
// include additional values
|
||||
signed[lowerCaseKey] = append(signed[lowerCaseKey], v...)
|
||||
continue
|
||||
}
|
||||
|
||||
headers = append(headers, lowerCaseKey)
|
||||
signed[lowerCaseKey] = v
|
||||
}
|
||||
sort.Strings(headers)
|
||||
|
||||
signedHeaders = strings.Join(headers, ";")
|
||||
|
||||
var canonicalHeaders strings.Builder
|
||||
n := len(headers)
|
||||
const colon = ':'
|
||||
for i := 0; i < n; i++ {
|
||||
if headers[i] == hostHeader {
|
||||
canonicalHeaders.WriteString(hostHeader)
|
||||
canonicalHeaders.WriteRune(colon)
|
||||
canonicalHeaders.WriteString(v4Internal.StripExcessSpaces(host))
|
||||
} else {
|
||||
canonicalHeaders.WriteString(headers[i])
|
||||
canonicalHeaders.WriteRune(colon)
|
||||
// Trim out leading, trailing, and dedup inner spaces from signed header values.
|
||||
values := signed[headers[i]]
|
||||
for j, v := range values {
|
||||
cleanedValue := strings.TrimSpace(v4Internal.StripExcessSpaces(v))
|
||||
canonicalHeaders.WriteString(cleanedValue)
|
||||
if j < len(values)-1 {
|
||||
canonicalHeaders.WriteRune(',')
|
||||
}
|
||||
}
|
||||
}
|
||||
canonicalHeaders.WriteRune('\n')
|
||||
}
|
||||
canonicalHeadersStr = canonicalHeaders.String()
|
||||
|
||||
return signed, signedHeaders, canonicalHeadersStr
|
||||
}
|
||||
|
||||
func (s *httpSigner) buildCanonicalString(method, uri, query, signedHeaders, canonicalHeaders string) string {
|
||||
return strings.Join([]string{
|
||||
method,
|
||||
uri,
|
||||
query,
|
||||
canonicalHeaders,
|
||||
signedHeaders,
|
||||
s.PayloadHash,
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
func (s *httpSigner) buildStringToSign(credentialScope, canonicalRequestString string) string {
|
||||
return strings.Join([]string{
|
||||
signingAlgorithm,
|
||||
s.Time.TimeFormat(),
|
||||
credentialScope,
|
||||
hex.EncodeToString(makeHash(sha256.New(), []byte(canonicalRequestString))),
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
func makeHash(hash hash.Hash, b []byte) []byte {
|
||||
hash.Reset()
|
||||
hash.Write(b)
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
|
||||
func (s *httpSigner) buildSignature(strToSign string) (string, error) {
|
||||
key := s.KeyDerivator.DeriveKey(s.Credentials, s.ServiceName, s.Region, s.Time)
|
||||
return hex.EncodeToString(v4Internal.HMACSHA256(key, []byte(strToSign))), nil
|
||||
}
|
||||
|
||||
func (s *httpSigner) setRequiredSigningFields(headers http.Header, query url.Values) {
|
||||
amzDate := s.Time.TimeFormat()
|
||||
|
||||
if s.IsPreSign {
|
||||
query.Set(v4Internal.AmzAlgorithmKey, signingAlgorithm)
|
||||
sessionToken := s.Credentials.SessionToken
|
||||
if !s.DisableSessionToken && len(sessionToken) > 0 {
|
||||
query.Set("X-Amz-Security-Token", sessionToken)
|
||||
}
|
||||
|
||||
query.Set(v4Internal.AmzDateKey, amzDate)
|
||||
return
|
||||
}
|
||||
|
||||
headers[v4Internal.AmzDateKey] = append(headers[v4Internal.AmzDateKey][:0], amzDate)
|
||||
|
||||
if !s.DisableSessionToken && len(s.Credentials.SessionToken) > 0 {
|
||||
headers[v4Internal.AmzSecurityTokenKey] = append(headers[v4Internal.AmzSecurityTokenKey][:0], s.Credentials.SessionToken)
|
||||
}
|
||||
}
|
||||
|
||||
func logSigningInfo(ctx context.Context, options SignerOptions, request *signedRequest, isPresign bool) {
|
||||
if !options.LogSigning {
|
||||
return
|
||||
}
|
||||
signedURLMsg := ""
|
||||
if isPresign {
|
||||
signedURLMsg = fmt.Sprintf(logSignedURLMsg, request.Request.URL.String())
|
||||
}
|
||||
logger := logging.WithContext(ctx, options.Logger)
|
||||
logger.Logf(logging.Debug, logSignInfoMsg, request.CanonicalString, request.StringToSign, signedURLMsg)
|
||||
}
|
||||
|
||||
type signedRequest struct {
|
||||
Request *http.Request
|
||||
SignedHeaders http.Header
|
||||
CanonicalString string
|
||||
StringToSign string
|
||||
PreSigned bool
|
||||
}
|
||||
|
||||
const logSignInfoMsg = `Request Signature:
|
||||
---[ CANONICAL STRING ]-----------------------------
|
||||
%s
|
||||
---[ STRING TO SIGN ]--------------------------------
|
||||
%s%s
|
||||
-----------------------------------------------------`
|
||||
const logSignedURLMsg = `
|
||||
---[ SIGNED URL ]------------------------------------
|
||||
%s`
|
||||
@@ -1,358 +0,0 @@
|
||||
package v4
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
|
||||
)
|
||||
|
||||
var testCredentials = aws.Credentials{AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION"}
|
||||
|
||||
func buildRequest(serviceName, region, body string) (*http.Request, string) {
|
||||
reader := strings.NewReader(body)
|
||||
return buildRequestWithBodyReader(serviceName, region, reader)
|
||||
}
|
||||
|
||||
func buildRequestWithBodyReader(serviceName, region string, body io.Reader) (*http.Request, string) {
|
||||
var bodyLen int
|
||||
|
||||
type lenner interface {
|
||||
Len() int
|
||||
}
|
||||
if lr, ok := body.(lenner); ok {
|
||||
bodyLen = lr.Len()
|
||||
}
|
||||
|
||||
endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
|
||||
req, _ := http.NewRequest("POST", endpoint, body)
|
||||
req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
|
||||
req.Header.Set("X-Amz-Target", "prefix.Operation")
|
||||
req.Header.Set("Content-Type", "application/x-amz-json-1.0")
|
||||
|
||||
if bodyLen > 0 {
|
||||
req.ContentLength = int64(bodyLen)
|
||||
}
|
||||
|
||||
req.Header.Set("X-Amz-Meta-Other-Header", "some-value=!@#$%^&* (+)")
|
||||
req.Header.Add("X-Amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
|
||||
req.Header.Add("X-amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
|
||||
|
||||
h := sha256.New()
|
||||
_, _ = io.Copy(h, body)
|
||||
payloadHash := hex.EncodeToString(h.Sum(nil))
|
||||
|
||||
return req, payloadHash
|
||||
}
|
||||
|
||||
func TestPresignRequest(t *testing.T) {
|
||||
req, body := buildRequest("dynamodb", "us-east-1", "{}")
|
||||
|
||||
query := req.URL.Query()
|
||||
query.Set("X-Amz-Expires", "300")
|
||||
req.URL.RawQuery = query.Encode()
|
||||
|
||||
signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
|
||||
signer := NewSigner()
|
||||
signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
expectedDate := "19700101T000000Z"
|
||||
expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
|
||||
expectedSig := "122f0b9e091e4ba84286097e2b3404a1f1f4c4aad479adda95b7dff0ccbe5581"
|
||||
expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
|
||||
expectedTarget := "prefix.Operation"
|
||||
|
||||
q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
|
||||
if err != nil {
|
||||
t.Errorf("expect no error, got %v", err)
|
||||
}
|
||||
|
||||
if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
|
||||
t.Errorf("expect %v to be empty", a)
|
||||
}
|
||||
if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
|
||||
for _, h := range strings.Split(expectedHeaders, ";") {
|
||||
v := headers.Get(h)
|
||||
if len(v) == 0 {
|
||||
t.Errorf("expect %v, to be present in header map", h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPresignBodyWithArrayRequest(t *testing.T) {
|
||||
req, body := buildRequest("dynamodb", "us-east-1", "{}")
|
||||
req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
|
||||
|
||||
query := req.URL.Query()
|
||||
query.Set("X-Amz-Expires", "300")
|
||||
req.URL.RawQuery = query.Encode()
|
||||
|
||||
signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
|
||||
signer := NewSigner()
|
||||
signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
|
||||
q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
|
||||
if err != nil {
|
||||
t.Errorf("expect no error, got %v", err)
|
||||
}
|
||||
|
||||
expectedDate := "19700101T000000Z"
|
||||
expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
|
||||
expectedSig := "e3ac55addee8711b76c6d608d762cff285fe8b627a057f8b5ec9268cf82c08b1"
|
||||
expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
|
||||
expectedTarget := "prefix.Operation"
|
||||
|
||||
if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
|
||||
t.Errorf("expect %v to be empty, was not", a)
|
||||
}
|
||||
if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
|
||||
for _, h := range strings.Split(expectedHeaders, ";") {
|
||||
v := headers.Get(h)
|
||||
if len(v) == 0 {
|
||||
t.Errorf("expect %v, to be present in header map", h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignRequest(t *testing.T) {
|
||||
req, body := buildRequest("dynamodb", "us-east-1", "{}")
|
||||
signer := NewSigner()
|
||||
signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
|
||||
err := signer.SignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
|
||||
expectedDate := "19700101T000000Z"
|
||||
expectedSig := "AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/dynamodb/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore;x-amz-security-token;x-amz-target, Signature=a518299330494908a70222cec6899f6f32f297f8595f6df1776d998936652ad9"
|
||||
|
||||
q := req.Header
|
||||
if e, a := expectedSig, q.Get("Authorization"); e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildCanonicalRequest(t *testing.T) {
|
||||
req, _ := buildRequest("dynamodb", "us-east-1", "{}")
|
||||
req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
|
||||
|
||||
ctx := &httpSigner{
|
||||
ServiceName: "dynamodb",
|
||||
Region: "us-east-1",
|
||||
Request: req,
|
||||
Time: v4Internal.NewSigningTime(time.Now()),
|
||||
KeyDerivator: v4Internal.NewSigningKeyDeriver(),
|
||||
}
|
||||
|
||||
build, err := ctx.Build()
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
expected := "https://example.org/bucket/key-._~,!@#$%^&*()?Foo=a&Foo=m&Foo=o&Foo=z"
|
||||
if e, a := expected, build.Request.URL.String(); e != a {
|
||||
t.Errorf("expect %v, got %v", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSigner_SignHTTP_NoReplaceRequestBody(t *testing.T) {
|
||||
req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
|
||||
req.Body = io.NopCloser(bytes.NewReader([]byte{}))
|
||||
|
||||
s := NewSigner()
|
||||
|
||||
origBody := req.Body
|
||||
|
||||
err := s.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
|
||||
if req.Body != origBody {
|
||||
t.Errorf("expect request body to not be chagned")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestHost(t *testing.T) {
|
||||
req, _ := buildRequest("dynamodb", "us-east-1", "{}")
|
||||
req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
|
||||
req.Host = "myhost"
|
||||
|
||||
query := req.URL.Query()
|
||||
query.Set("X-Amz-Expires", "5")
|
||||
req.URL.RawQuery = query.Encode()
|
||||
|
||||
ctx := &httpSigner{
|
||||
ServiceName: "dynamodb",
|
||||
Region: "us-east-1",
|
||||
Request: req,
|
||||
Time: v4Internal.NewSigningTime(time.Now()),
|
||||
KeyDerivator: v4Internal.NewSigningKeyDeriver(),
|
||||
}
|
||||
|
||||
build, err := ctx.Build()
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(build.CanonicalString, "host:"+req.Host) {
|
||||
t.Errorf("canonical host header invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSign_buildCanonicalHeadersContentLengthPresent(t *testing.T) {
|
||||
body := `{"description": "this is a test"}`
|
||||
req, _ := buildRequest("dynamodb", "us-east-1", body)
|
||||
req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
|
||||
req.Host = "myhost"
|
||||
|
||||
contentLength := fmt.Sprintf("%d", len([]byte(body)))
|
||||
req.Header.Add("Content-Length", contentLength)
|
||||
|
||||
query := req.URL.Query()
|
||||
query.Set("X-Amz-Expires", "5")
|
||||
req.URL.RawQuery = query.Encode()
|
||||
|
||||
ctx := &httpSigner{
|
||||
ServiceName: "dynamodb",
|
||||
Region: "us-east-1",
|
||||
Request: req,
|
||||
Time: v4Internal.NewSigningTime(time.Now()),
|
||||
KeyDerivator: v4Internal.NewSigningKeyDeriver(),
|
||||
}
|
||||
|
||||
_, err := ctx.Build()
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
//if !strings.Contains(build.CanonicalString, "content-length:"+contentLength+"\n") {
|
||||
// t.Errorf("canonical header content-length invalid")
|
||||
//}
|
||||
}
|
||||
|
||||
func TestSign_buildCanonicalHeaders(t *testing.T) {
|
||||
serviceName := "mockAPI"
|
||||
region := "mock-region"
|
||||
endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
|
||||
|
||||
req, err := http.NewRequest("POST", endpoint, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create request, %v", err)
|
||||
}
|
||||
|
||||
req.Header.Set("FooInnerSpace", " inner space ")
|
||||
req.Header.Set("FooLeadingSpace", " leading-space")
|
||||
req.Header.Add("FooMultipleSpace", "no-space")
|
||||
req.Header.Add("FooMultipleSpace", "\ttab-space")
|
||||
req.Header.Add("FooMultipleSpace", "trailing-space ")
|
||||
req.Header.Set("FooNoSpace", "no-space")
|
||||
req.Header.Set("FooTabSpace", "\ttab-space\t")
|
||||
req.Header.Set("FooTrailingSpace", "trailing-space ")
|
||||
req.Header.Set("FooWrappedSpace", " wrapped-space ")
|
||||
|
||||
ctx := &httpSigner{
|
||||
ServiceName: serviceName,
|
||||
Region: region,
|
||||
Request: req,
|
||||
Time: v4Internal.NewSigningTime(time.Date(2021, 10, 20, 12, 42, 0, 0, time.UTC)),
|
||||
KeyDerivator: v4Internal.NewSigningKeyDeriver(),
|
||||
}
|
||||
|
||||
build, err := ctx.Build()
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
expectCanonicalString := strings.Join([]string{
|
||||
`POST`,
|
||||
`/`,
|
||||
``,
|
||||
`fooinnerspace:inner space`,
|
||||
`fooleadingspace:leading-space`,
|
||||
`foomultiplespace:no-space,tab-space,trailing-space`,
|
||||
`foonospace:no-space`,
|
||||
`footabspace:tab-space`,
|
||||
`footrailingspace:trailing-space`,
|
||||
`foowrappedspace:wrapped-space`,
|
||||
`host:mockAPI.mock-region.amazonaws.com`,
|
||||
`x-amz-date:20211020T124200Z`,
|
||||
``,
|
||||
`fooinnerspace;fooleadingspace;foomultiplespace;foonospace;footabspace;footrailingspace;foowrappedspace;host;x-amz-date`,
|
||||
``,
|
||||
}, "\n")
|
||||
if diff := cmp.Diff(expectCanonicalString, build.CanonicalString); diff != "" {
|
||||
t.Errorf("expect match, got\n%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkPresignRequest(b *testing.B) {
|
||||
signer := NewSigner()
|
||||
req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
|
||||
|
||||
query := req.URL.Query()
|
||||
query.Set("X-Amz-Expires", "5")
|
||||
req.URL.RawQuery = query.Encode()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
signer.PresignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSignRequest(b *testing.B) {
|
||||
signer := NewSigner()
|
||||
req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
|
||||
for i := 0; i < b.N; i++ {
|
||||
signer.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
|
||||
}
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package azure
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
// Parses azure ResponseError into AWS APIError
|
||||
func azureErrToS3Err(apiErr error) error {
|
||||
var azErr *azcore.ResponseError
|
||||
if !errors.As(apiErr, &azErr) {
|
||||
return apiErr
|
||||
}
|
||||
|
||||
return azErrToS3err(azErr)
|
||||
}
|
||||
|
||||
func azErrToS3err(azErr *azcore.ResponseError) s3err.APIError {
|
||||
switch azErr.ErrorCode {
|
||||
case "ContainerAlreadyExists":
|
||||
return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
|
||||
case "InvalidResourceName", "ContainerNotFound":
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
case "BlobNotFound":
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
case "TagsTooLarge":
|
||||
return s3err.GetAPIError(s3err.ErrInvalidTag)
|
||||
case "Requested Range Not Satisfiable":
|
||||
return s3err.GetAPIError(s3err.ErrInvalidRange)
|
||||
}
|
||||
return s3err.APIError{
|
||||
Code: azErr.ErrorCode,
|
||||
Description: azErr.RawResponse.Status,
|
||||
HTTPStatusCode: azErr.StatusCode,
|
||||
}
|
||||
}
|
||||
|
||||
func parseMpError(mpErr error) error {
|
||||
err := azureErrToS3Err(mpErr)
|
||||
|
||||
serr, ok := err.(s3err.APIError)
|
||||
if !ok || serr.Code != "NoSuchKey" {
|
||||
return mpErr
|
||||
}
|
||||
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchUpload)
|
||||
}
|
||||
@@ -1,264 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package backend
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
"github.com/versity/versitygw/s3select"
|
||||
)
|
||||
|
||||
//go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
|
||||
type Backend interface {
|
||||
fmt.Stringer
|
||||
Shutdown()
|
||||
|
||||
// bucket operations
|
||||
ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
|
||||
HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
|
||||
GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
|
||||
CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
|
||||
PutBucketAcl(_ context.Context, bucket string, data []byte) error
|
||||
DeleteBucket(context.Context, *s3.DeleteBucketInput) error
|
||||
PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
|
||||
GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
|
||||
PutBucketPolicy(_ context.Context, bucket string, policy []byte) error
|
||||
GetBucketPolicy(_ context.Context, bucket string) ([]byte, error)
|
||||
DeleteBucketPolicy(_ context.Context, bucket string) error
|
||||
|
||||
// multipart operations
|
||||
CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
|
||||
CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
|
||||
AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
|
||||
ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
|
||||
ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
|
||||
UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
|
||||
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
|
||||
|
||||
// standard object operations
|
||||
PutObject(context.Context, *s3.PutObjectInput) (string, error)
|
||||
HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
|
||||
GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
|
||||
GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
|
||||
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error)
|
||||
CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
|
||||
ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
|
||||
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
|
||||
DeleteObject(context.Context, *s3.DeleteObjectInput) error
|
||||
DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error)
|
||||
PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
|
||||
ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
|
||||
|
||||
// special case object operations
|
||||
RestoreObject(context.Context, *s3.RestoreObjectInput) error
|
||||
SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
|
||||
|
||||
// bucket tagging operations
|
||||
GetBucketTagging(_ context.Context, bucket string) (map[string]string, error)
|
||||
PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error
|
||||
DeleteBucketTagging(_ context.Context, bucket string) error
|
||||
|
||||
// object tagging operations
|
||||
GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
|
||||
PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
|
||||
DeleteObjectTagging(_ context.Context, bucket, object string) error
|
||||
|
||||
// object lock operations
|
||||
PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error
|
||||
GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error)
|
||||
PutObjectRetention(_ context.Context, bucket, object, versionId string, retention []byte) error
|
||||
GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error)
|
||||
PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error
|
||||
GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error)
|
||||
|
||||
// non AWS actions
|
||||
ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
|
||||
ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
|
||||
}
|
||||
|
||||
type BackendUnsupported struct{}
|
||||
|
||||
var _ Backend = &BackendUnsupported{}
|
||||
|
||||
func New() Backend {
|
||||
return &BackendUnsupported{}
|
||||
}
|
||||
func (BackendUnsupported) Shutdown() {}
|
||||
func (BackendUnsupported) String() string {
|
||||
return "Unsupported"
|
||||
}
|
||||
func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
|
||||
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucketPolicy(_ context.Context, bucket string, policy []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetBucketPolicy(_ context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteBucketPolicy(_ context.Context, bucket string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
|
||||
return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
|
||||
return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
|
||||
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
|
||||
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
|
||||
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
|
||||
return s3response.GetObjectAttributesResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
return s3response.DeleteResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
|
||||
return func(w *bufio.Writer) {
|
||||
var getProgress s3select.GetProgress
|
||||
progress := input.RequestProgress
|
||||
if progress != nil && *progress.Enabled {
|
||||
getProgress = func() (bytesScanned int64, bytesProcessed int64) {
|
||||
return -1, -1
|
||||
}
|
||||
}
|
||||
mh := s3select.NewMessageHandler(ctx, w, getProgress)
|
||||
apiErr := s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
mh.FinishWithError(apiErr.Code, apiErr.Description)
|
||||
}
|
||||
}
|
||||
|
||||
func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteBucketTagging(_ context.Context, bucket string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteObjectTagging(_ context.Context, bucket, object string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) PutObjectLockConfiguration(_ context.Context, bucket string, config []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectLockConfiguration(_ context.Context, bucket string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutObjectRetention(_ context.Context, bucket, object, versionId string, retention []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectRetention(_ context.Context, bucket, object, versionId string) ([]byte, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutObjectLegalHold(_ context.Context, bucket, object, versionId string, status bool) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectLegalHold(_ context.Context, bucket, object, versionId string) (*bool, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {
|
||||
return []s3response.Bucket{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
@@ -1,122 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package backend
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
var (
|
||||
// RFC3339TimeFormat RFC3339 time format
|
||||
RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
|
||||
)
|
||||
|
||||
func IsValidBucketName(name string) bool { return true }
|
||||
|
||||
type ByBucketName []s3response.ListAllMyBucketsEntry
|
||||
|
||||
func (d ByBucketName) Len() int { return len(d) }
|
||||
func (d ByBucketName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
|
||||
func (d ByBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name }
|
||||
|
||||
type ByObjectName []types.Object
|
||||
|
||||
func (d ByObjectName) Len() int { return len(d) }
|
||||
func (d ByObjectName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
|
||||
func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
|
||||
|
||||
func GetStringPtr(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
func GetTimePtr(t time.Time) *time.Time {
|
||||
return &t
|
||||
}
|
||||
|
||||
var (
|
||||
errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
|
||||
)
|
||||
|
||||
// ParseRange parses input range header and returns startoffset, length, and
|
||||
// error. If no endoffset specified, then length is set to -1.
|
||||
func ParseRange(fi fs.FileInfo, acceptRange string) (int64, int64, error) {
|
||||
if acceptRange == "" {
|
||||
return 0, fi.Size(), nil
|
||||
}
|
||||
|
||||
rangeKv := strings.Split(acceptRange, "=")
|
||||
|
||||
if len(rangeKv) < 2 {
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
bRange := strings.Split(rangeKv[1], "-")
|
||||
if len(bRange) < 1 || len(bRange) > 2 {
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
endOffset := int64(-1)
|
||||
if len(bRange) == 1 || bRange[1] == "" {
|
||||
return startOffset, endOffset, nil
|
||||
}
|
||||
|
||||
endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
if endOffset < startOffset {
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
return startOffset, endOffset - startOffset + 1, nil
|
||||
}
|
||||
|
||||
func GetMultipartMD5(parts []types.CompletedPart) string {
|
||||
var partsEtagBytes []byte
|
||||
for _, part := range parts {
|
||||
partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
|
||||
}
|
||||
s3MD5 := fmt.Sprintf("%s-%d", md5String(partsEtagBytes), len(parts))
|
||||
return s3MD5
|
||||
}
|
||||
|
||||
func getEtagBytes(etag string) []byte {
|
||||
decode, err := hex.DecodeString(strings.ReplaceAll(etag, string('"'), ""))
|
||||
if err != nil {
|
||||
return []byte(etag)
|
||||
}
|
||||
return decode
|
||||
}
|
||||
|
||||
func md5String(data []byte) string {
|
||||
sum := md5.Sum(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
package meta
|
||||
|
||||
// MetadataStorer defines the interface for managing metadata.
|
||||
// When object == "", the operation is on the bucket.
|
||||
type MetadataStorer interface {
|
||||
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
|
||||
// Returns the value of the attribute, or an error if the attribute does not exist.
|
||||
RetrieveAttribute(bucket, object, attribute string) ([]byte, error)
|
||||
|
||||
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
|
||||
// If attribute already exists, new attribute should replace existing.
|
||||
// Returns an error if the operation fails.
|
||||
StoreAttribute(bucket, object, attribute string, value []byte) error
|
||||
|
||||
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
|
||||
// Returns an error if the operation fails.
|
||||
DeleteAttribute(bucket, object, attribute string) error
|
||||
|
||||
// ListAttributes lists all attributes for an object or a bucket.
|
||||
// Returns list of attribute names, or an error if the operation fails.
|
||||
ListAttributes(bucket, object string) ([]string, error)
|
||||
|
||||
// DeleteAttributes removes all attributes for an object or a bucket.
|
||||
// Returns an error if the operation fails.
|
||||
DeleteAttributes(bucket, object string) error
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
package meta
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/pkg/xattr"
|
||||
)
|
||||
|
||||
const (
|
||||
xattrPrefix = "user."
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNoSuchKey is returned when the key does not exist.
|
||||
ErrNoSuchKey = errors.New("no such key")
|
||||
)
|
||||
|
||||
type XattrMeta struct{}
|
||||
|
||||
// RetrieveAttribute retrieves the value of a specific attribute for an object in a bucket.
|
||||
func (x XattrMeta) RetrieveAttribute(bucket, object, attribute string) ([]byte, error) {
|
||||
b, err := xattr.Get(filepath.Join(bucket, object), xattrPrefix+attribute)
|
||||
if errors.Is(err, xattr.ENOATTR) {
|
||||
return nil, ErrNoSuchKey
|
||||
}
|
||||
return b, err
|
||||
}
|
||||
|
||||
// StoreAttribute stores the value of a specific attribute for an object in a bucket.
|
||||
func (x XattrMeta) StoreAttribute(bucket, object, attribute string, value []byte) error {
|
||||
return xattr.Set(filepath.Join(bucket, object), xattrPrefix+attribute, value)
|
||||
}
|
||||
|
||||
// DeleteAttribute removes the value of a specific attribute for an object in a bucket.
|
||||
func (x XattrMeta) DeleteAttribute(bucket, object, attribute string) error {
|
||||
err := xattr.Remove(filepath.Join(bucket, object), xattrPrefix+attribute)
|
||||
if errors.Is(err, xattr.ENOATTR) {
|
||||
return ErrNoSuchKey
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteAttributes is not implemented for xattr since xattrs
|
||||
// are automatically removed when the file is deleted.
|
||||
func (x XattrMeta) DeleteAttributes(bucket, object string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListAttributes lists all attributes for an object in a bucket.
|
||||
func (x XattrMeta) ListAttributes(bucket, object string) ([]string, error) {
|
||||
attrs, err := xattr.List(filepath.Join(bucket, object))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
attributes := make([]string, 0, len(attrs))
|
||||
for _, attr := range attrs {
|
||||
if !isUserAttr(attr) {
|
||||
continue
|
||||
}
|
||||
attributes = append(attributes, strings.TrimPrefix(attr, xattrPrefix))
|
||||
}
|
||||
return attributes, nil
|
||||
}
|
||||
|
||||
func isUserAttr(attr string) bool {
|
||||
return strings.HasPrefix(attr, xattrPrefix)
|
||||
}
|
||||
|
||||
// Test is a helper function to test if xattrs are supported.
|
||||
func (x XattrMeta) Test(path string) error {
|
||||
// check for platform support
|
||||
if !xattr.XATTR_SUPPORTED {
|
||||
return fmt.Errorf("xattrs are not supported on this platform")
|
||||
}
|
||||
|
||||
// check if the filesystem supports xattrs
|
||||
_, err := xattr.Get(path, "user.test")
|
||||
if errors.Is(err, syscall.ENOTSUP) {
|
||||
return fmt.Errorf("xattrs are not supported on this filesystem")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// MkdirAll borrowed from stdlib to add ability to set ownership
|
||||
// as directories are created
|
||||
|
||||
package backend
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
var (
|
||||
// TODO: make this configurable
|
||||
defaultDirPerm fs.FileMode = 0755
|
||||
)
|
||||
|
||||
// MkdirAll is similar to os.MkdirAll but it will return
|
||||
// ErrObjectParentIsFile when appropriate
|
||||
// MkdirAll creates a directory named path,
|
||||
// along with any necessary parents, and returns nil,
|
||||
// or else returns an error.
|
||||
// The permission bits perm (before umask) are used for all
|
||||
// directories that MkdirAll creates.
|
||||
// Any newly created directory is set to provided uid/gid ownership.
|
||||
// If path is already a directory, MkdirAll does nothing
|
||||
// and returns nil.
|
||||
// Any directoy created will be set to provided uid/gid ownership
|
||||
// if doChown is true.
|
||||
func MkdirAll(path string, uid, gid int, doChown bool) error {
|
||||
// Fast path: if we can tell whether path is a directory or file, stop with success or error.
|
||||
dir, err := os.Stat(path)
|
||||
if err == nil {
|
||||
if dir.IsDir() {
|
||||
return nil
|
||||
}
|
||||
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
|
||||
}
|
||||
|
||||
// Slow path: make sure parent exists and then call Mkdir for path.
|
||||
i := len(path)
|
||||
for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
|
||||
i--
|
||||
}
|
||||
|
||||
j := i
|
||||
for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
|
||||
j--
|
||||
}
|
||||
|
||||
if j > 1 {
|
||||
// Create parent.
|
||||
err = MkdirAll(path[:j-1], uid, gid, doChown)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Parent now exists; invoke Mkdir and use its result.
|
||||
err = os.Mkdir(path, defaultDirPerm)
|
||||
if err != nil {
|
||||
// Handle arguments like "foo/." by
|
||||
// double-checking that directory doesn't exist.
|
||||
dir, err1 := os.Lstat(path)
|
||||
if err1 == nil && dir.IsDir() {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if doChown {
|
||||
err = os.Chown(path, uid, gid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build linux
|
||||
// +build linux
|
||||
|
||||
package posix
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const procfddir = "/proc/self/fd"
|
||||
|
||||
type tmpfile struct {
|
||||
f *os.File
|
||||
bucket string
|
||||
objname string
|
||||
isOTmp bool
|
||||
size int64
|
||||
needsChown bool
|
||||
uid int
|
||||
gid int
|
||||
}
|
||||
|
||||
var (
|
||||
// TODO: make this configurable
|
||||
defaultFilePerm uint32 = 0644
|
||||
)
|
||||
|
||||
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
|
||||
uid, gid, doChown := p.getChownIDs(acct)
|
||||
|
||||
// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
|
||||
// This can help reduce contention within the namespace (parent directories),
|
||||
// etc. And will auto cleanup the inode on close if we never link this
|
||||
// file descriptor into the namespace.
|
||||
// Not all filesystems support this, so fallback to CreateTemp for when
|
||||
// this is not supported.
|
||||
fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
|
||||
if err != nil {
|
||||
// O_TMPFILE not supported, try fallback
|
||||
err = backend.MkdirAll(dir, uid, gid, doChown)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make temp dir: %w", err)
|
||||
}
|
||||
f, err := os.CreateTemp(dir,
|
||||
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tmp := &tmpfile{
|
||||
f: f,
|
||||
bucket: bucket,
|
||||
objname: obj,
|
||||
size: size,
|
||||
needsChown: doChown,
|
||||
uid: uid,
|
||||
gid: gid,
|
||||
}
|
||||
// falloc is best effort, its fine if this fails
|
||||
if size > 0 {
|
||||
tmp.falloc()
|
||||
}
|
||||
|
||||
if doChown {
|
||||
err := f.Chown(uid, gid)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("set temp file ownership: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return tmp, nil
|
||||
}
|
||||
|
||||
// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
|
||||
// later to link file into namespace
|
||||
f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
|
||||
|
||||
tmp := &tmpfile{
|
||||
f: f,
|
||||
bucket: bucket,
|
||||
objname: obj,
|
||||
isOTmp: true,
|
||||
size: size,
|
||||
needsChown: doChown,
|
||||
uid: uid,
|
||||
gid: gid,
|
||||
}
|
||||
|
||||
// falloc is best effort, its fine if this fails
|
||||
if size > 0 {
|
||||
tmp.falloc()
|
||||
}
|
||||
|
||||
if doChown {
|
||||
err := f.Chown(uid, gid)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("set temp file ownership: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return tmp, nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) falloc() error {
|
||||
err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fallocate: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
// We use Linkat/Rename as the atomic operation for object puts. The
|
||||
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
|
||||
// with any other simultaneous uploads. The final operation is to move the
|
||||
// temp file into place for the object. This ensures the object semantics
|
||||
// of last upload completed wins and is not some combination of writes
|
||||
// from simultaneous uploads.
|
||||
objPath := filepath.Join(tmp.bucket, tmp.objname)
|
||||
err := os.Remove(objPath)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("remove stale path: %w", err)
|
||||
}
|
||||
|
||||
dir := filepath.Dir(objPath)
|
||||
|
||||
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
|
||||
if err != nil {
|
||||
return fmt.Errorf("make parent dir: %w", err)
|
||||
}
|
||||
|
||||
if !tmp.isOTmp {
|
||||
// O_TMPFILE not suported, use fallback
|
||||
return tmp.fallbackLink()
|
||||
}
|
||||
|
||||
procdir, err := os.Open(procfddir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open proc dir: %w", err)
|
||||
}
|
||||
defer procdir.Close()
|
||||
|
||||
dirf, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open parent dir: %w", err)
|
||||
}
|
||||
defer dirf.Close()
|
||||
|
||||
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
|
||||
int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
|
||||
if err != nil {
|
||||
return fmt.Errorf("link tmpfile (%q in %q): %w",
|
||||
filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
|
||||
}
|
||||
|
||||
err = tmp.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("close tmpfile: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) fallbackLink() error {
|
||||
tempname := tmp.f.Name()
|
||||
// cleanup in case anything goes wrong, if rename succeeds then
|
||||
// this will no longer exist
|
||||
defer os.Remove(tempname)
|
||||
|
||||
// reset default file mode because CreateTemp uses 0600
|
||||
tmp.f.Chmod(fs.FileMode(defaultFilePerm))
|
||||
|
||||
err := tmp.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("close tmpfile: %w", err)
|
||||
}
|
||||
|
||||
objPath := filepath.Join(tmp.bucket, tmp.objname)
|
||||
err = os.Rename(tempname, objPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rename tmpfile: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
if int64(len(b)) > tmp.size {
|
||||
return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
|
||||
}
|
||||
|
||||
n, err := tmp.f.Write(b)
|
||||
tmp.size -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
tmp.f.Close()
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build !linux
|
||||
// +build !linux
|
||||
|
||||
package posix
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
)
|
||||
|
||||
type tmpfile struct {
|
||||
f *os.File
|
||||
bucket string
|
||||
objname string
|
||||
size int64
|
||||
}
|
||||
|
||||
func (p *Posix) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
|
||||
uid, gid, doChown := p.getChownIDs(acct)
|
||||
|
||||
// Create a temp file for upload while in progress (see link comments below).
|
||||
var err error
|
||||
err = backend.MkdirAll(dir, uid, gid, doChown)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make temp dir: %w", err)
|
||||
}
|
||||
f, err := os.CreateTemp(dir,
|
||||
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
|
||||
if doChown {
|
||||
err := f.Chown(uid, gid)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("set temp file ownership: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
|
||||
}
|
||||
|
||||
var (
|
||||
// TODO: make this configurable
|
||||
defaultFilePerm fs.FileMode = 0644
|
||||
)
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
tempname := tmp.f.Name()
|
||||
// cleanup in case anything goes wrong, if rename succeeds then
|
||||
// this will no longer exist
|
||||
defer os.Remove(tempname)
|
||||
|
||||
// We use Rename as the atomic operation for object puts. The upload is
|
||||
// written to a temp file to not conflict with any other simultaneous
|
||||
// uploads. The final operation is to move the temp file into place for
|
||||
// the object. This ensures the object semantics of last upload completed
|
||||
// wins and is not some combination of writes from simultaneous uploads.
|
||||
objPath := filepath.Join(tmp.bucket, tmp.objname)
|
||||
err := os.Remove(objPath)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("remove stale path: %w", err)
|
||||
}
|
||||
|
||||
// reset default file mode because CreateTemp uses 0600
|
||||
tmp.f.Chmod(defaultFilePerm)
|
||||
|
||||
err = tmp.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("close tmpfile: %w", err)
|
||||
}
|
||||
|
||||
err = os.Rename(tempname, objPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rename tmpfile: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
if int64(len(b)) > tmp.size {
|
||||
return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
|
||||
}
|
||||
|
||||
n, err := tmp.f.Write(b)
|
||||
tmp.size -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
tmp.f.Close()
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"net/http"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/smithy-go/middleware"
|
||||
)
|
||||
|
||||
func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
|
||||
cfg, err := s.getConfig(ctx, s.access, s.secret)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s3.NewFromConfig(cfg), nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
|
||||
creds := credentials.NewStaticCredentialsProvider(access, secret, "")
|
||||
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
|
||||
}
|
||||
client := &http.Client{Transport: tr}
|
||||
|
||||
opts := []func(*config.LoadOptions) error{
|
||||
config.WithRegion(s.awsRegion),
|
||||
config.WithCredentialsProvider(creds),
|
||||
config.WithHTTPClient(client),
|
||||
}
|
||||
|
||||
if s.endpoint != "" {
|
||||
opts = append(opts,
|
||||
config.WithEndpointResolverWithOptions(s))
|
||||
}
|
||||
|
||||
if s.disableChecksum {
|
||||
opts = append(opts,
|
||||
config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
|
||||
}
|
||||
|
||||
if s.debug {
|
||||
opts = append(opts,
|
||||
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
|
||||
}
|
||||
|
||||
return config.LoadDefaultConfig(ctx, opts...)
|
||||
}
|
||||
|
||||
// ResolveEndpoint is used for on prem or non-aws endpoints
|
||||
func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
PartitionID: "aws",
|
||||
URL: s.endpoint,
|
||||
SigningRegion: s.awsRegion,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
@@ -1,700 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
const aclKey string = "versitygwAcl"
|
||||
|
||||
type S3Proxy struct {
|
||||
backend.BackendUnsupported
|
||||
|
||||
client *s3.Client
|
||||
|
||||
access string
|
||||
secret string
|
||||
endpoint string
|
||||
awsRegion string
|
||||
disableChecksum bool
|
||||
sslSkipVerify bool
|
||||
debug bool
|
||||
}
|
||||
|
||||
func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
|
||||
s := &S3Proxy{
|
||||
access: access,
|
||||
secret: secret,
|
||||
endpoint: endpoint,
|
||||
awsRegion: region,
|
||||
disableChecksum: disableChecksum,
|
||||
sslSkipVerify: sslSkipVerify,
|
||||
debug: debug,
|
||||
}
|
||||
client, err := s.getClientWithCtx(context.Background())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.client = client
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
|
||||
output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{})
|
||||
if err != nil {
|
||||
return s3response.ListAllMyBucketsResult{}, handleError(err)
|
||||
}
|
||||
|
||||
var buckets []s3response.ListAllMyBucketsEntry
|
||||
for _, b := range output.Buckets {
|
||||
buckets = append(buckets, s3response.ListAllMyBucketsEntry{
|
||||
Name: *b.Name,
|
||||
CreationDate: *b.CreationDate,
|
||||
})
|
||||
}
|
||||
|
||||
return s3response.ListAllMyBucketsResult{
|
||||
Owner: s3response.CanonicalUser{
|
||||
ID: *output.Owner.ID,
|
||||
},
|
||||
Buckets: s3response.ListAllMyBucketsList{
|
||||
Bucket: buckets,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
|
||||
out, err := s.client.HeadBucket(ctx, input)
|
||||
return out, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error {
|
||||
_, err := s.client.CreateBucket(ctx, input)
|
||||
if err != nil {
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
var tagSet []types.Tag
|
||||
tagSet = append(tagSet, types.Tag{
|
||||
Key: backend.GetStringPtr(aclKey),
|
||||
Value: backend.GetStringPtr(base64Encode(acl)),
|
||||
})
|
||||
|
||||
_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
|
||||
Bucket: input.Bucket,
|
||||
Tagging: &types.Tagging{
|
||||
TagSet: tagSet,
|
||||
},
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
|
||||
_, err := s.client.DeleteBucket(ctx, input)
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
out, err := s.client.CreateMultipartUpload(ctx, input)
|
||||
return out, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
out, err := s.client.CompleteMultipartUpload(ctx, input)
|
||||
return out, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
|
||||
_, err := s.client.AbortMultipartUpload(ctx, input)
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
|
||||
output, err := s.client.ListMultipartUploads(ctx, input)
|
||||
if err != nil {
|
||||
return s3response.ListMultipartUploadsResult{}, handleError(err)
|
||||
}
|
||||
|
||||
var uploads []s3response.Upload
|
||||
for _, u := range output.Uploads {
|
||||
uploads = append(uploads, s3response.Upload{
|
||||
Key: *u.Key,
|
||||
UploadID: *u.UploadId,
|
||||
Initiator: s3response.Initiator{
|
||||
ID: *u.Initiator.ID,
|
||||
DisplayName: *u.Initiator.DisplayName,
|
||||
},
|
||||
Owner: s3response.Owner{
|
||||
ID: *u.Owner.ID,
|
||||
DisplayName: *u.Owner.DisplayName,
|
||||
},
|
||||
StorageClass: string(u.StorageClass),
|
||||
Initiated: u.Initiated.Format(backend.RFC3339TimeFormat),
|
||||
})
|
||||
}
|
||||
|
||||
var cps []s3response.CommonPrefix
|
||||
for _, c := range output.CommonPrefixes {
|
||||
cps = append(cps, s3response.CommonPrefix{
|
||||
Prefix: *c.Prefix,
|
||||
})
|
||||
}
|
||||
|
||||
return s3response.ListMultipartUploadsResult{
|
||||
Bucket: *output.Bucket,
|
||||
KeyMarker: *output.KeyMarker,
|
||||
UploadIDMarker: *output.UploadIdMarker,
|
||||
NextKeyMarker: *output.NextKeyMarker,
|
||||
NextUploadIDMarker: *output.NextUploadIdMarker,
|
||||
Delimiter: *output.Delimiter,
|
||||
Prefix: *output.Prefix,
|
||||
EncodingType: string(output.EncodingType),
|
||||
MaxUploads: int(*output.MaxUploads),
|
||||
IsTruncated: *output.IsTruncated,
|
||||
Uploads: uploads,
|
||||
CommonPrefixes: cps,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
|
||||
output, err := s.client.ListParts(ctx, input)
|
||||
if err != nil {
|
||||
return s3response.ListPartsResult{}, handleError(err)
|
||||
}
|
||||
|
||||
var parts []s3response.Part
|
||||
for _, p := range output.Parts {
|
||||
parts = append(parts, s3response.Part{
|
||||
PartNumber: int(*p.PartNumber),
|
||||
LastModified: p.LastModified.Format(backend.RFC3339TimeFormat),
|
||||
ETag: *p.ETag,
|
||||
Size: *p.Size,
|
||||
})
|
||||
}
|
||||
pnm, err := strconv.Atoi(*output.PartNumberMarker)
|
||||
if err != nil {
|
||||
return s3response.ListPartsResult{},
|
||||
fmt.Errorf("parse part number marker: %w", err)
|
||||
}
|
||||
|
||||
npmn, err := strconv.Atoi(*output.NextPartNumberMarker)
|
||||
if err != nil {
|
||||
return s3response.ListPartsResult{},
|
||||
fmt.Errorf("parse next part number marker: %w", err)
|
||||
}
|
||||
|
||||
return s3response.ListPartsResult{
|
||||
Bucket: *output.Bucket,
|
||||
Key: *output.Key,
|
||||
UploadID: *output.UploadId,
|
||||
Initiator: s3response.Initiator{
|
||||
ID: *output.Initiator.ID,
|
||||
DisplayName: *output.Initiator.DisplayName,
|
||||
},
|
||||
Owner: s3response.Owner{
|
||||
ID: *output.Owner.ID,
|
||||
DisplayName: *output.Owner.DisplayName,
|
||||
},
|
||||
StorageClass: string(output.StorageClass),
|
||||
PartNumberMarker: pnm,
|
||||
NextPartNumberMarker: npmn,
|
||||
MaxParts: int(*output.MaxParts),
|
||||
IsTruncated: *output.IsTruncated,
|
||||
Parts: parts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
|
||||
// streaming backend is not seekable,
|
||||
// use unsigned payload for streaming ops
|
||||
output, err := s.client.UploadPart(ctx, input, s3.WithAPIOptions(
|
||||
v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
|
||||
))
|
||||
if err != nil {
|
||||
return "", handleError(err)
|
||||
}
|
||||
|
||||
return *output.ETag, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
|
||||
output, err := s.client.UploadPartCopy(ctx, input)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectResult{}, handleError(err)
|
||||
}
|
||||
|
||||
return s3response.CopyObjectResult{
|
||||
LastModified: *output.CopyPartResult.LastModified,
|
||||
ETag: *output.CopyPartResult.ETag,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
|
||||
// streaming backend is not seekable,
|
||||
// use unsigned payload for streaming ops
|
||||
output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions(
|
||||
v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
|
||||
))
|
||||
if err != nil {
|
||||
return "", handleError(err)
|
||||
}
|
||||
|
||||
return *output.ETag, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
out, err := s.client.HeadObject(ctx, input)
|
||||
return out, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
|
||||
output, err := s.client.GetObject(ctx, input)
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
defer output.Body.Close()
|
||||
|
||||
_, err = io.Copy(w, output.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (s3response.GetObjectAttributesResult, error) {
|
||||
out, err := s.client.GetObjectAttributes(ctx, input)
|
||||
|
||||
parts := s3response.ObjectParts{}
|
||||
objParts := out.ObjectParts
|
||||
if objParts != nil {
|
||||
if objParts.PartNumberMarker != nil {
|
||||
partNumberMarker, err := strconv.Atoi(*objParts.PartNumberMarker)
|
||||
if err != nil {
|
||||
parts.PartNumberMarker = partNumberMarker
|
||||
}
|
||||
if objParts.NextPartNumberMarker != nil {
|
||||
nextPartNumberMarker, err := strconv.Atoi(*objParts.NextPartNumberMarker)
|
||||
if err != nil {
|
||||
parts.NextPartNumberMarker = nextPartNumberMarker
|
||||
}
|
||||
}
|
||||
if objParts.IsTruncated != nil {
|
||||
parts.IsTruncated = *objParts.IsTruncated
|
||||
}
|
||||
if objParts.MaxParts != nil {
|
||||
parts.MaxParts = int(*objParts.MaxParts)
|
||||
}
|
||||
parts.Parts = objParts.Parts
|
||||
}
|
||||
}
|
||||
|
||||
return s3response.GetObjectAttributesResult{
|
||||
ETag: out.ETag,
|
||||
LastModified: out.LastModified,
|
||||
ObjectSize: out.ObjectSize,
|
||||
StorageClass: &out.StorageClass,
|
||||
VersionId: out.VersionId,
|
||||
ObjectParts: &parts,
|
||||
}, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
|
||||
out, err := s.client.CopyObject(ctx, input)
|
||||
return out, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
out, err := s.client.ListObjects(ctx, input)
|
||||
return out, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
out, err := s.client.ListObjectsV2(ctx, input)
|
||||
return out, handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
|
||||
_, err := s.client.DeleteObject(ctx, input)
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteResult, error) {
|
||||
if len(input.Delete.Objects) == 0 {
|
||||
input.Delete.Objects = []types.ObjectIdentifier{}
|
||||
}
|
||||
|
||||
output, err := s.client.DeleteObjects(ctx, input)
|
||||
if err != nil {
|
||||
return s3response.DeleteResult{}, handleError(err)
|
||||
}
|
||||
|
||||
return s3response.DeleteResult{
|
||||
Deleted: output.Deleted,
|
||||
Error: output.Errors,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
|
||||
tagout, err := s.client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
|
||||
Bucket: input.Bucket,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
for _, tag := range tagout.TagSet {
|
||||
if *tag.Key == aclKey {
|
||||
acl, err := base64Decode(*tag.Value)
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
return acl, nil
|
||||
}
|
||||
}
|
||||
|
||||
return []byte{}, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
|
||||
tagout, err := s.client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
if err != nil {
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
var found bool
|
||||
for i, tag := range tagout.TagSet {
|
||||
if *tag.Key == aclKey {
|
||||
tagout.TagSet[i] = types.Tag{
|
||||
Key: backend.GetStringPtr(aclKey),
|
||||
Value: backend.GetStringPtr(base64Encode(data)),
|
||||
}
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
tagout.TagSet = append(tagout.TagSet, types.Tag{
|
||||
Key: backend.GetStringPtr(aclKey),
|
||||
Value: backend.GetStringPtr(base64Encode(data)),
|
||||
})
|
||||
}
|
||||
|
||||
_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
|
||||
Bucket: &bucket,
|
||||
Tagging: &types.Tagging{
|
||||
TagSet: tagout.TagSet,
|
||||
},
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
|
||||
tagging := &types.Tagging{
|
||||
TagSet: []types.Tag{},
|
||||
}
|
||||
for key, val := range tags {
|
||||
tagging.TagSet = append(tagging.TagSet, types.Tag{
|
||||
Key: &key,
|
||||
Value: &val,
|
||||
})
|
||||
}
|
||||
|
||||
_, err := s.client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
Tagging: tagging,
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
|
||||
output, err := s.client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
tags := make(map[string]string)
|
||||
for _, el := range output.TagSet {
|
||||
tags[*el.Key] = *el.Value
|
||||
}
|
||||
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
|
||||
_, err := s.client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutBucketPolicy(ctx context.Context, bucket string, policy []byte) error {
|
||||
_, err := s.client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
|
||||
Bucket: &bucket,
|
||||
Policy: backend.GetStringPtr(string(policy)),
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetBucketPolicy(ctx context.Context, bucket string) ([]byte, error) {
|
||||
policy, err := s.client.GetBucketPolicy(ctx, &s3.GetBucketPolicyInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
result := []byte{}
|
||||
if policy.Policy != nil {
|
||||
result = []byte(*policy.Policy)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) DeleteBucketPolicy(ctx context.Context, bucket string) error {
|
||||
_, err := s.client.DeleteBucketPolicy(ctx, &s3.DeleteBucketPolicyInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutObjectLockConfiguration(ctx context.Context, bucket string, config []byte) error {
|
||||
cfg, err := auth.ParseBucketLockConfigurationOutput(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = s.client.PutObjectLockConfiguration(ctx, &s3.PutObjectLockConfigurationInput{
|
||||
Bucket: &bucket,
|
||||
ObjectLockConfiguration: cfg,
|
||||
})
|
||||
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObjectLockConfiguration(ctx context.Context, bucket string) ([]byte, error) {
|
||||
resp, err := s.client.GetObjectLockConfiguration(ctx, &s3.GetObjectLockConfigurationInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
config := auth.BucketLockConfig{
|
||||
Enabled: resp.ObjectLockConfiguration.ObjectLockEnabled == types.ObjectLockEnabledEnabled,
|
||||
DefaultRetention: resp.ObjectLockConfiguration.Rule.DefaultRetention,
|
||||
}
|
||||
|
||||
return json.Marshal(config)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutObjectRetention(ctx context.Context, bucket, object, versionId string, retention []byte) error {
|
||||
ret, err := auth.ParseObjectLockRetentionOutput(retention)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = s.client.PutObjectRetention(ctx, &s3.PutObjectRetentionInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
VersionId: &versionId,
|
||||
Retention: ret,
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObjectRetention(ctx context.Context, bucket, object, versionId string) ([]byte, error) {
|
||||
resp, err := s.client.GetObjectRetention(ctx, &s3.GetObjectRetentionInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
VersionId: &versionId,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
return json.Marshal(resp.Retention)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutObjectLegalHold(ctx context.Context, bucket, object, versionId string, status bool) error {
|
||||
var st types.ObjectLockLegalHoldStatus
|
||||
if status {
|
||||
st = types.ObjectLockLegalHoldStatusOn
|
||||
} else {
|
||||
st = types.ObjectLockLegalHoldStatusOff
|
||||
}
|
||||
|
||||
_, err := s.client.PutObjectLegalHold(ctx, &s3.PutObjectLegalHoldInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
VersionId: &versionId,
|
||||
LegalHold: &types.ObjectLockLegalHold{
|
||||
Status: st,
|
||||
},
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObjectLegalHold(ctx context.Context, bucket, object, versionId string) (*bool, error) {
|
||||
resp, err := s.client.GetObjectLegalHold(ctx, &s3.GetObjectLegalHoldInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
VersionId: &versionId,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
status := resp.LegalHold.Status == types.ObjectLockLegalHoldStatusOn
|
||||
return &status, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, newOwner), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256([]byte{})
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode > 300 {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return fmt.Errorf(string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListBucketsAndOwners(ctx context.Context) ([]s3response.Bucket, error) {
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", s.endpoint), nil)
|
||||
if err != nil {
|
||||
return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256([]byte{})
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
|
||||
if signErr != nil {
|
||||
return []s3response.Bucket{}, fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return []s3response.Bucket{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var buckets []s3response.Bucket
|
||||
if err := json.Unmarshal(body, &buckets); err != nil {
|
||||
return []s3response.Bucket{}, err
|
||||
}
|
||||
|
||||
return buckets, nil
|
||||
}
|
||||
|
||||
func handleError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var ae smithy.APIError
|
||||
if errors.As(err, &ae) {
|
||||
apiErr := s3err.APIError{
|
||||
Code: ae.ErrorCode(),
|
||||
Description: ae.ErrorMessage(),
|
||||
}
|
||||
var re *awshttp.ResponseError
|
||||
if errors.As(err, &re) {
|
||||
apiErr.HTTPStatusCode = re.Response.StatusCode
|
||||
}
|
||||
return apiErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func base64Encode(input []byte) string {
|
||||
return base64.StdEncoding.EncodeToString(input)
|
||||
}
|
||||
|
||||
func base64Decode(encoded string) ([]byte, error) {
|
||||
decoded, err := base64.StdEncoding.DecodeString(encoded)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decoded, nil
|
||||
}
|
||||
@@ -1,833 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package scoutfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
type ScoutfsOpts struct {
|
||||
ChownUID bool
|
||||
ChownGID bool
|
||||
GlacierMode bool
|
||||
}
|
||||
|
||||
type ScoutFS struct {
|
||||
*posix.Posix
|
||||
rootfd *os.File
|
||||
rootdir string
|
||||
|
||||
// glaciermode enables the following behavior:
|
||||
// GET object: if file offline, return invalid object state
|
||||
// HEAD object: if file offline, set obj storage class to GLACIER
|
||||
// if file offline and staging, x-amz-restore: ongoing-request="true"
|
||||
// if file offline and not staging, x-amz-restore: ongoing-request="false"
|
||||
// if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
|
||||
// note: this expiry-date is not used but provided for client glacier compatibility
|
||||
// ListObjects: if file offline, set obj storage class to GLACIER
|
||||
// RestoreObject: add batch stage request to file
|
||||
glaciermode bool
|
||||
|
||||
// chownuid/gid enable chowning of files to the account uid/gid
|
||||
// when objects are uploaded
|
||||
chownuid bool
|
||||
chowngid bool
|
||||
|
||||
// euid/egid are the effective uid/gid of the running versitygw process
|
||||
// used to determine if chowning is needed
|
||||
euid int
|
||||
egid int
|
||||
}
|
||||
|
||||
var _ backend.Backend = &ScoutFS{}
|
||||
|
||||
const (
|
||||
metaTmpDir = ".sgwtmp"
|
||||
metaTmpMultipartDir = metaTmpDir + "/multipart"
|
||||
tagHdr = "X-Amz-Tagging"
|
||||
emptyMD5 = "d41d8cd98f00b204e9800998ecf8427e"
|
||||
etagkey = "user.etag"
|
||||
)
|
||||
|
||||
var (
|
||||
stageComplete = "ongoing-request=\"false\", expiry-date=\"Fri, 2 Dec 2050 00:00:00 GMT\""
|
||||
stageInProgress = "true"
|
||||
stageNotInProgress = "false"
|
||||
)
|
||||
|
||||
const (
|
||||
// ScoutFS special xattr types
|
||||
|
||||
systemPrefix = "scoutfs.hide."
|
||||
onameAttr = systemPrefix + "objname"
|
||||
flagskey = systemPrefix + "sam_flags"
|
||||
stagecopykey = systemPrefix + "sam_stagereq"
|
||||
)
|
||||
|
||||
const (
|
||||
// ScoutAM Flags
|
||||
|
||||
// Staging - file requested stage
|
||||
Staging uint64 = 1 << iota
|
||||
// StageFail - all copies failed to stage
|
||||
StageFail
|
||||
// NoArchive - no archive copies of file should be made
|
||||
NoArchive
|
||||
// ExtCacheRequested means file policy requests Ext Cache
|
||||
ExtCacheRequested
|
||||
// ExtCacheDone means this file ext cache copy has been
|
||||
// created already (and possibly pruned, so may not exist)
|
||||
ExtCacheDone
|
||||
)
|
||||
|
||||
func (s *ScoutFS) Shutdown() {
|
||||
s.Posix.Shutdown()
|
||||
s.rootfd.Close()
|
||||
_ = s.rootdir
|
||||
}
|
||||
|
||||
func (*ScoutFS) String() string {
|
||||
return "ScoutFS Gateway"
|
||||
}
|
||||
|
||||
// getChownIDs returns the uid and gid that should be used for chowning
|
||||
// the object to the account uid/gid. It also returns a boolean indicating
|
||||
// if chowning is needed.
|
||||
func (s *ScoutFS) getChownIDs(acct auth.Account) (int, int, bool) {
|
||||
uid := s.euid
|
||||
gid := s.egid
|
||||
var needsChown bool
|
||||
if s.chownuid && acct.UserID != s.euid {
|
||||
uid = acct.UserID
|
||||
needsChown = true
|
||||
}
|
||||
if s.chowngid && acct.GroupID != s.egid {
|
||||
gid = acct.GroupID
|
||||
needsChown = true
|
||||
}
|
||||
|
||||
return uid, gid, needsChown
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
|
||||
// ioctl to not have to read and copy the part data to the final object. This
|
||||
// saves a read and write cycle for all mutlipart uploads.
|
||||
func (s *ScoutFS) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
acct, ok := ctx.Value("account").(auth.Account)
|
||||
if !ok {
|
||||
acct = auth.Account{}
|
||||
}
|
||||
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
if input.Key == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if input.UploadId == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload)
|
||||
}
|
||||
if input.MultipartUpload == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
uploadID := *input.UploadId
|
||||
parts := input.MultipartUpload.Parts
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
sum, err := s.checkUploadIDExists(bucket, object, uploadID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
|
||||
|
||||
// check all parts ok
|
||||
last := len(parts) - 1
|
||||
partsize := int64(0)
|
||||
var totalsize int64
|
||||
for i, p := range parts {
|
||||
if p.PartNumber == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))
|
||||
fi, err := os.Lstat(partPath)
|
||||
if err != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
|
||||
if i == 0 {
|
||||
partsize = fi.Size()
|
||||
}
|
||||
totalsize += fi.Size()
|
||||
// all parts except the last need to be the same size
|
||||
if i < last && partsize != fi.Size() {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
// non-last part sizes need to be multiples of 4k for move blocks
|
||||
// TODO: fallback to no move blocks if not 4k aligned?
|
||||
if i == 0 && i < last && fi.Size()%4096 != 0 {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
|
||||
b, err := xattr.Get(partPath, "user.etag")
|
||||
etag := string(b)
|
||||
if err != nil {
|
||||
etag = ""
|
||||
}
|
||||
if etag != *parts[i].ETag {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
}
|
||||
|
||||
// use totalsize=0 because we wont be writing to the file, only moving
|
||||
// extents around. so we dont want to fallocate this.
|
||||
f, err := s.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0, acct)
|
||||
if err != nil {
|
||||
if errors.Is(err, syscall.EDQUOT) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrQuotaExceeded)
|
||||
}
|
||||
return nil, fmt.Errorf("open temp file: %w", err)
|
||||
}
|
||||
defer f.cleanup()
|
||||
|
||||
for _, p := range parts {
|
||||
pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open part %v: %v", *p.PartNumber, err)
|
||||
}
|
||||
|
||||
// scoutfs move data is a metadata only operation that moves the data
|
||||
// extent references from the source, appeding to the destination.
|
||||
// this needs to be 4k aligned.
|
||||
err = moveData(pf, f.f)
|
||||
pf.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("move blocks part %v: %v", *p.PartNumber, err)
|
||||
}
|
||||
}
|
||||
|
||||
userMetaData := make(map[string]string)
|
||||
upiddir := filepath.Join(objdir, uploadID)
|
||||
loadUserMetaData(upiddir, userMetaData)
|
||||
|
||||
objname := filepath.Join(bucket, object)
|
||||
dir := filepath.Dir(objname)
|
||||
if dir != "" {
|
||||
uid, gid, doChown := s.getChownIDs(acct)
|
||||
err = backend.MkdirAll(dir, uid, gid, doChown)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
err = f.link()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("link object in namespace: %w", err)
|
||||
}
|
||||
|
||||
for k, v := range userMetaData {
|
||||
err = xattr.Set(objname, "user."+k, []byte(v))
|
||||
if err != nil {
|
||||
// cleanup object if returning error
|
||||
os.Remove(objname)
|
||||
return nil, fmt.Errorf("set user attr %q: %w", k, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate s3 compatible md5sum for complete multipart.
|
||||
s3MD5 := backend.GetMultipartMD5(parts)
|
||||
|
||||
err = xattr.Set(objname, "user.etag", []byte(s3MD5))
|
||||
if err != nil {
|
||||
// cleanup object if returning error
|
||||
os.Remove(objname)
|
||||
return nil, fmt.Errorf("set etag attr: %w", err)
|
||||
}
|
||||
|
||||
// cleanup tmp dirs
|
||||
os.RemoveAll(upiddir)
|
||||
// use Remove for objdir in case there are still other uploads
|
||||
// for same object name outstanding
|
||||
os.Remove(objdir)
|
||||
|
||||
return &s3.CompleteMultipartUploadOutput{
|
||||
Bucket: &bucket,
|
||||
ETag: &s3MD5,
|
||||
Key: &object,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, error) {
|
||||
sum := sha256.Sum256([]byte(object))
|
||||
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
|
||||
|
||||
_, err := os.Stat(filepath.Join(objdir, uploadID))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchUpload)
|
||||
}
|
||||
if err != nil {
|
||||
return [32]byte{}, fmt.Errorf("stat upload: %w", err)
|
||||
}
|
||||
return sum, nil
|
||||
}
|
||||
|
||||
func loadUserMetaData(path string, m map[string]string) (contentType, contentEncoding string) {
|
||||
ents, err := xattr.List(path)
|
||||
if err != nil || len(ents) == 0 {
|
||||
return
|
||||
}
|
||||
for _, e := range ents {
|
||||
if !isValidMeta(e) {
|
||||
continue
|
||||
}
|
||||
b, err := xattr.Get(path, e)
|
||||
if err == errNoData {
|
||||
m[strings.TrimPrefix(e, "user.")] = ""
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
m[strings.TrimPrefix(e, "user.")] = string(b)
|
||||
}
|
||||
|
||||
b, err := xattr.Get(path, "user.content-type")
|
||||
contentType = string(b)
|
||||
if err != nil {
|
||||
contentType = ""
|
||||
}
|
||||
if contentType != "" {
|
||||
m["content-type"] = contentType
|
||||
}
|
||||
|
||||
b, err = xattr.Get(path, "user.content-encoding")
|
||||
contentEncoding = string(b)
|
||||
if err != nil {
|
||||
contentEncoding = ""
|
||||
}
|
||||
if contentEncoding != "" {
|
||||
m["content-encoding"] = contentEncoding
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func isValidMeta(val string) bool {
|
||||
if strings.HasPrefix(val, "user.X-Amz-Meta") {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(val, "user.Expires") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
objPath := filepath.Join(bucket, object)
|
||||
fi, err := os.Stat(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat object: %w", err)
|
||||
}
|
||||
|
||||
userMetaData := make(map[string]string)
|
||||
contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
|
||||
|
||||
b, err := xattr.Get(objPath, etagkey)
|
||||
etag := string(b)
|
||||
if err != nil {
|
||||
etag = ""
|
||||
}
|
||||
|
||||
stclass := types.StorageClassStandard
|
||||
requestOngoing := ""
|
||||
if s.glaciermode {
|
||||
requestOngoing = stageComplete
|
||||
|
||||
// Check if there are any offline exents associated with this file.
|
||||
// If so, we will set storage class to glacier.
|
||||
st, err := statMore(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat more: %w", err)
|
||||
}
|
||||
if st.Offline_blocks != 0 {
|
||||
stclass = types.StorageClassGlacier
|
||||
requestOngoing = stageNotInProgress
|
||||
|
||||
ok, err := isStaging(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("check stage status: %w", err)
|
||||
}
|
||||
if ok {
|
||||
requestOngoing = stageInProgress
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := fi.Size()
|
||||
|
||||
return &s3.HeadObjectOutput{
|
||||
ContentLength: &contentLength,
|
||||
ContentType: &contentType,
|
||||
ContentEncoding: &contentEncoding,
|
||||
ETag: &etag,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Metadata: userMetaData,
|
||||
StorageClass: stclass,
|
||||
Restore: &requestOngoing,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
acceptRange := *input.Range
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
objPath := filepath.Join(bucket, object)
|
||||
fi, err := os.Stat(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat object: %w", err)
|
||||
}
|
||||
|
||||
startOffset, length, err := backend.ParseRange(fi, acceptRange)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
objSize := fi.Size()
|
||||
if fi.IsDir() {
|
||||
// directory objects are always 0 len
|
||||
objSize = 0
|
||||
length = 0
|
||||
}
|
||||
|
||||
if length == -1 {
|
||||
length = fi.Size() - startOffset + 1
|
||||
}
|
||||
|
||||
if startOffset+length > fi.Size() {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
var contentRange string
|
||||
if acceptRange != "" {
|
||||
contentRange = fmt.Sprintf("bytes %v-%v/%v", startOffset, startOffset+length-1, objSize)
|
||||
}
|
||||
|
||||
if s.glaciermode {
|
||||
// Check if there are any offline exents associated with this file.
|
||||
// If so, we will return the InvalidObjectState error.
|
||||
st, err := statMore(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat more: %w", err)
|
||||
}
|
||||
if st.Offline_blocks != 0 {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidObjectState)
|
||||
}
|
||||
}
|
||||
|
||||
f, err := os.Open(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open object: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
rdr := io.NewSectionReader(f, startOffset, length)
|
||||
_, err = io.Copy(writer, rdr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("copy data: %w", err)
|
||||
}
|
||||
|
||||
userMetaData := make(map[string]string)
|
||||
|
||||
contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
|
||||
|
||||
b, err := xattr.Get(objPath, etagkey)
|
||||
etag := string(b)
|
||||
if err != nil {
|
||||
etag = ""
|
||||
}
|
||||
|
||||
tags, err := s.getXattrTags(bucket, object)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get object tags: %w", err)
|
||||
}
|
||||
|
||||
tagCount := int32(len(tags))
|
||||
|
||||
return &s3.GetObjectOutput{
|
||||
AcceptRanges: &acceptRange,
|
||||
ContentLength: &length,
|
||||
ContentEncoding: &contentEncoding,
|
||||
ContentType: &contentType,
|
||||
ETag: &etag,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Metadata: userMetaData,
|
||||
TagCount: &tagCount,
|
||||
StorageClass: types.StorageClassStandard,
|
||||
ContentRange: &contentRange,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error) {
|
||||
tags := make(map[string]string)
|
||||
b, err := xattr.Get(filepath.Join(bucket, object), "user."+tagHdr)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if isNoAttr(err) {
|
||||
return tags, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get tags: %w", err)
|
||||
}
|
||||
|
||||
err = json.Unmarshal(b, &tags)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unmarshal tags: %w", err)
|
||||
}
|
||||
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
bucket := *input.Bucket
|
||||
prefix := ""
|
||||
if input.Prefix != nil {
|
||||
prefix = *input.Prefix
|
||||
}
|
||||
marker := ""
|
||||
if input.Marker != nil {
|
||||
marker = *input.Marker
|
||||
}
|
||||
delim := ""
|
||||
if input.Delimiter != nil {
|
||||
delim = *input.Delimiter
|
||||
}
|
||||
maxkeys := int32(0)
|
||||
if input.MaxKeys != nil {
|
||||
maxkeys = *input.MaxKeys
|
||||
}
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
fileSystem := os.DirFS(bucket)
|
||||
results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
|
||||
s.fileToObj(bucket), []string{metaTmpDir})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("walk %v: %w", bucket, err)
|
||||
}
|
||||
|
||||
return &s3.ListObjectsOutput{
|
||||
CommonPrefixes: results.CommonPrefixes,
|
||||
Contents: results.Objects,
|
||||
Delimiter: &delim,
|
||||
IsTruncated: &results.Truncated,
|
||||
Marker: &marker,
|
||||
MaxKeys: &maxkeys,
|
||||
Name: &bucket,
|
||||
NextMarker: &results.NextMarker,
|
||||
Prefix: &prefix,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
bucket := *input.Bucket
|
||||
prefix := ""
|
||||
if input.Prefix != nil {
|
||||
prefix = *input.Prefix
|
||||
}
|
||||
marker := ""
|
||||
if input.ContinuationToken != nil {
|
||||
marker = *input.ContinuationToken
|
||||
}
|
||||
delim := ""
|
||||
if input.Delimiter != nil {
|
||||
delim = *input.Delimiter
|
||||
}
|
||||
maxkeys := int32(0)
|
||||
if input.MaxKeys != nil {
|
||||
maxkeys = *input.MaxKeys
|
||||
}
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
fileSystem := os.DirFS(bucket)
|
||||
results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
|
||||
s.fileToObj(bucket), []string{metaTmpDir})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("walk %v: %w", bucket, err)
|
||||
}
|
||||
|
||||
return &s3.ListObjectsV2Output{
|
||||
CommonPrefixes: results.CommonPrefixes,
|
||||
Contents: results.Objects,
|
||||
Delimiter: &delim,
|
||||
IsTruncated: &results.Truncated,
|
||||
ContinuationToken: &marker,
|
||||
MaxKeys: &maxkeys,
|
||||
Name: &bucket,
|
||||
NextContinuationToken: &results.NextMarker,
|
||||
Prefix: &prefix,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
|
||||
return func(path string, d fs.DirEntry) (types.Object, error) {
|
||||
objPath := filepath.Join(bucket, path)
|
||||
if d.IsDir() {
|
||||
// directory object only happens if directory empty
|
||||
// check to see if this is a directory object by checking etag
|
||||
etagBytes, err := xattr.Get(objPath, etagkey)
|
||||
if isNoAttr(err) || errors.Is(err, fs.ErrNotExist) {
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil {
|
||||
return types.Object{}, fmt.Errorf("get etag: %w", err)
|
||||
}
|
||||
etag := string(etagBytes)
|
||||
|
||||
fi, err := d.Info()
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil {
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
key := path + "/"
|
||||
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &key,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// file object, get object info and fill out object data
|
||||
etagBytes, err := xattr.Get(objPath, etagkey)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil && !isNoAttr(err) {
|
||||
return types.Object{}, fmt.Errorf("get etag: %w", err)
|
||||
}
|
||||
etag := string(etagBytes)
|
||||
|
||||
fi, err := d.Info()
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil {
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
sc := types.ObjectStorageClassStandard
|
||||
if s.glaciermode {
|
||||
// Check if there are any offline exents associated with this file.
|
||||
// If so, we will return the InvalidObjectState error.
|
||||
st, err := statMore(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil {
|
||||
return types.Object{}, fmt.Errorf("stat more: %w", err)
|
||||
}
|
||||
if st.Offline_blocks != 0 {
|
||||
sc = types.ObjectStorageClassGlacier
|
||||
}
|
||||
}
|
||||
|
||||
size := fi.Size()
|
||||
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Size: &size,
|
||||
StorageClass: sc,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// RestoreObject will set stage request on file if offline and do nothing if
|
||||
// file is online
|
||||
func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) error {
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
err = setStaging(filepath.Join(bucket, object))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("stage object: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func setStaging(objname string) error {
|
||||
b, err := xattr.Get(objname, flagskey)
|
||||
if err != nil && !isNoAttr(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
var oldflags uint64
|
||||
if !isNoAttr(err) {
|
||||
err = json.Unmarshal(b, &oldflags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
newflags := oldflags | Staging
|
||||
|
||||
if newflags == oldflags {
|
||||
// no flags change, just return
|
||||
return nil
|
||||
}
|
||||
|
||||
return fSetNewGlobalFlags(objname, newflags)
|
||||
}
|
||||
|
||||
func isStaging(objname string) (bool, error) {
|
||||
b, err := xattr.Get(objname, flagskey)
|
||||
if err != nil && !isNoAttr(err) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
var flags uint64
|
||||
if !isNoAttr(err) {
|
||||
err = json.Unmarshal(b, &flags)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
return flags&Staging == Staging, nil
|
||||
}
|
||||
|
||||
func fSetNewGlobalFlags(objname string, flags uint64) error {
|
||||
b, err := json.Marshal(&flags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return xattr.Set(objname, flagskey, b)
|
||||
}
|
||||
|
||||
func isNoAttr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
xerr, ok := err.(*xattr.Error)
|
||||
if ok && xerr.Err == xattr.ENOATTR {
|
||||
return true
|
||||
}
|
||||
if err == errNoData {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,206 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build linux && amd64
|
||||
|
||||
package scoutfs
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/versity/scoutfs-go"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/backend/meta"
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
)
|
||||
|
||||
func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
|
||||
p, err := posix.New(rootdir, meta.XattrMeta{}, posix.PosixOpts{
|
||||
ChownUID: opts.ChownUID,
|
||||
ChownGID: opts.ChownGID,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
f, err := os.Open(rootdir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open %v: %w", rootdir, err)
|
||||
}
|
||||
|
||||
return &ScoutFS{
|
||||
Posix: p,
|
||||
rootfd: f,
|
||||
rootdir: rootdir,
|
||||
chownuid: opts.ChownUID,
|
||||
chowngid: opts.ChownGID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
const procfddir = "/proc/self/fd"
|
||||
|
||||
type tmpfile struct {
|
||||
f *os.File
|
||||
bucket string
|
||||
objname string
|
||||
size int64
|
||||
needsChown bool
|
||||
uid int
|
||||
gid int
|
||||
}
|
||||
|
||||
var (
|
||||
// TODO: make this configurable
|
||||
defaultFilePerm uint32 = 0644
|
||||
)
|
||||
|
||||
func (s *ScoutFS) openTmpFile(dir, bucket, obj string, size int64, acct auth.Account) (*tmpfile, error) {
|
||||
uid, gid, doChown := s.getChownIDs(acct)
|
||||
|
||||
// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
|
||||
// This can help reduce contention within the namespace (parent directories),
|
||||
// etc. And will auto cleanup the inode on close if we never link this
|
||||
// file descriptor into the namespace.
|
||||
fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, defaultFilePerm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
|
||||
// later to link file into namespace
|
||||
f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
|
||||
|
||||
tmp := &tmpfile{
|
||||
f: f,
|
||||
bucket: bucket,
|
||||
objname: obj,
|
||||
size: size,
|
||||
needsChown: doChown,
|
||||
uid: uid,
|
||||
gid: gid,
|
||||
}
|
||||
|
||||
// falloc is best effort, its fine if this fails
|
||||
if size > 0 {
|
||||
tmp.falloc()
|
||||
}
|
||||
|
||||
if doChown {
|
||||
err := f.Chown(uid, gid)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("set temp file ownership: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return tmp, nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) falloc() error {
|
||||
err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fallocate: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
// We use Linkat/Rename as the atomic operation for object puts. The
|
||||
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
|
||||
// with any other simultaneous uploads. The final operation is to move the
|
||||
// temp file into place for the object. This ensures the object semantics
|
||||
// of last upload completed wins and is not some combination of writes
|
||||
// from simultaneous uploads.
|
||||
objPath := filepath.Join(tmp.bucket, tmp.objname)
|
||||
err := os.Remove(objPath)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("remove stale path: %w", err)
|
||||
}
|
||||
|
||||
dir := filepath.Dir(objPath)
|
||||
|
||||
err = backend.MkdirAll(dir, tmp.uid, tmp.gid, tmp.needsChown)
|
||||
if err != nil {
|
||||
return fmt.Errorf("make parent dir: %w", err)
|
||||
}
|
||||
|
||||
procdir, err := os.Open(procfddir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open proc dir: %w", err)
|
||||
}
|
||||
defer procdir.Close()
|
||||
|
||||
dirf, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open parent dir: %w", err)
|
||||
}
|
||||
defer dirf.Close()
|
||||
|
||||
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
|
||||
int(dirf.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
|
||||
if err != nil {
|
||||
return fmt.Errorf("link tmpfile: %w", err)
|
||||
}
|
||||
|
||||
err = tmp.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("close tmpfile: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
if int64(len(b)) > tmp.size {
|
||||
return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
|
||||
}
|
||||
|
||||
n, err := tmp.f.Write(b)
|
||||
tmp.size -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
tmp.f.Close()
|
||||
}
|
||||
|
||||
func moveData(from *os.File, to *os.File) error {
|
||||
return scoutfs.MoveData(from, to)
|
||||
}
|
||||
|
||||
func statMore(path string) (stat, error) {
|
||||
st, err := scoutfs.StatMore(path)
|
||||
if err != nil {
|
||||
return stat{}, err
|
||||
}
|
||||
var s stat
|
||||
|
||||
s.Meta_seq = st.Meta_seq
|
||||
s.Data_seq = st.Data_seq
|
||||
s.Data_version = st.Data_version
|
||||
s.Online_blocks = st.Online_blocks
|
||||
s.Offline_blocks = st.Offline_blocks
|
||||
s.Crtime_sec = st.Crtime_sec
|
||||
s.Crtime_nsec = st.Crtime_nsec
|
||||
|
||||
return s, nil
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build !(linux && amd64)
|
||||
|
||||
package scoutfs
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/versity/versitygw/auth"
|
||||
)
|
||||
|
||||
func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
|
||||
return nil, fmt.Errorf("scoutfs only available on linux")
|
||||
}
|
||||
|
||||
type tmpfile struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
var (
|
||||
errNotSupported = errors.New("not supported")
|
||||
)
|
||||
|
||||
func (s *ScoutFS) openTmpFile(_, _, _ string, _ int64, _ auth.Account) (*tmpfile, error) {
|
||||
// make these look used for static check
|
||||
_ = s.chownuid
|
||||
_ = s.chowngid
|
||||
_ = s.euid
|
||||
_ = s.egid
|
||||
return nil, errNotSupported
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
return errNotSupported
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
return 0, errNotSupported
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
}
|
||||
|
||||
func moveData(_, _ *os.File) error {
|
||||
return errNotSupported
|
||||
}
|
||||
|
||||
func statMore(_ string) (stat, error) {
|
||||
return stat{}, errNotSupported
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package scoutfs
|
||||
|
||||
type stat struct {
|
||||
Meta_seq uint64
|
||||
Data_seq uint64
|
||||
Data_version uint64
|
||||
Online_blocks uint64
|
||||
Offline_blocks uint64
|
||||
Crtime_sec uint64
|
||||
Crtime_nsec uint32
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build !freebsd && !openbsd && !netbsd
|
||||
// +build !freebsd,!openbsd,!netbsd
|
||||
|
||||
package scoutfs
|
||||
|
||||
import "syscall"
|
||||
|
||||
var (
|
||||
errNoData = syscall.ENODATA
|
||||
)
|
||||
@@ -1,24 +0,0 @@
|
||||
// Copyright 2024 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build freebsd || openbsd || netbsd
|
||||
// +build freebsd openbsd netbsd
|
||||
|
||||
package scoutfs
|
||||
|
||||
import "syscall"
|
||||
|
||||
var (
|
||||
errNoData = syscall.ENOATTR
|
||||
)
|
||||
222
backend/walk.go
@@ -1,222 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package backend
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
)
|
||||
|
||||
type WalkResults struct {
|
||||
CommonPrefixes []types.CommonPrefix
|
||||
Objects []types.Object
|
||||
Truncated bool
|
||||
NextMarker string
|
||||
}
|
||||
|
||||
type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
|
||||
|
||||
var ErrSkipObj = errors.New("skip this object")
|
||||
|
||||
// Walk walks the supplied fs.FS and returns results compatible with list
|
||||
// objects responses
|
||||
func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
|
||||
cpmap := make(map[string]struct{})
|
||||
var objects []types.Object
|
||||
|
||||
var pastMarker bool
|
||||
if marker == "" {
|
||||
pastMarker = true
|
||||
}
|
||||
|
||||
pastMax := max == 0
|
||||
var newMarker string
|
||||
var truncated bool
|
||||
|
||||
err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Ignore the root directory
|
||||
if path == "." {
|
||||
return nil
|
||||
}
|
||||
if contains(d.Name(), skipdirs) {
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
if pastMax {
|
||||
if len(objects) != 0 {
|
||||
newMarker = *objects[len(objects)-1].Key
|
||||
truncated = true
|
||||
}
|
||||
return fs.SkipAll
|
||||
}
|
||||
|
||||
if d.IsDir() {
|
||||
// If prefix is defined and the directory does not match prefix,
|
||||
// do not descend into the directory because nothing will
|
||||
// match this prefix. Make sure to append the / at the end of
|
||||
// directories since this is implied as a directory path name.
|
||||
// If path is a prefix of prefix, then path could still be
|
||||
// building to match. So only skip if path isn't a prefix of prefix
|
||||
// and prefix isn't a prefix of path.
|
||||
if prefix != "" &&
|
||||
!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
|
||||
!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
// TODO: can we do better here rather than a second readdir
|
||||
// per directory?
|
||||
ents, err := fs.ReadDir(fileSystem, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("readdir %q: %w", path, err)
|
||||
}
|
||||
if len(ents) == 0 {
|
||||
dirobj, err := getObj(path, d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("directory to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, dirobj)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if !pastMarker {
|
||||
if path == marker {
|
||||
pastMarker = true
|
||||
return nil
|
||||
}
|
||||
if path < marker {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// If object doesn't have prefix, don't include in results.
|
||||
if prefix != "" && !strings.HasPrefix(path, prefix) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if delimiter == "" {
|
||||
// If no delimiter specified, then all files with matching
|
||||
// prefix are included in results
|
||||
obj, err := getObj(path, d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("file to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, obj)
|
||||
|
||||
if max > 0 && (len(objects)+len(cpmap)) == int(max) {
|
||||
pastMax = true
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Since delimiter is specified, we only want results that
|
||||
// do not contain the delimiter beyond the prefix. If the
|
||||
// delimiter exists past the prefix, then the substring
|
||||
// between the prefix and delimiter is part of common prefixes.
|
||||
//
|
||||
// For example:
|
||||
// prefix = A/
|
||||
// delimiter = /
|
||||
// and objects:
|
||||
// A/file
|
||||
// A/B/file
|
||||
// B/C
|
||||
// would return:
|
||||
// objects: A/file
|
||||
// common prefix: A/B/
|
||||
//
|
||||
// Note: No objects are included past the common prefix since
|
||||
// these are all rolled up into the common prefix.
|
||||
// Note: The delimiter can be anything, so we have to operate on
|
||||
// the full path without any assumptions on posix directory hierarchy
|
||||
// here. Usually the delimiter will be "/", but thats not required.
|
||||
suffix := strings.TrimPrefix(path, prefix)
|
||||
before, _, found := strings.Cut(suffix, delimiter)
|
||||
if !found {
|
||||
obj, err := getObj(path, d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("file to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, obj)
|
||||
if (len(objects) + len(cpmap)) == int(max) {
|
||||
pastMax = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Common prefixes are a set, so should not have duplicates.
|
||||
// These are abstractly a "directory", so need to include the
|
||||
// delimiter at the end.
|
||||
cpmap[prefix+before+delimiter] = struct{}{}
|
||||
if (len(objects) + len(cpmap)) == int(max) {
|
||||
pastMax = true
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return WalkResults{}, err
|
||||
}
|
||||
|
||||
var commonPrefixStrings []string
|
||||
for k := range cpmap {
|
||||
commonPrefixStrings = append(commonPrefixStrings, k)
|
||||
}
|
||||
sort.Strings(commonPrefixStrings)
|
||||
commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
|
||||
for _, cp := range commonPrefixStrings {
|
||||
pfx := cp
|
||||
commonPrefixes = append(commonPrefixes, types.CommonPrefix{
|
||||
Prefix: &pfx,
|
||||
})
|
||||
}
|
||||
|
||||
return WalkResults{
|
||||
CommonPrefixes: commonPrefixes,
|
||||
Objects: objects,
|
||||
Truncated: truncated,
|
||||
NextMarker: newMarker,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func contains(a string, strs []string) bool {
|
||||
for _, s := range strs {
|
||||
if s == a {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,206 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/backend"
|
||||
)
|
||||
|
||||
type walkTest struct {
|
||||
fsys fs.FS
|
||||
expected backend.WalkResults
|
||||
getobj backend.GetObjFunc
|
||||
}
|
||||
|
||||
func getObj(path string, d fs.DirEntry) (types.Object, error) {
|
||||
if d.IsDir() {
|
||||
etag := getMD5(path)
|
||||
|
||||
fi, err := d.Info()
|
||||
if err != nil {
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
etag := getMD5(path)
|
||||
|
||||
fi, err := d.Info()
|
||||
if err != nil {
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
size := fi.Size()
|
||||
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Size: &size,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func getMD5(text string) string {
|
||||
hash := md5.Sum([]byte(text))
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
func TestWalk(t *testing.T) {
|
||||
tests := []walkTest{
|
||||
{
|
||||
// test case from
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html
|
||||
fsys: fstest.MapFS{
|
||||
"sample.jpg": {},
|
||||
"photos/2006/January/sample.jpg": {},
|
||||
"photos/2006/February/sample2.jpg": {},
|
||||
"photos/2006/February/sample3.jpg": {},
|
||||
"photos/2006/February/sample4.jpg": {},
|
||||
},
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetStringPtr("photos/"),
|
||||
}},
|
||||
Objects: []types.Object{{
|
||||
Key: backend.GetStringPtr("sample.jpg"),
|
||||
}},
|
||||
},
|
||||
getobj: getObj,
|
||||
},
|
||||
{
|
||||
// test case single dir/single file
|
||||
fsys: fstest.MapFS{
|
||||
"test/file": {},
|
||||
},
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetStringPtr("test/"),
|
||||
}},
|
||||
Objects: []types.Object{},
|
||||
},
|
||||
getobj: getObj,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
|
||||
if err != nil {
|
||||
t.Fatalf("walk: %v", err)
|
||||
}
|
||||
|
||||
compareResults(res, tt.expected, t)
|
||||
}
|
||||
}
|
||||
|
||||
func compareResults(got, wanted backend.WalkResults, t *testing.T) {
|
||||
if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
|
||||
t.Errorf("unexpected common prefix, got %v wanted %v",
|
||||
printCommonPrefixes(got.CommonPrefixes),
|
||||
printCommonPrefixes(wanted.CommonPrefixes))
|
||||
}
|
||||
|
||||
if !compareObjects(got.Objects, wanted.Objects) {
|
||||
t.Errorf("unexpected object, got %v wanted %v",
|
||||
printObjects(got.Objects),
|
||||
printObjects(wanted.Objects))
|
||||
}
|
||||
}
|
||||
|
||||
func compareCommonPrefix(a, b []types.CommonPrefix) bool {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return true
|
||||
}
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, cp := range a {
|
||||
if containsCommonPrefix(cp, b) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsCommonPrefix(c types.CommonPrefix, list []types.CommonPrefix) bool {
|
||||
for _, cp := range list {
|
||||
if *c.Prefix == *cp.Prefix {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func printCommonPrefixes(list []types.CommonPrefix) string {
|
||||
res := "["
|
||||
for _, cp := range list {
|
||||
if res == "[" {
|
||||
res = res + *cp.Prefix
|
||||
} else {
|
||||
res = res + ", " + *cp.Prefix
|
||||
}
|
||||
}
|
||||
return res + "]"
|
||||
}
|
||||
|
||||
func compareObjects(a, b []types.Object) bool {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return true
|
||||
}
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, cp := range a {
|
||||
if containsObject(cp, b) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsObject(c types.Object, list []types.Object) bool {
|
||||
for _, cp := range list {
|
||||
if *c.Key == *cp.Key {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func printObjects(list []types.Object) string {
|
||||
res := "["
|
||||
for _, cp := range list {
|
||||
if res == "[" {
|
||||
res = res + *cp.Key
|
||||
} else {
|
||||
res = res + ", " + *cp.Key
|
||||
}
|
||||
}
|
||||
return res + "]"
|
||||
}
|
||||
@@ -1,447 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
var (
|
||||
adminAccess string
|
||||
adminSecret string
|
||||
adminEndpoint string
|
||||
allowInsecure bool
|
||||
)
|
||||
|
||||
func adminCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "admin",
|
||||
Usage: "admin CLI tool",
|
||||
Description: `Admin CLI tool for interacting with admin APIs.`,
|
||||
Subcommands: []*cli.Command{
|
||||
{
|
||||
Name: "create-user",
|
||||
Usage: "Create a new user",
|
||||
Action: createUser,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "access key id for the new user",
|
||||
Required: true,
|
||||
Aliases: []string{"a"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "secret",
|
||||
Usage: "secret access key for the new user",
|
||||
Required: true,
|
||||
Aliases: []string{"s"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "role",
|
||||
Usage: "role for the new user",
|
||||
Required: true,
|
||||
Aliases: []string{"r"},
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "user-id",
|
||||
Usage: "userID for the new user",
|
||||
Aliases: []string{"ui"},
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "group-id",
|
||||
Usage: "groupID for the new user",
|
||||
Aliases: []string{"gi"},
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "project-id",
|
||||
Usage: "projectID for the new user",
|
||||
Aliases: []string{"pi"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "delete-user",
|
||||
Usage: "Delete a user",
|
||||
Action: deleteUser,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "access key id of the user to be deleted",
|
||||
Required: true,
|
||||
Aliases: []string{"a"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "list-users",
|
||||
Usage: "List all the gateway users",
|
||||
Action: listUsers,
|
||||
},
|
||||
{
|
||||
Name: "change-bucket-owner",
|
||||
Usage: "Changes the bucket owner",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "bucket",
|
||||
Usage: "the bucket name to change the owner",
|
||||
Required: true,
|
||||
Aliases: []string{"b"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "owner",
|
||||
Usage: "the user access key id, who should be the bucket owner",
|
||||
Required: true,
|
||||
Aliases: []string{"o"},
|
||||
},
|
||||
},
|
||||
Action: changeBucketOwner,
|
||||
},
|
||||
{
|
||||
Name: "list-buckets",
|
||||
Usage: "Lists all the gateway buckets and owners.",
|
||||
Action: listBuckets,
|
||||
},
|
||||
},
|
||||
Flags: []cli.Flag{
|
||||
// TODO: create a configuration file for this
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "admin access key id",
|
||||
EnvVars: []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"},
|
||||
Aliases: []string{"a"},
|
||||
Required: true,
|
||||
Destination: &adminAccess,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "secret",
|
||||
Usage: "admin secret access key",
|
||||
EnvVars: []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"},
|
||||
Aliases: []string{"s"},
|
||||
Required: true,
|
||||
Destination: &adminSecret,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "endpoint-url",
|
||||
Usage: "admin apis endpoint url",
|
||||
EnvVars: []string{"ADMIN_ENDPOINT_URL"},
|
||||
Aliases: []string{"er"},
|
||||
Required: true,
|
||||
Destination: &adminEndpoint,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "allow-insecure",
|
||||
Usage: "disable tls certificate verification for the admin endpoint",
|
||||
EnvVars: []string{"ADMIN_ALLOW_INSECURE"},
|
||||
Aliases: []string{"ai"},
|
||||
Destination: &allowInsecure,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func initHTTPClient() *http.Client {
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: allowInsecure},
|
||||
}
|
||||
return &http.Client{Transport: tr}
|
||||
}
|
||||
|
||||
func createUser(ctx *cli.Context) error {
|
||||
access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
|
||||
userID, groupID, projectID := ctx.Int("user-id"), ctx.Int("group-id"), ctx.Int("projectID")
|
||||
if access == "" || secret == "" {
|
||||
return fmt.Errorf("invalid input parameters for the new user")
|
||||
}
|
||||
if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
|
||||
return fmt.Errorf("invalid input parameter for role: %v", role)
|
||||
}
|
||||
|
||||
acc := auth.Account{
|
||||
Access: access,
|
||||
Secret: secret,
|
||||
Role: auth.Role(role),
|
||||
UserID: userID,
|
||||
GroupID: groupID,
|
||||
ProjectID: projectID,
|
||||
}
|
||||
|
||||
accJson, err := json.Marshal(acc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse user data: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user", adminEndpoint), bytes.NewBuffer(accJson))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256(accJson)
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return fmt.Errorf("%s", body)
|
||||
}
|
||||
|
||||
fmt.Printf("%s\n", body)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteUser(ctx *cli.Context) error {
|
||||
access := ctx.String("access")
|
||||
if access == "" {
|
||||
return fmt.Errorf("invalid input parameter for the new user")
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256([]byte{})
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return fmt.Errorf("%s", body)
|
||||
}
|
||||
|
||||
fmt.Printf("%s\n", body)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func listUsers(ctx *cli.Context) error {
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-users", adminEndpoint), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256([]byte{})
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return fmt.Errorf("%s", body)
|
||||
}
|
||||
|
||||
var accs []auth.Account
|
||||
if err := json.Unmarshal(body, &accs); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
printAcctTable(accs)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
const (
|
||||
// account table formatting
|
||||
minwidth int = 2 // minimal cell width including any padding
|
||||
tabwidth int = 0 // width of tab characters (equivalent number of spaces)
|
||||
padding int = 2 // padding added to a cell before computing its width
|
||||
padchar byte = ' ' // ASCII char used for padding
|
||||
flags uint = 0 // formatting control flags
|
||||
)
|
||||
|
||||
func printAcctTable(accs []auth.Account) {
|
||||
w := new(tabwriter.Writer)
|
||||
w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
|
||||
fmt.Fprintln(w, "Account\tRole\tUserID\tGroupID\tProjectID")
|
||||
fmt.Fprintln(w, "-------\t----\t------\t-------\t---------")
|
||||
for _, acc := range accs {
|
||||
fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", acc.Access, acc.Role, acc.UserID, acc.GroupID, acc.ProjectID)
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
w.Flush()
|
||||
}
|
||||
|
||||
func changeBucketOwner(ctx *cli.Context) error {
|
||||
bucket, owner := ctx.String("bucket"), ctx.String("owner")
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", adminEndpoint, bucket, owner), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256([]byte{})
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return fmt.Errorf("%s", body)
|
||||
}
|
||||
|
||||
fmt.Println(string(body))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func printBuckets(buckets []s3response.Bucket) {
|
||||
w := new(tabwriter.Writer)
|
||||
w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
|
||||
fmt.Fprintln(w, "Bucket\tOwner")
|
||||
fmt.Fprintln(w, "-------\t----")
|
||||
for _, acc := range buckets {
|
||||
fmt.Fprintf(w, "%v\t%v\n", acc.Name, acc.Owner)
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
w.Flush()
|
||||
}
|
||||
|
||||
func listBuckets(ctx *cli.Context) error {
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", adminEndpoint), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256([]byte{})
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := initHTTPClient()
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return fmt.Errorf("%s", body)
|
||||
}
|
||||
|
||||
var buckets []s3response.Bucket
|
||||
if err := json.Unmarshal(body, &buckets); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
printBuckets(buckets)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/backend/azure"
|
||||
)
|
||||
|
||||
var (
|
||||
azAccount, azKey, azServiceURL, azSASToken string
|
||||
)
|
||||
|
||||
func azureCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "azure",
|
||||
Usage: "azure blob storage backend",
|
||||
Description: `direct translation from s3 objects to azure blobs`,
|
||||
Action: runAzure,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "account",
|
||||
Usage: "azure account name",
|
||||
EnvVars: []string{"AZ_ACCOUNT_NAME"},
|
||||
Aliases: []string{"a"},
|
||||
Destination: &azAccount,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "access-key",
|
||||
Usage: "azure account key",
|
||||
EnvVars: []string{"AZ_ACCESS_KEY"},
|
||||
Aliases: []string{"k"},
|
||||
Destination: &azKey,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "sas-token",
|
||||
Usage: "azure blob storage SAS token",
|
||||
EnvVars: []string{"AZ_SAS_TOKEN"},
|
||||
Aliases: []string{"st"},
|
||||
Destination: &azSASToken,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "url",
|
||||
Usage: "azure service URL",
|
||||
EnvVars: []string{"AZ_ENDPOINT"},
|
||||
Aliases: []string{"u"},
|
||||
Destination: &azServiceURL,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func runAzure(ctx *cli.Context) error {
|
||||
be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init azure: %w", err)
|
||||
}
|
||||
|
||||
return runGateway(ctx.Context, be)
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/versity/versitygw/backend/meta"
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
"github.com/versity/versitygw/tests/integration"
|
||||
)
|
||||
|
||||
const (
|
||||
tdir = "tempdir"
|
||||
)
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
|
||||
func initEnv(dir string) {
|
||||
// both
|
||||
debug = true
|
||||
region = "us-east-1"
|
||||
|
||||
// server
|
||||
rootUserAccess = "user"
|
||||
rootUserSecret = "pass"
|
||||
iamDir = dir
|
||||
port = "127.0.0.1:7070"
|
||||
|
||||
// client
|
||||
awsID = "user"
|
||||
awsSecret = "pass"
|
||||
endpoint = "http://127.0.0.1:7070"
|
||||
}
|
||||
|
||||
func initPosix(ctx context.Context) {
|
||||
path, err := os.Getwd()
|
||||
if err != nil {
|
||||
log.Fatalf("get current directory: %v", err)
|
||||
}
|
||||
|
||||
tempdir := filepath.Join(path, tdir)
|
||||
initEnv(tempdir)
|
||||
|
||||
err = os.RemoveAll(tempdir)
|
||||
if err != nil {
|
||||
log.Fatalf("remove temp directory: %v", err)
|
||||
}
|
||||
|
||||
err = os.Mkdir(tempdir, 0755)
|
||||
if err != nil {
|
||||
log.Fatalf("make temp directory: %v", err)
|
||||
}
|
||||
|
||||
be, err := posix.New(tempdir, meta.XattrMeta{}, posix.PosixOpts{})
|
||||
if err != nil {
|
||||
log.Fatalf("init posix: %v", err)
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
err = runGateway(ctx, be)
|
||||
if err != nil && err != context.Canceled {
|
||||
log.Fatalf("run gateway: %v", err)
|
||||
}
|
||||
|
||||
err := os.RemoveAll(tempdir)
|
||||
if err != nil {
|
||||
log.Fatalf("remove temp directory: %v", err)
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
func TestIntegration(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
initPosix(ctx)
|
||||
|
||||
opts := []integration.Option{
|
||||
integration.WithAccess(awsID),
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
|
||||
s := integration.NewS3Conf(opts...)
|
||||
|
||||
// replace below with desired test
|
||||
err := integration.HeadBucket_non_existing_bucket(s)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
cancel()
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -1,581 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
var (
|
||||
port, admPort string
|
||||
rootUserAccess string
|
||||
rootUserSecret string
|
||||
region string
|
||||
admCertFile, admKeyFile string
|
||||
certFile, keyFile string
|
||||
kafkaURL, kafkaTopic, kafkaKey string
|
||||
natsURL, natsTopic string
|
||||
eventWebhookURL string
|
||||
eventConfigFilePath string
|
||||
logWebhookURL string
|
||||
accessLog string
|
||||
healthPath string
|
||||
debug bool
|
||||
pprof string
|
||||
quiet bool
|
||||
iamDir string
|
||||
ldapURL, ldapBindDN, ldapPassword string
|
||||
ldapQueryBase, ldapObjClasses string
|
||||
ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
|
||||
s3IamAccess, s3IamSecret string
|
||||
s3IamRegion, s3IamBucket string
|
||||
s3IamEndpoint string
|
||||
s3IamSslNoVerify, s3IamDebug bool
|
||||
iamCacheDisable bool
|
||||
iamCacheTTL int
|
||||
iamCachePrune int
|
||||
)
|
||||
|
||||
var (
|
||||
// Version is the latest tag (set within Makefile)
|
||||
Version = "git"
|
||||
// Build is the commit hash (set within Makefile)
|
||||
Build = "norev"
|
||||
// BuildTime is the date/time of build (set within Makefile)
|
||||
BuildTime = "none"
|
||||
)
|
||||
|
||||
func main() {
|
||||
setupSignalHandler()
|
||||
|
||||
app := initApp()
|
||||
|
||||
app.Commands = []*cli.Command{
|
||||
posixCommand(),
|
||||
scoutfsCommand(),
|
||||
s3Command(),
|
||||
azureCommand(),
|
||||
adminCommand(),
|
||||
testCommand(),
|
||||
utilsCommand(),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() {
|
||||
<-sigDone
|
||||
fmt.Fprintf(os.Stderr, "terminating signal caught, shutting down\n")
|
||||
cancel()
|
||||
}()
|
||||
|
||||
if err := app.RunContext(ctx, os.Args); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func initApp() *cli.App {
|
||||
return &cli.App{
|
||||
Name: "versitygw",
|
||||
Usage: "Start S3 gateway service with specified backend storage.",
|
||||
Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
|
||||
to access the supported backend storage as if it was a native S3 service.`,
|
||||
Action: func(ctx *cli.Context) error {
|
||||
return ctx.App.Command("help").Run(ctx)
|
||||
},
|
||||
Flags: initFlags(),
|
||||
}
|
||||
}
|
||||
|
||||
func initFlags() []cli.Flag {
|
||||
return []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "version",
|
||||
Usage: "list versitygw version",
|
||||
Aliases: []string{"v"},
|
||||
Action: func(*cli.Context, bool) error {
|
||||
fmt.Println("Version :", Version)
|
||||
fmt.Println("Build :", Build)
|
||||
fmt.Println("BuildTime:", BuildTime)
|
||||
os.Exit(0)
|
||||
return nil
|
||||
},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "port",
|
||||
Usage: "gateway listen address <ip>:<port> or :<port>",
|
||||
EnvVars: []string{"VGW_PORT"},
|
||||
Value: ":7070",
|
||||
Destination: &port,
|
||||
Aliases: []string{"p"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "root user access key",
|
||||
EnvVars: []string{"ROOT_ACCESS_KEY_ID", "ROOT_ACCESS_KEY"},
|
||||
Aliases: []string{"a"},
|
||||
Destination: &rootUserAccess,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "secret",
|
||||
Usage: "root user secret access key",
|
||||
EnvVars: []string{"ROOT_SECRET_ACCESS_KEY", "ROOT_SECRET_KEY"},
|
||||
Aliases: []string{"s"},
|
||||
Destination: &rootUserSecret,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "region",
|
||||
Usage: "s3 region string",
|
||||
EnvVars: []string{"VGW_REGION"},
|
||||
Value: "us-east-1",
|
||||
Destination: ®ion,
|
||||
Aliases: []string{"r"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "cert",
|
||||
Usage: "TLS cert file",
|
||||
EnvVars: []string{"VGW_CERT"},
|
||||
Destination: &certFile,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "key",
|
||||
Usage: "TLS key file",
|
||||
EnvVars: []string{"VGW_KEY"},
|
||||
Destination: &keyFile,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "admin-port",
|
||||
Usage: "gateway admin server listen address <ip>:<port> or :<port>",
|
||||
EnvVars: []string{"VGW_ADMIN_PORT"},
|
||||
Destination: &admPort,
|
||||
Aliases: []string{"ap"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "admin-cert",
|
||||
Usage: "TLS cert file for admin server",
|
||||
EnvVars: []string{"VGW_ADMIN_CERT"},
|
||||
Destination: &admCertFile,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "admin-cert-key",
|
||||
Usage: "TLS key file for admin server",
|
||||
EnvVars: []string{"VGW_ADMIN_CERT_KEY"},
|
||||
Destination: &admKeyFile,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "debug",
|
||||
Usage: "enable debug output",
|
||||
EnvVars: []string{"VGW_DEBUG"},
|
||||
Destination: &debug,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "pprof",
|
||||
Usage: "enable pprof debug on specified port",
|
||||
EnvVars: []string{"VGW_PPROF"},
|
||||
Destination: &pprof,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "quiet",
|
||||
Usage: "silence stdout request logging output",
|
||||
EnvVars: []string{"VGW_QUIET"},
|
||||
Destination: &quiet,
|
||||
Aliases: []string{"q"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "access-log",
|
||||
Usage: "enable server access logging to specified file",
|
||||
EnvVars: []string{"LOGFILE", "VGW_ACCESS_LOG"},
|
||||
Destination: &accessLog,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "log-webhook-url",
|
||||
Usage: "webhook url to send the audit logs",
|
||||
EnvVars: []string{"WEBHOOK", "VGW_LOG_WEBHOOK_URL"},
|
||||
Destination: &logWebhookURL,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "event-kafka-url",
|
||||
Usage: "kafka server url to send the bucket notifications.",
|
||||
EnvVars: []string{"VGW_EVENT_KAFKA_URL"},
|
||||
Destination: &kafkaURL,
|
||||
Aliases: []string{"eku"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "event-kafka-topic",
|
||||
Usage: "kafka server pub-sub topic to send the bucket notifications to",
|
||||
EnvVars: []string{"VGW_EVENT_KAFKA_TOPIC"},
|
||||
Destination: &kafkaTopic,
|
||||
Aliases: []string{"ekt"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "event-kafka-key",
|
||||
Usage: "kafka server put-sub topic key to send the bucket notifications to",
|
||||
EnvVars: []string{"VGW_EVENT_KAFKA_KEY"},
|
||||
Destination: &kafkaKey,
|
||||
Aliases: []string{"ekk"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "event-nats-url",
|
||||
Usage: "nats server url to send the bucket notifications",
|
||||
EnvVars: []string{"VGW_EVENT_NATS_URL"},
|
||||
Destination: &natsURL,
|
||||
Aliases: []string{"enu"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "event-nats-topic",
|
||||
Usage: "nats server pub-sub topic to send the bucket notifications to",
|
||||
EnvVars: []string{"VGW_EVENT_NATS_TOPIC"},
|
||||
Destination: &natsTopic,
|
||||
Aliases: []string{"ent"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "event-webhook-url",
|
||||
Usage: "webhook url to send bucket notifications",
|
||||
EnvVars: []string{"VGW_EVENT_WEBHOOK_URL"},
|
||||
Destination: &eventWebhookURL,
|
||||
Aliases: []string{"ewu"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "event-filter",
|
||||
Usage: "bucket event notifications filters configuration file path",
|
||||
EnvVars: []string{"VGW_EVENT_FILTER"},
|
||||
Destination: &eventConfigFilePath,
|
||||
Aliases: []string{"ef"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-dir",
|
||||
Usage: "if defined, run internal iam service within this directory",
|
||||
EnvVars: []string{"VGW_IAM_DIR"},
|
||||
Destination: &iamDir,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-ldap-url",
|
||||
Usage: "ldap server url to store iam data",
|
||||
EnvVars: []string{"VGW_IAM_LDAP_URL"},
|
||||
Destination: &ldapURL,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-ldap-bind-dn",
|
||||
Usage: "ldap server binding dn, example: 'cn=admin,dc=example,dc=com'",
|
||||
EnvVars: []string{"VGW_IAM_LDAP_BIND_DN"},
|
||||
Destination: &ldapBindDN,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-ldap-bind-pass",
|
||||
Usage: "ldap server user password",
|
||||
EnvVars: []string{"VGW_IAM_LDAP_BIND_PASS"},
|
||||
Destination: &ldapPassword,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-ldap-query-base",
|
||||
Usage: "ldap server destination query, example: 'ou=iam,dc=example,dc=com'",
|
||||
EnvVars: []string{"VGW_IAM_LDAP_QUERY_BASE"},
|
||||
Destination: &ldapQueryBase,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-ldap-object-classes",
|
||||
Usage: "ldap server object classes used to store the data. provide it as comma separated string, example: 'top,person'",
|
||||
EnvVars: []string{"VGW_IAM_LDAP_OBJECT_CLASSES"},
|
||||
Destination: &ldapObjClasses,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-ldap-access-atr",
|
||||
Usage: "ldap server user access key id attribute name",
|
||||
EnvVars: []string{"VGW_IAM_LDAP_ACCESS_ATR"},
|
||||
Destination: &ldapAccessAtr,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-ldap-secret-atr",
|
||||
Usage: "ldap server user secret access key attribute name",
|
||||
EnvVars: []string{"VGW_IAM_LDAP_SECRET_ATR"},
|
||||
Destination: &ldapSecAtr,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iam-ldap-role-atr",
|
||||
Usage: "ldap server user role attribute name",
|
||||
EnvVars: []string{"VGW_IAM_LDAP_ROLE_ATR"},
|
||||
Destination: &ldapRoleAtr,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "s3-iam-access",
|
||||
Usage: "s3 IAM access key",
|
||||
EnvVars: []string{"VGW_S3_IAM_ACCESS_KEY"},
|
||||
Destination: &s3IamAccess,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "s3-iam-secret",
|
||||
Usage: "s3 IAM secret key",
|
||||
EnvVars: []string{"VGW_S3_IAM_SECRET_KEY"},
|
||||
Destination: &s3IamSecret,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "s3-iam-region",
|
||||
Usage: "s3 IAM region",
|
||||
EnvVars: []string{"VGW_S3_IAM_REGION"},
|
||||
Destination: &s3IamRegion,
|
||||
Value: "us-east-1",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "s3-iam-bucket",
|
||||
Usage: "s3 IAM bucket",
|
||||
EnvVars: []string{"VGW_S3_IAM_BUCKET"},
|
||||
Destination: &s3IamBucket,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "s3-iam-endpoint",
|
||||
Usage: "s3 IAM endpoint",
|
||||
EnvVars: []string{"VGW_S3_IAM_ENDPOINT"},
|
||||
Destination: &s3IamEndpoint,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "s3-iam-noverify",
|
||||
Usage: "s3 IAM disable ssl verification",
|
||||
EnvVars: []string{"VGW_S3_IAM_NO_VERIFY"},
|
||||
Destination: &s3IamSslNoVerify,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "s3-iam-debug",
|
||||
Usage: "s3 IAM debug output",
|
||||
EnvVars: []string{"VGW_S3_IAM_DEBUG"},
|
||||
Destination: &s3IamDebug,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "iam-cache-disable",
|
||||
Usage: "disable local iam cache",
|
||||
EnvVars: []string{"VGW_IAM_CACHE_DISABLE"},
|
||||
Destination: &iamCacheDisable,
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "iam-cache-ttl",
|
||||
Usage: "local iam cache entry ttl (seconds)",
|
||||
EnvVars: []string{"VGW_IAM_CACHE_TTL"},
|
||||
Value: 120,
|
||||
Destination: &iamCacheTTL,
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "iam-cache-prune",
|
||||
Usage: "local iam cache cleanup interval (seconds)",
|
||||
EnvVars: []string{"VGW_IAM_CACHE_PRUNE"},
|
||||
Value: 3600,
|
||||
Destination: &iamCachePrune,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "health",
|
||||
Usage: `health check endpoint path. Health endpoint will be configured on GET http method: GET <health>
|
||||
NOTICE: the path has to be specified with '/'. e.g /health`,
|
||||
EnvVars: []string{"VGW_HEALTH"},
|
||||
Destination: &healthPath,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
if rootUserAccess == "" || rootUserSecret == "" {
|
||||
return fmt.Errorf("root user access and secret key must be provided")
|
||||
}
|
||||
|
||||
if pprof != "" {
|
||||
// listen on specified port for pprof debug
|
||||
// point browser to http://<ip:port>/debug/pprof/
|
||||
go func() {
|
||||
log.Fatal(http.ListenAndServe(pprof, nil))
|
||||
}()
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
AppName: "versitygw",
|
||||
ServerHeader: "VERSITYGW",
|
||||
StreamRequestBody: true,
|
||||
DisableKeepalive: true,
|
||||
})
|
||||
|
||||
var opts []s3api.Option
|
||||
|
||||
if certFile != "" || keyFile != "" {
|
||||
if certFile == "" {
|
||||
return fmt.Errorf("TLS key specified without cert file")
|
||||
}
|
||||
if keyFile == "" {
|
||||
return fmt.Errorf("TLS cert specified without key file")
|
||||
}
|
||||
|
||||
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("tls: load certs: %v", err)
|
||||
}
|
||||
opts = append(opts, s3api.WithTLS(cert))
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, s3api.WithDebug())
|
||||
}
|
||||
if admPort == "" {
|
||||
opts = append(opts, s3api.WithAdminServer())
|
||||
}
|
||||
if quiet {
|
||||
opts = append(opts, s3api.WithQuiet())
|
||||
}
|
||||
if healthPath != "" {
|
||||
opts = append(opts, s3api.WithHealth(healthPath))
|
||||
}
|
||||
|
||||
admApp := fiber.New(fiber.Config{
|
||||
AppName: "versitygw",
|
||||
ServerHeader: "VERSITYGW",
|
||||
})
|
||||
|
||||
var admOpts []s3api.AdminOpt
|
||||
|
||||
if admCertFile != "" || admKeyFile != "" {
|
||||
if admCertFile == "" {
|
||||
return fmt.Errorf("TLS key specified without cert file")
|
||||
}
|
||||
if admKeyFile == "" {
|
||||
return fmt.Errorf("TLS cert specified without key file")
|
||||
}
|
||||
|
||||
cert, err := tls.LoadX509KeyPair(admCertFile, admKeyFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("tls: load certs: %v", err)
|
||||
}
|
||||
admOpts = append(admOpts, s3api.WithAdminSrvTLS(cert))
|
||||
}
|
||||
|
||||
iam, err := auth.New(&auth.Opts{
|
||||
Dir: iamDir,
|
||||
LDAPServerURL: ldapURL,
|
||||
LDAPBindDN: ldapBindDN,
|
||||
LDAPPassword: ldapPassword,
|
||||
LDAPQueryBase: ldapQueryBase,
|
||||
LDAPObjClasses: ldapObjClasses,
|
||||
LDAPAccessAtr: ldapAccessAtr,
|
||||
LDAPSecretAtr: ldapSecAtr,
|
||||
LDAPRoleAtr: ldapRoleAtr,
|
||||
S3Access: s3IamAccess,
|
||||
S3Secret: s3IamSecret,
|
||||
S3Region: s3IamRegion,
|
||||
S3Bucket: s3IamBucket,
|
||||
S3Endpoint: s3IamEndpoint,
|
||||
S3DisableSSlVerfiy: s3IamSslNoVerify,
|
||||
S3Debug: s3IamDebug,
|
||||
CacheDisable: iamCacheDisable,
|
||||
CacheTTL: iamCacheTTL,
|
||||
CachePrune: iamCachePrune,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("setup iam: %w", err)
|
||||
}
|
||||
|
||||
logger, err := s3log.InitLogger(&s3log.LogConfig{
|
||||
LogFile: accessLog,
|
||||
WebhookURL: logWebhookURL,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("setup logger: %w", err)
|
||||
}
|
||||
|
||||
evSender, err := s3event.InitEventSender(&s3event.EventConfig{
|
||||
KafkaURL: kafkaURL,
|
||||
KafkaTopic: kafkaTopic,
|
||||
KafkaTopicKey: kafkaKey,
|
||||
NatsURL: natsURL,
|
||||
NatsTopic: natsTopic,
|
||||
WebhookURL: eventWebhookURL,
|
||||
FilterConfigFilePath: eventConfigFilePath,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("init bucket event notifications: %w", err)
|
||||
}
|
||||
|
||||
srv, err := s3api.New(app, be, middlewares.RootUserConfig{
|
||||
Access: rootUserAccess,
|
||||
Secret: rootUserSecret,
|
||||
}, port, region, iam, logger, evSender, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init gateway: %v", err)
|
||||
}
|
||||
|
||||
admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
|
||||
|
||||
c := make(chan error, 2)
|
||||
go func() { c <- srv.Serve() }()
|
||||
if admPort != "" {
|
||||
go func() { c <- admSrv.Serve() }()
|
||||
}
|
||||
|
||||
// for/select blocks until shutdown
|
||||
Loop:
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break Loop
|
||||
case err = <-c:
|
||||
break Loop
|
||||
case <-sigHup:
|
||||
if logger != nil {
|
||||
err = logger.HangUp()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("HUP logger: %w", err)
|
||||
break Loop
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
saveErr := err
|
||||
|
||||
be.Shutdown()
|
||||
|
||||
err = iam.Shutdown()
|
||||
if err != nil {
|
||||
if saveErr == nil {
|
||||
saveErr = err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "shutdown iam: %v\n", err)
|
||||
}
|
||||
|
||||
if logger != nil {
|
||||
err := logger.Shutdown()
|
||||
if err != nil {
|
||||
if saveErr == nil {
|
||||
saveErr = err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
if evSender != nil {
|
||||
err := evSender.Close()
|
||||
if err != nil {
|
||||
if saveErr == nil {
|
||||
saveErr = err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "close event sender: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
return saveErr
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/backend/meta"
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
)
|
||||
|
||||
var (
|
||||
chownuid, chowngid bool
|
||||
)
|
||||
|
||||
func posixCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "posix",
|
||||
Usage: "posix filesystem storage backend",
|
||||
Description: `Any posix filesystem that supports extended attributes. The top level
|
||||
directory for the gateway must be provided. All sub directories of the
|
||||
top level directory are treated as buckets, and all files/directories
|
||||
below the "bucket directory" are treated as the objects. The object
|
||||
name is split on "/" separator to translate to posix storage.
|
||||
For example:
|
||||
top level: /mnt/fs/gwroot
|
||||
bucket: mybucket
|
||||
object: a/b/c/myobject
|
||||
will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
|
||||
Action: runPosix,
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "chuid",
|
||||
Usage: "chown newly created files and directories to client account UID",
|
||||
EnvVars: []string{"VGW_CHOWN_UID"},
|
||||
Destination: &chownuid,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "chgid",
|
||||
Usage: "chown newly created files and directories to client account GID",
|
||||
EnvVars: []string{"VGW_CHOWN_GID"},
|
||||
Destination: &chowngid,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func runPosix(ctx *cli.Context) error {
|
||||
if ctx.NArg() == 0 {
|
||||
return fmt.Errorf("no directory provided for operation")
|
||||
}
|
||||
|
||||
gwroot := (ctx.Args().Get(0))
|
||||
err := meta.XattrMeta{}.Test(gwroot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("posix xattr check: %v", err)
|
||||
}
|
||||
|
||||
be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
|
||||
ChownUID: chownuid,
|
||||
ChownGID: chowngid,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("init posix: %v", err)
|
||||
}
|
||||
|
||||
return runGateway(ctx.Context, be)
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/backend/s3proxy"
|
||||
)
|
||||
|
||||
var (
|
||||
s3proxyAccess string
|
||||
s3proxySecret string
|
||||
s3proxyEndpoint string
|
||||
s3proxyRegion string
|
||||
s3proxyDisableChecksum bool
|
||||
s3proxySslSkipVerify bool
|
||||
s3proxyDebug bool
|
||||
)
|
||||
|
||||
func s3Command() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "s3",
|
||||
Usage: "s3 storage backend",
|
||||
Description: `This runs the gateway like an s3 proxy redirecting requests
|
||||
to an s3 storage backend service.`,
|
||||
Action: runS3,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "s3 proxy server access key id",
|
||||
Value: "",
|
||||
Required: true,
|
||||
EnvVars: []string{"VGW_S3_ACCESS_KEY"},
|
||||
Destination: &s3proxyAccess,
|
||||
Aliases: []string{"a"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "secret",
|
||||
Usage: "s3 proxy server secret access key",
|
||||
Value: "",
|
||||
Required: true,
|
||||
EnvVars: []string{"VGW_S3_SECRET_KEY"},
|
||||
Destination: &s3proxySecret,
|
||||
Aliases: []string{"s"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "endpoint",
|
||||
Usage: "s3 service endpoint, default AWS if not specified",
|
||||
Value: "",
|
||||
EnvVars: []string{"VGW_S3_ENDPOINT"},
|
||||
Destination: &s3proxyEndpoint,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "region",
|
||||
Usage: "s3 service region, default 'us-east-1' if not specified",
|
||||
Value: "us-east-1",
|
||||
EnvVars: []string{"VGW_S3_REGION"},
|
||||
Destination: &s3proxyRegion,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "disable-checksum",
|
||||
Usage: "disable gateway to server object checksums",
|
||||
Value: false,
|
||||
EnvVars: []string{"VGW_S3_DISABLE_CHECKSUM"},
|
||||
Destination: &s3proxyDisableChecksum,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "ssl-skip-verify",
|
||||
Usage: "skip ssl cert verification for s3 service",
|
||||
EnvVars: []string{"VGW_S3_SSL_SKIP_VERIFY"},
|
||||
Value: false,
|
||||
Destination: &s3proxySslSkipVerify,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "debug",
|
||||
Usage: "output extra debug tracing",
|
||||
Value: false,
|
||||
EnvVars: []string{"VGW_S3_DEBUG"},
|
||||
Destination: &s3proxyDebug,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func runS3(ctx *cli.Context) error {
|
||||
be, err := s3proxy.New(s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
|
||||
s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init s3 backend: %w", err)
|
||||
}
|
||||
return runGateway(ctx.Context, be)
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/backend/scoutfs"
|
||||
)
|
||||
|
||||
var (
|
||||
glacier bool
|
||||
)
|
||||
|
||||
func scoutfsCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "scoutfs",
|
||||
Usage: "scoutfs filesystem storage backend",
|
||||
Description: `Support for ScoutFS.
|
||||
The top level directory for the gateway must be provided. All sub directories
|
||||
of the top level directory are treated as buckets, and all files/directories
|
||||
below the "bucket directory" are treated as the objects. The object name is
|
||||
split on "/" separator to translate to posix storage.
|
||||
For example:
|
||||
top level: /mnt/fs/gwroot
|
||||
bucket: mybucket
|
||||
object: a/b/c/myobject
|
||||
will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
|
||||
|
||||
ScoutFS contains optimizations for multipart uploads using extent
|
||||
move interfaces as well as support for tiered filesystems.`,
|
||||
Action: runScoutfs,
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "glacier",
|
||||
Usage: "enable glacier emulation mode",
|
||||
Aliases: []string{"g"},
|
||||
EnvVars: []string{"VGW_SCOUTFS_GLACIER"},
|
||||
Destination: &glacier,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "chuid",
|
||||
Usage: "chown newly created files and directories to client account UID",
|
||||
EnvVars: []string{"VGW_CHOWN_UID"},
|
||||
Destination: &chownuid,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "chgid",
|
||||
Usage: "chown newly created files and directories to client account GID",
|
||||
EnvVars: []string{"VGW_CHOWN_GID"},
|
||||
Destination: &chowngid,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func runScoutfs(ctx *cli.Context) error {
|
||||
if ctx.NArg() == 0 {
|
||||
return fmt.Errorf("no directory provided for operation")
|
||||
}
|
||||
|
||||
var opts scoutfs.ScoutfsOpts
|
||||
opts.GlacierMode = glacier
|
||||
opts.ChownUID = chownuid
|
||||
opts.ChownGID = chowngid
|
||||
|
||||
be, err := scoutfs.New(ctx.Args().Get(0), opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init scoutfs: %v", err)
|
||||
}
|
||||
|
||||
return runGateway(ctx.Context, be)
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
var (
|
||||
sigDone = make(chan bool, 1)
|
||||
sigHup = make(chan bool, 1)
|
||||
)
|
||||
|
||||
func setupSignalHandler() {
|
||||
sigs := make(chan os.Signal, 1)
|
||||
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
|
||||
|
||||
go func() {
|
||||
for sig := range sigs {
|
||||
fmt.Fprintf(os.Stderr, "caught signal %v\n", sig)
|
||||
switch sig {
|
||||
case syscall.SIGINT, syscall.SIGTERM:
|
||||
sigDone <- true
|
||||
case syscall.SIGHUP:
|
||||
sigHup <- true
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -1,318 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/tests/integration"
|
||||
)
|
||||
|
||||
var (
|
||||
awsID string
|
||||
awsSecret string
|
||||
endpoint string
|
||||
prefix string
|
||||
dstBucket string
|
||||
partSize int64
|
||||
objSize int64
|
||||
concurrency int
|
||||
files int
|
||||
totalReqs int
|
||||
upload bool
|
||||
download bool
|
||||
pathStyle bool
|
||||
checksumDisable bool
|
||||
)
|
||||
|
||||
func testCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "test",
|
||||
Usage: "Client side testing command for the gateway",
|
||||
Description: `The testing CLI is used to test group of versitygw actions.
|
||||
It also includes some performance and stress testing`,
|
||||
Subcommands: initTestCommands(),
|
||||
Flags: initTestFlags(),
|
||||
}
|
||||
}
|
||||
|
||||
func initTestFlags() []cli.Flag {
|
||||
return []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "aws user access key",
|
||||
EnvVars: []string{"AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY"},
|
||||
Aliases: []string{"a"},
|
||||
Destination: &awsID,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "secret",
|
||||
Usage: "aws user secret access key",
|
||||
EnvVars: []string{"AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY"},
|
||||
Aliases: []string{"s"},
|
||||
Destination: &awsSecret,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "endpoint",
|
||||
Usage: "s3 server endpoint",
|
||||
Destination: &endpoint,
|
||||
Aliases: []string{"e"},
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "debug",
|
||||
Usage: "enable debug mode",
|
||||
Aliases: []string{"d"},
|
||||
Destination: &debug,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func initTestCommands() []*cli.Command {
|
||||
return append([]*cli.Command{
|
||||
{
|
||||
Name: "full-flow",
|
||||
Usage: "Tests the full flow of gateway.",
|
||||
Description: `Runs all the available tests to test the full flow of the gateway.`,
|
||||
Action: getAction(integration.TestFullFlow),
|
||||
},
|
||||
{
|
||||
Name: "posix",
|
||||
Usage: "Tests posix specific features",
|
||||
Action: getAction(integration.TestPosix),
|
||||
},
|
||||
{
|
||||
Name: "iam",
|
||||
Usage: "Tests iam service",
|
||||
Action: getAction(integration.TestIAM),
|
||||
},
|
||||
{
|
||||
Name: "access-control",
|
||||
Usage: "Tests gateway access control with bucket ACLs and Policies",
|
||||
Action: getAction(integration.TestAccessControl),
|
||||
},
|
||||
{
|
||||
Name: "bench",
|
||||
Usage: "Runs download/upload performance test on the gateway",
|
||||
Description: `Uploads/downloads some number(specified by flags) of files with some capacity(bytes).
|
||||
Logs the results to the console`,
|
||||
Flags: []cli.Flag{
|
||||
&cli.IntFlag{
|
||||
Name: "files",
|
||||
Usage: "Number of objects to read/write",
|
||||
Value: 1,
|
||||
Destination: &files,
|
||||
},
|
||||
&cli.Int64Flag{
|
||||
Name: "objsize",
|
||||
Usage: "Uploading object size",
|
||||
Value: 0,
|
||||
Destination: &objSize,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "prefix",
|
||||
Usage: "Object name prefix",
|
||||
Destination: &prefix,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "upload",
|
||||
Usage: "Upload data to the gateway",
|
||||
Value: false,
|
||||
Destination: &upload,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "download",
|
||||
Usage: "Download data to the gateway",
|
||||
Value: false,
|
||||
Destination: &download,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "bucket",
|
||||
Usage: "Destination bucket name to read/write data",
|
||||
Destination: &dstBucket,
|
||||
},
|
||||
&cli.Int64Flag{
|
||||
Name: "partSize",
|
||||
Usage: "Upload/download size per thread",
|
||||
Value: 64 * 1024 * 1024,
|
||||
Destination: &partSize,
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "concurrency",
|
||||
Usage: "Upload/download threads per object",
|
||||
Value: 1,
|
||||
Destination: &concurrency,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "pathStyle",
|
||||
Usage: "Use Pathstyle bucket addressing",
|
||||
Value: false,
|
||||
Destination: &pathStyle,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "checksumDis",
|
||||
Usage: "Disable server checksum",
|
||||
Value: false,
|
||||
Destination: &checksumDisable,
|
||||
},
|
||||
},
|
||||
Action: func(ctx *cli.Context) error {
|
||||
if upload && download {
|
||||
return fmt.Errorf("must only specify one of upload or download")
|
||||
}
|
||||
if !upload && !download {
|
||||
return fmt.Errorf("must specify one of upload or download")
|
||||
}
|
||||
|
||||
if dstBucket == "" {
|
||||
return fmt.Errorf("must specify bucket")
|
||||
}
|
||||
|
||||
opts := []integration.Option{
|
||||
integration.WithAccess(awsID),
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
integration.WithConcurrency(concurrency),
|
||||
integration.WithPartSize(partSize),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
if pathStyle {
|
||||
opts = append(opts, integration.WithPathStyle())
|
||||
}
|
||||
if checksumDisable {
|
||||
opts = append(opts, integration.WithDisableChecksum())
|
||||
}
|
||||
|
||||
s3conf := integration.NewS3Conf(opts...)
|
||||
|
||||
if upload {
|
||||
return integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
|
||||
} else {
|
||||
return integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "throughput",
|
||||
Usage: "Runs throughput performance test on the gateway",
|
||||
Description: `Calls HeadBucket action the number of times and concurrency level specified with flags by measuring gateway throughput.`,
|
||||
Flags: []cli.Flag{
|
||||
&cli.IntFlag{
|
||||
Name: "reqs",
|
||||
Usage: "Total number of requests to send.",
|
||||
Value: 1000,
|
||||
Destination: &totalReqs,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "bucket",
|
||||
Usage: "Destination bucket name to make the requests",
|
||||
Destination: &dstBucket,
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "concurrency",
|
||||
Usage: "threads per request",
|
||||
Value: 1,
|
||||
Destination: &concurrency,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "checksumDis",
|
||||
Usage: "Disable server checksum",
|
||||
Value: false,
|
||||
Destination: &checksumDisable,
|
||||
},
|
||||
},
|
||||
Action: func(ctx *cli.Context) error {
|
||||
if dstBucket == "" {
|
||||
return fmt.Errorf("must specify the destination bucket")
|
||||
}
|
||||
|
||||
opts := []integration.Option{
|
||||
integration.WithAccess(awsID),
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
integration.WithConcurrency(concurrency),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
if checksumDisable {
|
||||
opts = append(opts, integration.WithDisableChecksum())
|
||||
}
|
||||
|
||||
s3conf := integration.NewS3Conf(opts...)
|
||||
|
||||
return integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
|
||||
},
|
||||
},
|
||||
}, extractIntTests()...)
|
||||
}
|
||||
|
||||
type testFunc func(*integration.S3Conf)
|
||||
|
||||
func getAction(tf testFunc) func(*cli.Context) error {
|
||||
return func(ctx *cli.Context) error {
|
||||
opts := []integration.Option{
|
||||
integration.WithAccess(awsID),
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
|
||||
s := integration.NewS3Conf(opts...)
|
||||
tf(s)
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("RAN:", integration.RunCount, "PASS:", integration.PassCount, "FAIL:", integration.FailCount)
|
||||
if integration.FailCount > 0 {
|
||||
return fmt.Errorf("test failed with %v errors", integration.FailCount)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func extractIntTests() (commands []*cli.Command) {
|
||||
tests := integration.GetIntTests()
|
||||
for key, val := range tests {
|
||||
k := key
|
||||
testFunc := val
|
||||
commands = append(commands, &cli.Command{
|
||||
Name: k,
|
||||
Usage: fmt.Sprintf("Runs %v integration test", key),
|
||||
Action: func(ctx *cli.Context) error {
|
||||
opts := []integration.Option{
|
||||
integration.WithAccess(awsID),
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
|
||||
s := integration.NewS3Conf(opts...)
|
||||
err := testFunc(s)
|
||||
return err
|
||||
},
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
)
|
||||
|
||||
func utilsCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "utils",
|
||||
Usage: "utility helper CLI tool",
|
||||
Subcommands: []*cli.Command{
|
||||
{
|
||||
Name: "gen-event-filter-config",
|
||||
Aliases: []string{"gefc"},
|
||||
Usage: "Create a new configuration file for bucket event notifications filter.",
|
||||
Action: generateEventFiltersConfig,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "path",
|
||||
Usage: "the path where the config file has to be created",
|
||||
Aliases: []string{"p"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func generateEventFiltersConfig(ctx *cli.Context) error {
|
||||
pathFlag := ctx.String("path")
|
||||
path, err := filepath.Abs(filepath.Join(pathFlag, "event_config.json"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
config := s3event.EventFilter{
|
||||
s3event.EventObjectCreated: true,
|
||||
s3event.EventObjectCreatedPut: true,
|
||||
s3event.EventObjectCreatedPost: true,
|
||||
s3event.EventObjectCreatedCopy: true,
|
||||
s3event.EventCompleteMultipartUpload: true,
|
||||
s3event.EventObjectRemoved: true,
|
||||
s3event.EventObjectRemovedDelete: true,
|
||||
s3event.EventObjectRemovedDeleteObjects: true,
|
||||
s3event.EventObjectTagging: true,
|
||||
s3event.EventObjectTaggingPut: true,
|
||||
s3event.EventObjectTaggingDelete: true,
|
||||
s3event.EventObjectAclPut: true,
|
||||
s3event.EventObjectRestore: true,
|
||||
s3event.EventObjectRestorePost: true,
|
||||
s3event.EventObjectRestoreCompleted: true,
|
||||
}
|
||||
|
||||
configBytes, err := json.Marshal(config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse event config: %w", err)
|
||||
}
|
||||
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create config file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = file.Write(configBytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("write config file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
version: '3'
|
||||
|
||||
services:
|
||||
no_certs:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile_test_bats
|
||||
args:
|
||||
- CONFIG_FILE=tests/.env.nocerts
|
||||
static_buckets:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile_test_bats
|
||||
args:
|
||||
- CONFIG_FILE=tests/.env.static
|
||||
posix_backend:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile_test_bats
|
||||
args:
|
||||
- CONFIG_FILE=tests/.env.default
|
||||
s3_backend:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile_test_bats
|
||||
args:
|
||||
- CONFIG_FILE=tests/.env.s3
|
||||
- SECRETS_FILE=tests/.secrets.s3
|
||||
direct:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile_test_bats
|
||||
args:
|
||||
- CONFIG_FILE=tests/.env.direct
|
||||
- SECRETS_FILE=tests/.secrets.direct
|
||||
@@ -1,43 +0,0 @@
|
||||
version: "3"
|
||||
services:
|
||||
posix:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: ./Dockerfile.dev
|
||||
args:
|
||||
- IAM_DIR=${IAM_DIR}
|
||||
- SETUP_DIR=${SETUP_DIR}
|
||||
volumes:
|
||||
- ./:/app
|
||||
ports:
|
||||
- "${POSIX_PORT}:${POSIX_PORT}"
|
||||
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$POSIX_PORT -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR posix $SETUP_DIR"]
|
||||
proxy:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: ./Dockerfile.dev
|
||||
volumes:
|
||||
- ./:/app
|
||||
ports:
|
||||
- "${PROXY_PORT}:${PROXY_PORT}"
|
||||
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$PROXY_PORT s3 -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --endpoint http://posix:$POSIX_PORT"]
|
||||
azurite:
|
||||
image: mcr.microsoft.com/azure-storage/azurite
|
||||
ports:
|
||||
- "10000:10000"
|
||||
- "10001:10001"
|
||||
- "10002:10002"
|
||||
restart: always
|
||||
hostname: azurite
|
||||
command: "azurite --oauth basic --cert /tests/certs/azurite.pem --key /tests/certs/azurite-key.pem --blobHost 0.0.0.0"
|
||||
volumes:
|
||||
- ./tests/certs:/tests/certs
|
||||
azuritegw:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: ./Dockerfile.dev
|
||||
volumes:
|
||||
- ./:/app
|
||||
ports:
|
||||
- 7070:7070
|
||||
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]
|
||||
@@ -1,309 +0,0 @@
|
||||
###################################
|
||||
# VersityGW systemd configuration #
|
||||
###################################
|
||||
|
||||
# Copy this file to /etc/versitygw.d/ and rename it to a unique service name.
|
||||
# For example, if the service name is "mygateway", then the file should be
|
||||
# named /etc/versitygw.d/mygateway.conf.
|
||||
# The systemd template file /lib/systemd/system/versitygw@.service will
|
||||
# automatically load the configuration file for the service name.
|
||||
# To start the gateway, use the following command:
|
||||
# systemctl start versitygw@mygateway
|
||||
# To enable the gateway to start on boot, use the following command:
|
||||
# systemctl enable versitygw@mygateway
|
||||
# To stop the gateway, use the following command:
|
||||
# systemctl stop versitygw@mygateway
|
||||
|
||||
# There can be multiple gateway services running on the same host. Each
|
||||
# gateway service must have a unique service name with a unique configuration
|
||||
# file in /etc/versitygw.d/. They must also listen on different ports and/or
|
||||
# interfaces using the VGW_PORT option.
|
||||
|
||||
##############################
|
||||
# VersityGW Required Options #
|
||||
##############################
|
||||
|
||||
# VGW_BACKEND must be defined, and must be one of: posix, scoutfs, or s3
|
||||
# This defines the backend that the VGW will use for data access.
|
||||
VGW_BACKEND=posix
|
||||
|
||||
# When VGW_BACKEND is posix or scoutfs, VGW_BACKEND_ARG must be defined
|
||||
# as the the top level directory for the gateway.
|
||||
# All sub directories of the top level directory are treated as buckets,
|
||||
# and all files/directories below the "bucket directory" are treated as
|
||||
# the objects. The object name is split on "/" separator to translate
|
||||
# to posix storage.
|
||||
# For example:
|
||||
# (VGW_BACKEND_ARG) top level: /mnt/fs/gwroot
|
||||
# bucket: mybucket
|
||||
# object: a/b/c/myobject
|
||||
# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
|
||||
VGW_BACKEND_ARG=
|
||||
|
||||
############################
|
||||
# VersityGW Global Options #
|
||||
############################
|
||||
|
||||
# commented options are the default values
|
||||
|
||||
# The following must be set, and do not have default values
|
||||
# The access and secret options will specify the root account credentials.
|
||||
# The root account is granted full authorization to all API requests after
|
||||
# authentication.
|
||||
ROOT_ACCESS_KEY_ID=
|
||||
ROOT_SECRET_ACCESS_KEY=
|
||||
|
||||
# The following are optional, and have the default values as listed
|
||||
|
||||
# The VGW_PORT option will specify the listening port for the S3 server.
|
||||
# This option can use either the form <ip>:<port> which will listen only
|
||||
# on the network interface that matches the IP on the specified port, or
|
||||
# :<port> which will listen on all network interfaces on the specified port.
|
||||
# The <ip> spec can either be IP dotted notation or a resolvable hostname.
|
||||
# The <port> spec can either be a numeric port or the service name typically
|
||||
# in /etc/services.
|
||||
#VGW_PORT=:7070
|
||||
|
||||
# The VGW_REGION option will specify the region that the S3 server will
|
||||
# report to clients. This option is optional, and defaults to "us-east-1".
|
||||
#VGW_REGION=us-east-1
|
||||
|
||||
# The VGW_CERT and VGW_KEY options will specify the SSL certificate and
|
||||
# private key that the S3 server will use for SSL connections. This option
|
||||
# is optional, and defaults to not using SSL.
|
||||
#VGW_CERT=
|
||||
#VGW_KEY=
|
||||
|
||||
# The VGW_ADMIN_PORT option will specify the listening port for the admin
|
||||
# server. The admin server endpoint can optionally be set to listen on a
|
||||
# different interface or port than the S3 service. This allows for better
|
||||
# control of firewall restrictions to the admin endpoint. The certs for this
|
||||
# can be different certs than specified for the S3 service. The default when
|
||||
# these are not specified is to have the admin server listen on the same
|
||||
# endpoint as the S3 service.
|
||||
# When VGW_ADMIN_CERT and VGW_ADMIN_CERT_KEY are specified, the admin
|
||||
# server will use SSL.
|
||||
#VGW_ADMIN_PORT=
|
||||
#VGW_ADMIN_CERT=
|
||||
#VGW_ADMIN_CERT_KEY=
|
||||
|
||||
# The VGW_QUIET option when set will supress the S3 server request summary
|
||||
# logging to stdout.
|
||||
#VGW_QUIET=false
|
||||
|
||||
# The VGW_HEALTH option when set will specify the URL to accept health checks
|
||||
# on. The health check endpoint is often used for load balancers to verify
|
||||
# gateway is alive. The health endpoint masks any bucket with this setting.
|
||||
# For example, if the health endpoint is set to /health, the gateway will not
|
||||
# allow creating or listing contents of a bucket called "health". The health
|
||||
# endpoint is unauthenticated, and returns a 200 status for GET.
|
||||
#VGW_HEALTH=
|
||||
|
||||
###############
|
||||
# Access Logs #
|
||||
###############
|
||||
|
||||
# The VGW_ACCESS_LOG option when set will specify the file to log all S3
|
||||
# server requests to. This option is optional, and defaults to not logging.
|
||||
# It is suggested to use absolute paths for the server log file because the
|
||||
# server may chdir into the backend root directory and change locations for
|
||||
# relative paths.
|
||||
# The log file format follows the AWS S3 access log format documented in
|
||||
# https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html.
|
||||
#VGW_ACCESS_LOG=
|
||||
|
||||
# The VGW_LOG_WEBHOOK_URL option when set will specify the URL to send the
|
||||
# S3 server request access logs to. The access logs are JSON encoded when
|
||||
# sent to the webhook.
|
||||
#VGW_LOG_WEBHOOK_URL=
|
||||
|
||||
##############
|
||||
# Event Logs #
|
||||
##############
|
||||
|
||||
# The gateway events are similar to AWS S3 events, and are documented in the
|
||||
# wiki:
|
||||
# https://github.com/versity/versitygw/wiki/Events-Notifications.
|
||||
|
||||
# The VGW_EVENT_FILTER option specifies a config file that contains the
|
||||
# event filter rules. The event filter rules are used to determine which
|
||||
# events are sent to the configured event services.
|
||||
# Use the following to generate a default rules file in /etc/versitygw.d/:
|
||||
# versitygw utils gen-event-filter-config -p /etc/versitygw.d
|
||||
# The resulting file, /etc/versitygw.d/event_config.json, can be modified and
|
||||
# specified in the VGW_EVENT_FILTER option.
|
||||
# When VGW_EVENT_FILTER is not specified, all events are sent to the configured
|
||||
# event service.
|
||||
#VGW_EVENT_FILTER=
|
||||
|
||||
# Bucket events can be sent to a Kafka message bus. When VGW_EVENT_KAFKA_URL,
|
||||
# VGW_EVENT_KAFKA_TOPIC, and optionally VGW_EVENT_KAFKA_KEY are specified, all
|
||||
# configured bucket events will be sent to the kafka service.
|
||||
#VGW_EVENT_KAFKA_URL=
|
||||
#VGW_EVENT_KAFKA_TOPIC=
|
||||
#VGW_EVENT_KAFKA_KEY=
|
||||
|
||||
# Bucket events can be sent to a NATS messaging service. When VGW_EVENT_NATS_URL
|
||||
# and VGW_EVENT_NATS_TOPIC are specified, all configured bucket events will be
|
||||
# sent to the the NATS messaging service.
|
||||
#VGW_EVENT_NATS_URL=
|
||||
#VGW_EVENT_NATS_TOPIC=
|
||||
|
||||
# Bucket events can be sent to a webhook. When VGW_EVENT_WEBHOOK_URL is
|
||||
# specified, all configured bucket events will be sent to the webhook.
|
||||
#VGW_EVENT_WEBHOOK_URL=
|
||||
|
||||
#######################
|
||||
# Debug / Diagnostics #
|
||||
#######################
|
||||
|
||||
# The VGW_DEBUG option enables verbose debug log output to stdout. This output
|
||||
# includes details for signature verification steps. This is generally only
|
||||
# useful for debugging the S3 server, and should not be used in production.
|
||||
#VGW_DEBUG=false
|
||||
|
||||
# The VGW_PPROF option enables the pprof HTTP server for profiling the S3
|
||||
# server. See the following for more information:
|
||||
# https://pkg.go.dev/net/http/pprof
|
||||
# To enable, set the VGW_PPROF option to the listening address for the pprof
|
||||
# server. For example, to listen on localhost port 6060, set the option to
|
||||
# "localhost:6060".
|
||||
#VGW_PPROF=
|
||||
|
||||
################
|
||||
# IAM services #
|
||||
################
|
||||
|
||||
# The VGW_IAM_DIR option will enable the internal IAM service with accounts
|
||||
# stored in a file under the specified directory. This is provided to minimize
|
||||
# dependencies on outside services for basic functionality. The local account
|
||||
# files are plain text and only protected with file permissions. This IAM
|
||||
# service is added for convenience, but is not considered as secure or scalable
|
||||
# as a dedicated IAM service.
|
||||
#VGW_IAM_DIR=
|
||||
|
||||
# The ldap options will enable the LDAP IAM service with accounts stored in an
|
||||
# external LDAP service. The VGW_IAM_LDAP_ACCESS_ATR, VGW_IAM_LDAP_SECRET_ATR,
|
||||
# and VGW_IAM_LDAP_ROLE_ATR define the LDAP attributes that map to access,
|
||||
# secret credentials and role respectively. The other options are used to
|
||||
# connect to the LDAP service.
|
||||
#VGW_IAM_LDAP_URL=
|
||||
#VGW_IAM_LDAP_BASE_DN=
|
||||
#VGW_IAM_LDAP_BIND_DN=
|
||||
#VGW_IAM_LDAP_BIND_PASS=
|
||||
#VGW_IAM_LDAP_QUERY_BASE=
|
||||
#VGW_IAM_LDAP_OBJECT_CLASSES=
|
||||
#VGW_IAM_LDAP_ACCESS_ATR=
|
||||
#VGW_IAM_LDAP_SECRET_ATR=
|
||||
#VGW_IAM_LDAP_ROLE_ATR=
|
||||
|
||||
# The VGW_S3 IAM service is similar to the internal IAM service, but instead
|
||||
# stores the account information JSON encoded in an S3 object. This should use
|
||||
# a bucket that is not accessible to general users when using s3 backend to
|
||||
# prevent access to account credentials. This IAM service is added for
|
||||
# convenience, but is not considered as secure or scalable as a dedicated IAM
|
||||
# service.
|
||||
#VGW_S3_IAM_ACCESS_KEY=
|
||||
#VGW_S3_IAM_SECRET_KEY=
|
||||
#VGW_S3_IAM_REGION=
|
||||
#VGW_S3_IAM_ENDPOINT=
|
||||
#VGW_S3_IAM_BUCKET=
|
||||
#VGW_S3_IAM_NO_VERIFY=
|
||||
|
||||
###############
|
||||
# IAM caching #
|
||||
###############
|
||||
|
||||
# The IAM cache is intended to ease the load on the IAM service and increase
|
||||
# the Gateway performance by caching accounts and credentials for the TTL time
|
||||
# interval. Disabling this will cause a request to the configured IAM service
|
||||
# for each incoming request to retrieve the corresponding account credentials.
|
||||
# The cache is enabled by default. The TTL specifies how long to cache
|
||||
# credentials, and the prune value determines the interval for expired entries
|
||||
# to be removed from the cache. Increasing the TTL may lessen the load on the
|
||||
# IAM service backend, but may have out of date account info until the next
|
||||
# interval. Increasing the prune value may reduce memory use at the cost of
|
||||
# added CPU to check cache expirations.
|
||||
#VGW_IAM_CACHE_DISABLE=false
|
||||
#VGW_IAM_CACHE_TTL=120
|
||||
#VGW_IAM_CACHE_PRUNE=3600
|
||||
|
||||
######################################
|
||||
# VersityGW Backend Specific Options #
|
||||
######################################
|
||||
|
||||
#########
|
||||
# posix #
|
||||
#########
|
||||
|
||||
# The posix backend translates S3 requests to file access in a local filesystem.
|
||||
# The posix backend requires a filesystem that supports extended attributes.
|
||||
# The top level directory for the gateway must be provided. All sub directories
|
||||
# of the top level directory are treated as buckets, and all files/directories
|
||||
# below the "bucket directory" are treated as the objects. The object
|
||||
# name is split on "/" separator to translate to posix storage.
|
||||
# For example:
|
||||
# top level (VGW_BACKEND_ARG): /mnt/fs/gwroot
|
||||
# bucket: mybucket
|
||||
# object: a/b/c/myobject
|
||||
# will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
|
||||
|
||||
# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
|
||||
# change the ownership of newly created files and directories to the IAM
|
||||
# account UID/GID.
|
||||
#VGW_CHOWN_UID=false
|
||||
#VGW_CHOWN_GID=false
|
||||
|
||||
###########
|
||||
# scoutfs #
|
||||
###########
|
||||
|
||||
# The scoutfs backend requires a ScoutFS filesystem type for the backend
|
||||
# path. The object to posix name mappings follow the same rules as posix for
|
||||
# scoutfs. The glacier mode functionality requires ScoutAM to be configured
|
||||
# for tiering data from the ScoutFS filesystem to a mass stroage system.
|
||||
# The mass storage system is often one or more tape libraries. Due to the
|
||||
# high latency of tape, the glacier mode functionality is designed to
|
||||
# give feedback to clients about object state and offer ability to request
|
||||
# data to be staged back to disk without the client dealing with long
|
||||
# request timeout settings.
|
||||
|
||||
# The VGW_SCOUTFS_GLACIER option enables the following Glacier API behavior.
|
||||
# GET object: if file offline, return invalid object state
|
||||
# HEAD object: if file offline, set obj storage class to GLACIER
|
||||
# if file offline and staging, x-amz-restore: ongoing-request="true"
|
||||
# if file offline and not staging, x-amz-restore: ongoing-request="false"
|
||||
# if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
|
||||
# note: this expiry-date is not used but provided for client glacier compatibility
|
||||
# ListObjects: if file offline, set obj storage class to GLACIER
|
||||
# RestoreObject: add batch stage request to file
|
||||
#VGW_SCOUTFS_GLACIER=false
|
||||
|
||||
# The VGW_CHOWN_UID and VGW_CHOWN_GID options will enable the gateway to
|
||||
# change the ownership of newly created files and directories to the IAM
|
||||
# account UID/GID.
|
||||
#VGW_CHOWN_UID=false
|
||||
#VGW_CHOWN_GID=false
|
||||
|
||||
######
|
||||
# s3 #
|
||||
######
|
||||
|
||||
# The s3 backend allows the gateway to forward requests to an S3 compatible
|
||||
# service. This allows the gateway to act as a proxy for another S3 service.
|
||||
# The backend S3 access is all done with a single configured account. The
|
||||
# gateway will manage incoming multi-tenant access with the gateway configured
|
||||
# IAM service. This gives stroage admins the ability to manage local gateway
|
||||
# accounts while maintaining full control and a single account for the backend
|
||||
# S3 service.
|
||||
|
||||
# When s3 backend selected, the VGW_S3_ACCESS_KEY and VGW_S3_SECRET_KEY must
|
||||
# be defined. The VGW_S3_REGION and VGW_S3_ENDPOINT are optional, and will
|
||||
# default to "us-east-1" and "https://s3.amazonaws.com" respectively.
|
||||
#VGW_S3_ACCESS_KEY=
|
||||
#VGW_S3_SECRET_KEY=
|
||||
#VGW_S3_REGION=
|
||||
#VGW_S3_ENDPOINT=
|
||||
#VGW_S3_DISABLE_CHECKSUM=false
|
||||
#VGW_S3_SSL_SKIP_VERIFY=false
|
||||
#VGW_S3_DEBUG=false
|
||||
@@ -1,2 +0,0 @@
|
||||
#!/bin/bash
|
||||
systemctl daemon-reload
|
||||
@@ -1,33 +0,0 @@
|
||||
[Unit]
|
||||
Description=VersityGW
|
||||
Documentation=https://github.com/versity/versitygw/wiki
|
||||
Wants=network-online.target
|
||||
After=network-online.target remote-fs.target
|
||||
AssertFileIsExecutable=/usr/bin/versitygw
|
||||
AssertPathExists=/etc/versitygw.d/%i.conf
|
||||
|
||||
[Service]
|
||||
WorkingDirectory=/root
|
||||
StandardOutput=syslog
|
||||
StandardError=syslog
|
||||
SyslogIdentifier=versitygw-%i
|
||||
|
||||
User=root
|
||||
Group=root
|
||||
|
||||
EnvironmentFile=/etc/versitygw.d/%i.conf
|
||||
|
||||
ExecStart=/bin/bash -c 'if [[ ! ("${VGW_BACKEND}" == "posix" || "${VGW_BACKEND}" == "scoutfs" || "${VGW_BACKEND}" == "s3") ]]; then echo "VGW_BACKEND environment variable not set to one of posix, scoutfs, or s3"; exit 1; fi && exec /usr/bin/versitygw "$VGW_BACKEND" "$VGW_BACKEND_ARG"'
|
||||
|
||||
# Let systemd restart this service always
|
||||
Restart=always
|
||||
|
||||
# Specifies the maximum file descriptor number that can be opened by this process
|
||||
LimitNOFILE=65536
|
||||
|
||||
# Specifies the maximum number of threads this process can create
|
||||
TasksMax=infinity
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
70
go.mod
@@ -1,70 +0,0 @@
|
||||
module github.com/versity/versitygw
|
||||
|
||||
go 1.21.0
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
|
||||
github.com/aws/aws-sdk-go-v2 v1.26.1
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
|
||||
github.com/aws/smithy-go v1.20.2
|
||||
github.com/go-ldap/ldap/v3 v3.4.8
|
||||
github.com/gofiber/fiber/v2 v2.52.4
|
||||
github.com/google/go-cmp v0.6.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/nats-io/nats.go v1.34.1
|
||||
github.com/pkg/xattr v0.4.9
|
||||
github.com/segmentio/kafka-go v0.4.47
|
||||
github.com/urfave/cli/v2 v2.27.2
|
||||
github.com/valyala/fasthttp v1.52.0
|
||||
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
|
||||
golang.org/x/sys v0.19.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.6 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.7 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
golang.org/x/crypto v0.22.0 // indirect
|
||||
golang.org/x/net v0.24.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.11
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.11
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
|
||||
github.com/klauspost/compress v1.17.8 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/tcplisten v1.0.0 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
|
||||
)
|
||||
229
go.sum
@@ -1,229 +0,0 @@
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 h1:FDif4R1+UUR+00q6wquyX90K7A8dN+R5E8GEadoP7sU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2/go.mod h1:aiYBYui4BJ/BJCAIKs92XiPyQfTaBWqvHujDwKb6CBU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0 h1:sUFnFjzDUie80h24I7mrKtwCKgLY9L8h5Tp2x9+TWqk=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.6.0/go.mod h1:52JbnQTp15qg5mRkMBHwp0j0ZFwHJ42Sx3zVV5RE9p0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
|
||||
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
|
||||
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
|
||||
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
|
||||
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.6 h1:CYsqysemXfEaQbyrLJmdsCRuufHoLa3P/gGWGl5TDrM=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.6/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ=
|
||||
github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk=
|
||||
github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
|
||||
github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
|
||||
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
|
||||
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
|
||||
github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
|
||||
github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg=
|
||||
github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8=
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
|
||||
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
|
||||
github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
|
||||
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
|
||||
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||
github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
|
||||
github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
|
||||
github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI=
|
||||
github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
|
||||
github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 h1:Wx1o3pNrCzsHIIDyZ2MLRr6tF/1FhAr7HNDn80QqDWE=
|
||||
github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw=
|
||||
github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
|
||||
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
|
||||
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
|
||||
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
|
||||
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
|
||||
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
51
runtests.sh
@@ -1,51 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# make temp dirs
|
||||
rm -rf /tmp/gw
|
||||
mkdir /tmp/gw
|
||||
rm -rf /tmp/covdata
|
||||
mkdir /tmp/covdata
|
||||
|
||||
# run server in background
|
||||
GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass --iam-dir /tmp/gw posix /tmp/gw &
|
||||
GW_PID=$!
|
||||
|
||||
# wait a second for server to start up
|
||||
sleep 1
|
||||
|
||||
# check if server is still running
|
||||
if ! kill -0 $GW_PID; then
|
||||
echo "server no longer running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# run tests
|
||||
# full flow tests
|
||||
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow; then
|
||||
echo "full flow tests failed"
|
||||
kill $GW_PID
|
||||
exit 1
|
||||
fi
|
||||
# posix tests
|
||||
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 posix; then
|
||||
echo "posix tests failed"
|
||||
kill $GW_PID
|
||||
exit 1
|
||||
fi
|
||||
# iam tests
|
||||
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 iam; then
|
||||
echo "iam tests failed"
|
||||
kill $GW_PID
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# kill off server
|
||||
kill $GW_PID
|
||||
exit 0
|
||||
|
||||
# if the above binary was built with -cover enabled (make testbin),
|
||||
# then the following can be used for code coverage reports:
|
||||
# go tool covdata percent -i=/tmp/covdata
|
||||
# go tool covdata textfmt -i=/tmp/covdata -o profile.txt
|
||||
# go tool cover -html=profile.txt
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
)
|
||||
|
||||
type S3AdminRouter struct{}
|
||||
|
||||
func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
|
||||
controller := controllers.NewAdminController(iam, be)
|
||||
|
||||
// CreateUser admin api
|
||||
app.Patch("/create-user", controller.CreateUser)
|
||||
|
||||
// DeleteUsers admin api
|
||||
app.Patch("/delete-user", controller.DeleteUser)
|
||||
|
||||
// ListUsers admin api
|
||||
app.Patch("/list-users", controller.ListUsers)
|
||||
|
||||
// ChangeBucketOwner admin api
|
||||
app.Patch("/change-bucket-owner", controller.ChangeBucketOwner)
|
||||
|
||||
// ListBucketsAndOwners admin api
|
||||
app.Patch("/list-buckets", controller.ListBuckets)
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
)
|
||||
|
||||
type S3AdminServer struct {
|
||||
app *fiber.App
|
||||
backend backend.Backend
|
||||
router *S3AdminRouter
|
||||
port string
|
||||
cert *tls.Certificate
|
||||
}
|
||||
|
||||
func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
|
||||
server := &S3AdminServer{
|
||||
app: app,
|
||||
backend: be,
|
||||
router: new(S3AdminRouter),
|
||||
port: port,
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(server)
|
||||
}
|
||||
|
||||
// Logging middlewares
|
||||
app.Use(logger.New())
|
||||
app.Use(middlewares.DecodeURL(nil))
|
||||
|
||||
// Authentication middlewares
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
|
||||
app.Use(middlewares.VerifyMD5Body(nil))
|
||||
app.Use(middlewares.AclParser(be, nil))
|
||||
|
||||
server.router.Init(app, be, iam)
|
||||
|
||||
return server
|
||||
}
|
||||
|
||||
type AdminOpt func(s *S3AdminServer)
|
||||
|
||||
func WithAdminSrvTLS(cert tls.Certificate) AdminOpt {
|
||||
return func(s *S3AdminServer) { s.cert = &cert }
|
||||
}
|
||||
|
||||
func (sa *S3AdminServer) Serve() (err error) {
|
||||
if sa.cert != nil {
|
||||
return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
|
||||
}
|
||||
return sa.app.Listen(sa.port)
|
||||
}
|
||||
@@ -1,122 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
)
|
||||
|
||||
type AdminController struct {
|
||||
iam auth.IAMService
|
||||
be backend.Backend
|
||||
}
|
||||
|
||||
func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
|
||||
return AdminController{iam: iam, be: be}
|
||||
}
|
||||
|
||||
func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
var usr auth.Account
|
||||
err := json.Unmarshal(ctx.Body(), &usr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse request body: %w", err)
|
||||
}
|
||||
|
||||
if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
|
||||
return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
|
||||
}
|
||||
|
||||
err = c.iam.CreateAccount(usr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create user: %w", err)
|
||||
}
|
||||
|
||||
return ctx.SendString("The user has been created successfully")
|
||||
}
|
||||
|
||||
func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
|
||||
access := ctx.Query("access")
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
|
||||
err := c.iam.DeleteUserAccount(access)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ctx.SendString("The user has been deleted successfully")
|
||||
}
|
||||
|
||||
func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
accs, err := c.iam.ListUserAccounts()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ctx.JSON(accs)
|
||||
}
|
||||
|
||||
func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
owner := ctx.Query("owner")
|
||||
bucket := ctx.Query("bucket")
|
||||
|
||||
accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(accs) > 0 {
|
||||
return fmt.Errorf("user specified as the new bucket owner does not exist")
|
||||
}
|
||||
|
||||
err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ctx.Status(201).SendString("Bucket owner has been updated successfully")
|
||||
}
|
||||
|
||||
func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
if acct.Role != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
|
||||
buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ctx.JSON(buckets)
|
||||
}
|
||||