Compare commits

..

1 Commits
v0.7 ... assets

Author SHA1 Message Date
Ben McClelland
17b42c369a add wiki assets 2023-06-10 19:49:40 -07:00
83 changed files with 26 additions and 20616 deletions

View File

@@ -1,27 +0,0 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: bug
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior.
**Expected behavior**
A clear and concise description of what you expected to happen.
**Server Version**
output of
```
./versitygw -version
uname -a
```
**Additional context**
Describe s3 client and version if applicable.

View File

@@ -1,14 +0,0 @@
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: enhancement
assignees: ''
---
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Additional context**
Add any other context or screenshots about the feature request here.

View File

@@ -1,10 +0,0 @@
version: 2
updates:
- package-ecosystem: "gomod"
directory: "/"
schedule:
interval: "weekly"
groups:
dev-dependencies:
patterns:
- "*"

View File

@@ -1,30 +0,0 @@
name: functional tests
on: pull_request
jobs:
build:
name: RunTests
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 'stable'
id: go
- name: Get Dependencies
run: |
go get -v -t -d ./...
- name: Build and Run
run: |
make testbin
./runtests.sh
- name: Coverage Report
run: |
go tool covdata percent -i=/tmp/covdata

View File

@@ -1,38 +0,0 @@
name: general
on: pull_request
jobs:
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 'stable'
id: go
- name: Verify all files pass gofmt formatting
run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then gofmt -s -d .; exit 1; fi
- name: Get dependencies
run: |
go get -v -t -d ./...
- name: Build
run: make
- name: Test
run: go test -coverprofile profile.txt -race -v -timeout 30s -tags=github ./...
- name: Install govulncheck
run: go install golang.org/x/vuln/cmd/govulncheck@latest
shell: bash
- name: Run govulncheck
run: govulncheck ./...
shell: bash

View File

@@ -1,31 +0,0 @@
name: goreleaser
on:
push:
# run only against tags
tags:
- '*'
permissions:
contents: write
# packages: write
# issues: write
jobs:
goreleaser:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- run: git fetch --force --tags
- uses: actions/setup-go@v4
with:
go-version: stable
- uses: goreleaser/goreleaser-action@v4
with:
distribution: goreleaser
version: latest
args: release --clean
env:
GITHUB_TOKEN: ${{ secrets.TOKEN }}

View File

@@ -1,22 +0,0 @@
name: staticcheck
on: pull_request
jobs:
build:
name: Check
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 'stable'
id: go
- name: "staticcheck"
uses: dominikh/staticcheck-action@v1.3.0
with:
install-go: false

17
.gitignore vendored
View File

@@ -7,8 +7,6 @@
*.dll
*.so
*.dylib
cmd/versitygw/versitygw
/versitygw
# Test binary, built with `go test -c`
*.test
@@ -21,18 +19,3 @@ cmd/versitygw/versitygw
# Go workspace file
go.work
# ignore IntelliJ directories
.idea
# auto generated VERSION file
VERSION
# build output
/versitygw.spec
*.tar
*.tar.gz
**/rand.data
/profile.txt
dist/

View File

@@ -1,51 +0,0 @@
before:
hooks:
- go mod tidy
builds:
- goos:
- linux
- darwin
# windows is untested, we can start doing windows releases
# if someone is interested in taking on testing
# - windows
main: ./cmd/versitygw
binary: ./cmd/versitygw
id: versitygw
goarch:
- amd64
- arm64
ldflags:
- -X=main.Build={{.Commit}} -X=main.BuildTime={{.Date}} -X=main.Version={{.Version}}
archives:
- format: tar.gz
# this name template makes the OS and Arch compatible with the results of uname.
name_template: >-
{{ .ProjectName }}_
{{- title .Os }}_
{{- if eq .Arch "amd64" }}x86_64
{{- else if eq .Arch "386" }}i386
{{- else }}{{ .Arch }}{{ end }}
{{- if .Arm }}v{{ .Arm }}{{ end }}
# use zip for windows archives
format_overrides:
- goos: windows
format: zip
checksum:
name_template: 'checksums.txt'
snapshot:
name_template: "{{ incpatch .Version }}-next"
changelog:
sort: asc
filters:
exclude:
- '^docs:'
- '^test:'
- '^Merge '
# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
# vim: set ts=2 sw=2 tw=0 fo=cnqoj

View File

@@ -1,128 +0,0 @@
# Contributor Covenant Code of Conduct
## Our Pledge
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, religion, or sexual identity
and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to a positive environment for our
community include:
* Demonstrating empathy and kindness toward other people
* Being respectful of differing opinions, viewpoints, and experiences
* Giving and gracefully accepting constructive feedback
* Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
* Focusing on what is best not just for us as individuals, but for the
overall community
Examples of unacceptable behavior include:
* The use of sexualized language or imagery, and sexual attention or
advances of any kind
* Trolling, insulting or derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or email
address, without their explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Enforcement Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
versitygw@versity.com.
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series
of actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or
permanent ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within
the community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.0, available at
https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
Community Impact Guidelines were inspired by [Mozilla's code of conduct
enforcement ladder](https://github.com/mozilla/diversity).
[homepage]: https://www.contributor-covenant.org
For answers to common questions about this code of conduct, see the FAQ at
https://www.contributor-covenant.org/faq. Translations are available at
https://www.contributor-covenant.org/translations.

View File

@@ -1,76 +0,0 @@
# Copyright 2023 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Go parameters
GOCMD=go
GOBUILD=$(GOCMD) build
GOCLEAN=$(GOCMD) clean
GOTEST=$(GOCMD) test
BIN=versitygw
VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
BUILD := $(shell git rev-parse --short HEAD || echo release-rpm)
TIME := `date -u '+%Y-%m-%d_%I:%M:%S%p'`
LDFLAGS=-ldflags "-X=main.Build=$(BUILD) -X=main.BuildTime=$(TIME) -X=main.Version=$(VERSION)"
all: build
build: $(BIN)
.PHONY: $(BIN)
$(BIN):
$(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go
testbin:
$(GOBUILD) $(LDFLAGS) -o $(BIN) -cover -race cmd/$(BIN)/*.go
.PHONY: test
test:
$(GOTEST) ./...
.PHONY: check
check:
# note this requires staticcheck be in your PATH:
# export PATH=$PATH:~/go/bin
# go install honnef.co/go/tools/cmd/staticcheck@latest
staticcheck ./...
golint ./...
gofmt -s -l .
.PHONY: clean
clean:
$(GOCLEAN)
.PHONY: cleanall
cleanall: clean
rm -f $(BIN)
rm -f versitygw-*.tar
rm -f versitygw-*.tar.gz
rm -f versitygw.spec
%.spec: %.spec.in
sed -e 's/@@VERSION@@/$(VERSION)/g' < $< > $@+
mv $@+ $@
TARFILE = $(BIN)-$(VERSION).tar
dist: $(BIN).spec
echo $(VERSION) >VERSION
git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
@ tar rf $(TARFILE) --transform="s@\(.*\)@$(BIN)-$(VERSION)/\1@" $(BIN).spec VERSION
rm -f VERSION
rm -f $(BIN).spec
gzip -f $(TARFILE)

2
NOTICE
View File

@@ -1,2 +0,0 @@
versitygw - Versity S3 Gateway
Copyright 2023 Versity Software

View File

@@ -1,67 +0,0 @@
# The Versity S3 Gateway:<br/>A High-Performance S3 Translation Service
<picture>
<source media="(prefers-color-scheme: dark)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo-white.svg">
<source media="(prefers-color-scheme: light)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo.svg">
<a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
</picture>
[![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)
**Current status:** Beta: Most clients functional, work in progress for more test coverage. Issue reports welcome.
See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
* Share filesystem directory via S3 protocol
* Simple to deploy S3 server with a single command
* Protocol compatibility allows common access to files via posix or S3
Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage and Versitys open source ScoutFS filesystem.
The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateways stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance.
The S3 HTTP(S) server and routing is implemented using the [Fiber](https://gofiber.io) web framework. This framework is actively developed with a focus on performance. S3 API compatibility leverages the official [aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) whenever possible for maximum service compatibility with AWS S3.
## Getting Started
See the [Quickstart](https://github.com/versity/versitygw/wiki/Quickstart) documentation.
### Run the gateway with posix backend:
```
mkdir /tmp/vgw
ROOT_ACCESS_KEY="testuser" ROOT_SECRET_KEY="secret" ./versitygw --port :10000 posix /tmp/vgw
```
This will enable an S3 server on the current host listening on port 10000 and hosting the directory `/tmp/vgw`.
To get the usage output, run the following:
```
./versitygw --help
```
The command format is
```
versitygw [global options] command [command options] [arguments...]
```
The global options are specified before the backend type and the backend options are specified after.
***
#### Versity gives you clarity and control over your archival storage, so you can allocate more resources to your core mission.
### Contact
![versity logo](https://www.versity.com/wp-content/uploads/2022/12/cropped-android-chrome-512x512-1-32x32.png)
info@versity.com <br />
+1 844 726 8826
### @versitysoftware
[![linkedin](https://github.com/versity/versitygw/blob/assets/assets/linkedin.jpg)](https://www.linkedin.com/company/versity/) &nbsp;
[![twitter](https://github.com/versity/versitygw/blob/assets/assets/twitter.jpg)](https://twitter.com/VersitySoftware) &nbsp;
[![facebook](https://github.com/versity/versitygw/blob/assets/assets/facebook.jpg)](https://www.facebook.com/versitysoftware) &nbsp;
[![instagram](https://github.com/versity/versitygw/blob/assets/assets/instagram.jpg)](https://www.instagram.com/versitysoftware/) &nbsp;

BIN
assets/facebook.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 470 B

BIN
assets/instagram.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 461 B

BIN
assets/linkedin.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 391 B

13
assets/logo-white.svg Normal file
View File

@@ -0,0 +1,13 @@
<svg width="209" height="73" viewBox="0 0 209 73" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M47.427 30.0292C42.767 30.9134 40.4568 30.3528 40.2354 29.406C40.1625 29.0919 40.2887 28.6812 40.6122 28.1848C42.2518 25.6629 47.9677 22.1097 54.5119 19.5417C57.6668 18.3011 60.7686 17.3735 63.4818 16.8593C68.1376 15.976 70.4445 16.5363 70.6646 17.4827C70.7388 17.7976 70.6123 18.2078 70.2897 18.7048C68.6525 21.2235 62.9369 24.7783 56.3908 27.3497C53.2389 28.5879 50.1396 29.5145 47.4284 30.0287L47.427 30.0292Z" fill="white"/>
<path d="M48.4866 39.9975C42.4383 40.4446 38.4817 39.0018 38.7133 37.4258C38.763 37.0777 39.0232 36.6989 39.4845 36.301C41.0086 34.9865 44.833 33.4548 49.9427 32.9982C50.1102 32.9836 50.2764 32.9697 50.4411 32.9574C56.4783 32.5107 60.4292 33.9536 60.1996 35.5285C60.1487 35.8758 59.8879 36.2549 59.4247 36.6544C57.8987 37.9699 54.0743 39.5016 48.9726 39.9577C48.8092 39.9718 48.648 39.9852 48.4871 39.997L48.4866 39.9975Z" fill="white"/>
<path d="M44.5318 47.6969C44.5093 44.8455 42.0643 42.5515 39.0706 42.5727C37.1309 42.5866 35.4376 43.57 34.4912 45.0354L34.4904 45.0349L34.4755 45.0595C34.4417 45.1127 34.4089 45.1664 34.3773 45.2211L29.6958 52.93L10.1292 20.2803C8.6447 17.8035 5.33413 16.9425 2.73347 18.3555C0.192505 19.737 -0.72811 22.7797 0.616915 25.2285L0.615815 25.2295L0.643322 25.2748C0.666427 25.3162 0.688702 25.3579 0.712632 25.3987C0.736837 25.4396 0.763518 25.4786 0.788548 25.5179L28.8811 72.4019L28.8822 72.3956C29.031 72.6689 29.3281 72.8577 29.673 72.8577C29.9979 72.8577 30.2801 72.6909 30.4363 72.4433L30.4374 72.4467L43.8764 50.2044L43.8731 50.2028C44.2991 49.4586 44.5387 48.6041 44.5318 47.6969Z" fill="white"/>
<path d="M69.6803 0.677278C68.2968 -1.32435 60.4824 1.26118 52.2266 6.45006C43.9703 11.641 38.3998 17.4709 39.7834 19.472C41.1669 21.4739 48.9818 18.8889 57.2376 13.6979C65.4929 8.50826 71.0644 2.67916 69.6803 0.677278Z" fill="white"/>
<path d="M108.898 20.9902L101.884 39.4324H101.798L94.2404 20.9902H87.6453L99.6694 51.7645H103.261L115.498 20.9902H108.898Z" fill="white"/>
<path d="M118.544 38.6442C118.626 38.1404 118.765 37.6502 118.962 37.1708C119.155 36.6966 119.415 36.2812 119.733 35.921C120.056 35.5652 120.444 35.2711 120.903 35.0454C121.363 34.8191 121.884 34.7061 122.468 34.7061C123.024 34.7061 123.523 34.8191 123.97 35.0454C124.415 35.2711 124.807 35.5704 125.141 35.9406C125.473 36.3138 125.741 36.7284 125.932 37.1912C126.127 37.6563 126.269 38.1404 126.35 38.6442H118.544ZM129.504 33.9109C128.711 32.956 127.701 32.2004 126.479 31.6458C125.252 31.0894 123.803 30.8105 122.135 30.8105C120.464 30.8105 119.009 31.0807 117.772 31.6236C116.534 32.1708 115.504 32.9168 114.681 33.8708C113.86 34.8261 113.242 35.9458 112.823 37.233C112.405 38.5185 112.197 39.9027 112.197 41.3869C112.197 42.8712 112.433 44.238 112.907 45.4838C113.38 46.7301 114.055 47.8124 114.933 48.726C115.807 49.6369 116.88 50.3485 118.147 50.8501C119.415 51.3556 120.838 51.6087 122.427 51.6087C124.6 51.6087 126.509 51.0785 128.166 50.0223C129.822 48.9665 131.014 47.4522 131.737 45.4838L126.35 44.6495C125.96 45.4181 125.454 46.0528 124.827 46.5552C124.2 47.0607 123.399 47.3139 122.427 47.3139C121.676 47.3139 121.036 47.1525 120.503 46.8397C119.975 46.5226 119.559 46.1185 119.256 45.6122C118.948 45.1158 118.722 44.5568 118.587 43.94C118.445 43.3197 118.376 42.7098 118.376 42.1055H131.863V41.4282C131.863 39.9705 131.668 38.5907 131.277 37.2935C130.887 35.9915 130.298 34.8648 129.504 33.9109Z" fill="white"/>
<path d="M145.732 30.8105C144.645 30.8105 143.674 31.0424 142.81 31.5039C141.947 31.9694 141.223 32.6245 140.638 33.4759H140.557V31.4487H134.71V50.9689H140.557V42.1821C140.557 41.4413 140.597 40.684 140.68 39.9127C140.765 39.1471 140.959 38.4511 141.265 37.8294C141.572 37.2039 142.023 36.7028 142.621 36.3178C143.222 35.9345 144.021 35.7392 145.025 35.7392C145.915 35.7392 146.721 35.9806 147.448 36.4565L148.155 31.2864C147.765 31.1546 147.367 31.0424 146.966 30.9489C146.562 30.8562 146.152 30.8105 145.732 30.8105Z" fill="white"/>
<path d="M162.394 40.9323C161.99 40.4929 161.537 40.1267 161.033 39.8387C160.538 39.5464 160.021 39.2923 159.492 39.0792C158.962 38.8669 158.447 38.679 157.948 38.505C157.444 38.3323 156.993 38.1483 156.591 37.9503C156.187 37.7498 155.868 37.5314 155.631 37.2935C155.395 37.0529 155.273 36.7497 155.273 36.3773C155.273 35.9001 155.477 35.5025 155.883 35.1837C156.283 34.8661 156.722 34.7056 157.195 34.7056C157.779 34.7056 158.339 34.8396 158.865 35.1063C159.396 35.3707 159.868 35.6883 160.286 36.0598L162.789 32.8403C161.955 32.1482 160.919 31.6401 159.681 31.3069C158.44 30.9754 157.28 30.8105 156.193 30.8105C155.219 30.8105 154.336 30.9706 153.539 31.2864C152.751 31.6062 152.067 32.0425 151.496 32.6002C150.925 33.1574 150.479 33.8269 150.16 34.6086C149.839 35.3929 149.68 36.2312 149.68 37.1321C149.68 37.9273 149.799 38.5985 150.034 39.1405C150.272 39.6869 150.577 40.1549 150.952 40.5525C151.327 40.9519 151.765 41.2816 152.268 41.5483C152.772 41.8141 153.269 42.0511 153.772 42.263C154.273 42.4761 154.775 42.6636 155.273 42.8381C155.776 43.0112 156.216 43.2044 156.591 43.4167C156.965 43.6281 157.271 43.8738 157.509 44.1522C157.743 44.432 157.861 44.7821 157.861 45.2032C157.861 45.8396 157.634 46.3547 157.175 46.7371C156.716 47.1194 156.167 47.3139 155.525 47.3139C154.661 47.3139 153.861 47.0868 153.124 46.6348C152.39 46.1863 151.739 45.6552 151.181 45.0479L148.511 48.3444C149.399 49.3802 150.479 50.1841 151.747 50.7531C153.011 51.3234 154.342 51.6087 155.735 51.6087C156.819 51.6087 157.841 51.4508 158.805 51.1293C159.76 50.81 160.605 50.3555 161.329 49.7591C162.051 49.1635 162.63 48.4388 163.06 47.591C163.494 46.7414 163.709 45.7735 163.709 44.6886C163.709 43.8408 163.59 43.1117 163.352 42.4992C163.118 41.8915 162.797 41.369 162.394 40.9323Z" fill="white"/>
<path d="M169.093 22.4395C168.2 22.4395 167.45 22.7401 166.841 23.3317C166.233 23.9237 165.926 24.6332 165.926 25.458C165.926 26.3045 166.233 27.0171 166.841 27.5952C167.45 28.1777 168.2 28.4674 169.093 28.4674C169.98 28.4674 170.731 28.1777 171.339 27.5952C171.951 27.0171 172.257 26.3045 172.257 25.458C172.257 24.6332 171.951 23.9237 171.339 23.3317C170.731 22.7401 169.98 22.4395 169.093 22.4395Z" fill="white"/>
<path d="M172.014 31.4492H166.167V50.9694H172.014V31.4492Z" fill="white"/>
<path d="M202.482 31.4492L197.425 41.9607L192.336 31.4492H185.418H184.587H184.212V22.9521H178.364V31.4492H174.405V36.139H178.364V50.9693H184.212V36.139H188.688L194.422 47.9882L189.519 57.6672H195.949L209 31.4492H202.482Z" fill="white"/>
</svg>

After

Width:  |  Height:  |  Size: 6.4 KiB

13
assets/logo.svg Normal file
View File

@@ -0,0 +1,13 @@
<svg width="150" height="53" viewBox="0 0 150 53" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M33.903 21.4665C30.5718 22.0985 28.9204 21.6977 28.7621 21.021C28.71 20.7965 28.8003 20.5028 29.0315 20.148C30.2036 18.3452 34.2896 15.8052 38.9676 13.9695C41.2229 13.0827 43.4402 12.4196 45.3797 12.052C48.7079 11.4206 50.357 11.8211 50.5143 12.4977C50.5673 12.7228 50.4769 13.016 50.2463 13.3713C49.076 15.1718 44.9902 17.7129 40.3107 19.551C38.0576 20.4362 35.8421 21.0985 33.904 21.4661L33.903 21.4665Z" fill="#191B2A"/>
<path d="M34.6603 28.592C30.3367 28.9116 27.5083 27.8802 27.6739 26.7536C27.7095 26.5048 27.8955 26.234 28.2252 25.9495C29.3147 25.0099 32.0485 23.915 35.7012 23.5886C35.8209 23.5781 35.9397 23.5682 36.0575 23.5594C40.3731 23.2401 43.1974 24.2715 43.0332 25.3973C42.9969 25.6456 42.8105 25.9166 42.4794 26.2021C41.3885 27.1426 38.6547 28.2375 35.0077 28.5635C34.8909 28.5736 34.7757 28.5832 34.6607 28.5916L34.6603 28.592Z" fill="#191B2A"/>
<path d="M31.8333 34.096C31.8172 32.0577 30.0694 30.4178 27.9294 30.433C26.5428 30.4429 25.3324 31.1459 24.6558 32.1934L24.6552 32.193L24.6446 32.2106C24.6204 32.2487 24.597 32.287 24.5744 32.3262L21.2279 37.8368L7.24078 14.4974C6.17961 12.7269 3.81307 12.1114 1.95401 13.1214C0.137611 14.109 -0.520485 16.2841 0.440998 18.0346L0.440212 18.0353L0.459875 18.0677C0.476391 18.0973 0.492314 18.1271 0.509421 18.1563C0.526723 18.1855 0.545796 18.2134 0.563688 18.2415L20.6455 51.7562L20.6463 51.7517C20.7527 51.947 20.965 52.082 21.2116 52.082C21.4438 52.082 21.6455 51.9628 21.7572 51.7858L21.758 51.7882L31.3648 35.8884L31.3624 35.8873C31.667 35.3553 31.8382 34.7444 31.8333 34.096Z" fill="#191B2A"/>
<path d="M49.8105 0.484148C48.8215 -0.9467 43.2354 0.90155 37.3338 4.61078C31.4318 8.32151 27.4498 12.489 28.4388 13.9194C29.4278 15.3505 35.0143 13.5026 40.9159 9.79187C46.8171 6.08208 50.7999 1.91518 49.8105 0.484148Z" fill="#191B2A"/>
<path d="M77.8452 15.0049L72.8313 28.1882H72.7699L67.3671 15.0049H62.6526L71.248 37.0037H73.8157L82.5626 15.0049H77.8452Z" fill="#191B2A"/>
<path d="M84.7403 27.625C84.7988 27.2649 84.898 26.9144 85.0391 26.5717C85.1775 26.2328 85.363 25.9358 85.5906 25.6783C85.8211 25.424 86.0986 25.2138 86.4268 25.0524C86.7556 24.8907 87.1281 24.8098 87.5454 24.8098C87.9428 24.8098 88.2996 24.8907 88.619 25.0524C88.9373 25.2138 89.2178 25.4277 89.4565 25.6923C89.6938 25.9591 89.8848 26.2555 90.0216 26.5863C90.1614 26.9188 90.2623 27.2649 90.3207 27.625H84.7403ZM92.575 24.2414C92.0085 23.5588 91.2866 23.0187 90.4128 22.6222C89.5358 22.2245 88.5001 22.0251 87.3074 22.0251C86.113 22.0251 85.073 22.2183 84.1885 22.6063C83.3034 22.9975 82.5674 23.5308 81.9787 24.2128C81.3923 24.8956 80.9502 25.6961 80.6504 26.6162C80.352 27.5351 80.2031 28.5246 80.2031 29.5856C80.2031 30.6466 80.3723 31.6237 80.7108 32.5143C81.0488 33.4052 81.5314 34.1788 82.1593 34.8319C82.7839 35.483 83.5512 35.9918 84.4566 36.3503C85.363 36.7116 86.3801 36.8926 87.516 36.8926C89.0695 36.8926 90.434 36.5135 91.6183 35.7585C92.8022 35.0038 93.6544 33.9214 94.1716 32.5143L90.3207 31.9178C90.0419 32.4673 89.6798 32.921 89.2318 33.2801C88.7838 33.6415 88.2111 33.8225 87.516 33.8225C86.9795 33.8225 86.5218 33.7071 86.1411 33.4835C85.7636 33.2568 85.4658 32.9679 85.249 32.606C85.029 32.2512 84.8677 31.8516 84.7713 31.4106C84.6695 30.9672 84.6202 30.5312 84.6202 30.0993H94.2611V29.6151C94.2611 28.5731 94.1217 27.5867 93.8425 26.6594C93.5636 25.7287 93.1428 24.9233 92.575 24.2414Z" fill="#191B2A"/>
<path d="M104.175 22.0251C103.398 22.0251 102.704 22.1909 102.087 22.5208C101.47 22.8536 100.952 23.3219 100.534 23.9304H100.476V22.4813H96.2964V36.4352H100.476V30.154C100.476 29.6245 100.505 29.0831 100.564 28.5318C100.625 27.9845 100.763 27.4869 100.982 27.0425C101.202 26.5954 101.524 26.2372 101.952 25.962C102.381 25.688 102.952 25.5484 103.67 25.5484C104.306 25.5484 104.883 25.721 105.402 26.0611L105.908 22.3653C105.629 22.2711 105.345 22.1909 105.058 22.124C104.769 22.0578 104.476 22.0251 104.175 22.0251Z" fill="#191B2A"/>
<path d="M116.086 29.2606C115.797 28.9465 115.474 28.6847 115.114 28.4789C114.759 28.2699 114.39 28.0883 114.012 27.9359C113.633 27.7842 113.265 27.6498 112.908 27.5255C112.548 27.402 112.225 27.2705 111.938 27.129C111.649 26.9856 111.421 26.8295 111.252 26.6594C111.083 26.4875 110.996 26.2707 110.996 26.0045C110.996 25.6634 111.142 25.3792 111.432 25.1513C111.718 24.9242 112.031 24.8095 112.37 24.8095C112.787 24.8095 113.187 24.9053 113.563 25.0959C113.944 25.285 114.28 25.512 114.579 25.7775L116.369 23.4761C115.772 22.9813 115.032 22.6182 114.147 22.38C113.26 22.143 112.43 22.0251 111.653 22.0251C110.958 22.0251 110.326 22.1396 109.756 22.3653C109.193 22.5939 108.704 22.9058 108.296 23.3044C107.887 23.7028 107.569 24.1814 107.341 24.7402C107.111 25.3008 106.998 25.9001 106.998 26.5441C106.998 27.1125 107.083 27.5923 107.251 27.9798C107.421 28.3703 107.639 28.7049 107.907 28.9891C108.175 29.2746 108.489 29.5103 108.848 29.7009C109.208 29.8909 109.563 30.0604 109.923 30.2119C110.281 30.3642 110.64 30.4983 110.996 30.623C111.355 30.7467 111.67 30.8848 111.938 31.0365C112.206 31.1877 112.424 31.3634 112.594 31.5624C112.762 31.7623 112.846 32.0127 112.846 32.3137C112.846 32.7686 112.684 33.1368 112.355 33.4101C112.027 33.6835 111.635 33.8225 111.176 33.8225C110.559 33.8225 109.987 33.6601 109.46 33.337C108.935 33.0164 108.47 32.6368 108.071 32.2027L106.162 34.5591C106.797 35.2995 107.569 35.8742 108.475 36.2809C109.379 36.6886 110.331 36.8926 111.326 36.8926C112.101 36.8926 112.832 36.7797 113.521 36.5499C114.204 36.3217 114.807 35.9967 115.325 35.5704C115.841 35.1447 116.255 34.6266 116.562 34.0205C116.872 33.4132 117.026 32.7213 117.026 31.9458C117.026 31.3397 116.941 30.8186 116.771 30.3807C116.604 29.9463 116.375 29.5728 116.086 29.2606Z" fill="#191B2A"/>
<path d="M120.875 16.041C120.237 16.041 119.701 16.2559 119.265 16.6788C118.83 17.102 118.611 17.6092 118.611 18.1988C118.611 18.8039 118.83 19.3133 119.265 19.7266C119.701 20.1429 120.237 20.35 120.875 20.35C121.509 20.35 122.046 20.1429 122.481 19.7266C122.918 19.3133 123.137 18.8039 123.137 18.1988C123.137 17.6092 122.918 17.102 122.481 16.6788C122.046 16.2559 121.509 16.041 120.875 16.041Z" fill="#191B2A"/>
<path d="M122.963 22.4814H118.784V36.4353H122.963V22.4814Z" fill="#191B2A"/>
<path d="M144.743 22.4813L141.128 29.9954L137.49 22.4813H132.545H131.951H131.683V16.4072H127.503V22.4813H124.673V25.8338H127.503V36.4351H131.683V25.8338H134.882L138.982 34.3041L135.477 41.223H140.073L149.402 22.4813H144.743Z" fill="#191B2A"/>
</svg>

After

Width:  |  Height:  |  Size: 6.4 KiB

BIN
assets/scoutam_gateway.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 66 KiB

BIN
assets/twitter.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 448 B

BIN
assets/workflow.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 69 KiB

View File

@@ -1,264 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"encoding/json"
"fmt"
"os"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
)
type ACL struct {
ACL types.BucketCannedACL
Owner string
Grantees []Grantee
}
type Grantee struct {
Permission types.Permission
Access string
}
type GetBucketAclOutput struct {
Owner *types.Owner
AccessControlList AccessControlList
}
type AccessControlList struct {
Grants []types.Grant `xml:"Grant"`
}
type AccessControlPolicy struct {
AccessControlList AccessControlList `xml:"AccessControlList"`
Owner types.Owner
}
func ParseACL(data []byte) (ACL, error) {
if len(data) == 0 {
return ACL{}, nil
}
var acl ACL
if err := json.Unmarshal(data, &acl); err != nil {
return acl, fmt.Errorf("parse acl: %w", err)
}
return acl, nil
}
func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
var acl ACL
if err := json.Unmarshal(data, &acl); err != nil {
return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
}
grants := []types.Grant{}
for _, elem := range acl.Grantees {
acs := elem.Access
grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
}
return GetBucketAclOutput{
Owner: &types.Owner{
ID: &acl.Owner,
},
AccessControlList: AccessControlList{
Grants: grants,
},
}, nil
}
func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
if input == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
if acl.Owner != *input.AccessControlPolicy.Owner.ID {
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
}
// if the ACL is specified, set the ACL, else replace the grantees
if input.ACL != "" {
acl.ACL = input.ACL
acl.Grantees = []Grantee{}
} else {
grantees := []Grantee{}
accs := []string{}
if input.GrantRead != nil {
fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{}
if *input.GrantFullControl != "" {
fullControlList = splitUnique(*input.GrantFullControl, ",")
for _, str := range fullControlList {
grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
}
}
if *input.GrantRead != "" {
readList = splitUnique(*input.GrantRead, ",")
for _, str := range readList {
grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
}
}
if *input.GrantReadACP != "" {
readACPList = splitUnique(*input.GrantReadACP, ",")
for _, str := range readACPList {
grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
}
}
if *input.GrantWrite != "" {
writeList = splitUnique(*input.GrantWrite, ",")
for _, str := range writeList {
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
}
}
if *input.GrantWriteACP != "" {
writeACPList = splitUnique(*input.GrantWriteACP, ",")
for _, str := range writeACPList {
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
}
}
accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
} else {
cache := make(map[string]bool)
for _, grt := range input.AccessControlPolicy.Grants {
grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
if _, ok := cache[*grt.Grantee.ID]; !ok {
cache[*grt.Grantee.ID] = true
accs = append(accs, *grt.Grantee.ID)
}
}
}
// Check if the specified accounts exist
accList, err := CheckIfAccountsExist(accs, iam)
if err != nil {
return nil, err
}
if len(accList) > 0 {
return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
}
acl.Grantees = grantees
acl.ACL = ""
}
result, err := json.Marshal(acl)
if err != nil {
return nil, err
}
return result, nil
}
func CheckIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
result := []string{}
for _, acc := range accs {
_, err := iam.GetUserAccount(acc)
if err != nil && err != ErrNoSuchUser {
return nil, fmt.Errorf("check user account: %w", err)
}
if err == ErrNoSuchUser {
result = append(result, acc)
}
}
return result, nil
}
func splitUnique(s, divider string) []string {
elements := strings.Split(s, divider)
uniqueElements := make(map[string]bool)
result := make([]string, 0, len(elements))
for _, element := range elements {
if _, ok := uniqueElements[element]; !ok {
result = append(result, element)
uniqueElements[element] = true
}
}
return result
}
func VerifyACL(acl ACL, bucket, access string, permission types.Permission, isRoot bool) error {
if isRoot {
return nil
}
if acl.Owner == access {
return nil
}
if acl.ACL != "" {
if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
return nil
} else {
grantee := Grantee{Access: access, Permission: permission}
granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
isFound := false
for _, grt := range acl.Grantees {
if grt == grantee || grt == granteeFullCtrl {
isFound = true
break
}
}
if isFound {
return nil
}
}
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
func IsAdmin(access string, isRoot bool) error {
var data IAMConfig
if isRoot {
return nil
}
file, err := os.ReadFile("users.json")
if err != nil {
return fmt.Errorf("unable to read config file: %w", err)
}
if err := json.Unmarshal(file, &data); err != nil {
return err
}
acc, ok := data.AccessAccounts[access]
if !ok {
return fmt.Errorf("user does not exist")
}
if acc.Role == "admin" {
return nil
}
return s3err.GetAPIError(s3err.ErrAccessDenied)
}

View File

@@ -1,38 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"errors"
)
// Account is a gateway IAM account
type Account struct {
Access string `json:"access"`
Secret string `json:"secret"`
Role string `json:"role"`
}
// IAMService is the interface for all IAM service implementations
//
//go:generate moq -out ../s3api/controllers/iam_moq_test.go -pkg controllers . IAMService
type IAMService interface {
CreateAccount(access string, account Account) error
GetUserAccount(access string) (Account, error)
DeleteUserAccount(access string) error
ListUserAccounts() ([]Account, error)
}
var ErrNoSuchUser = errors.New("user not found")

View File

@@ -1,212 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"encoding/json"
"fmt"
"hash/crc32"
"sync"
)
// IAMServiceInternal manages the internal IAM service
type IAMServiceInternal struct {
storer Storer
mu sync.RWMutex
accts IAMConfig
serial uint32
}
// UpdateAcctFunc accepts the current data and returns the new data to be stored
type UpdateAcctFunc func([]byte) ([]byte, error)
// Storer is the interface to manage the peristent IAM data for the internal
// IAM service
type Storer interface {
InitIAM() error
GetIAM() ([]byte, error)
StoreIAM(UpdateAcctFunc) error
}
// IAMConfig stores all internal IAM accounts
type IAMConfig struct {
AccessAccounts map[string]Account `json:"accessAccounts"`
}
var _ IAMService = &IAMServiceInternal{}
// NewInternal creates a new instance for the Internal IAM service
func NewInternal(s Storer) (*IAMServiceInternal, error) {
i := &IAMServiceInternal{
storer: s,
}
err := i.updateCache()
if err != nil {
return nil, fmt.Errorf("refresh iam cache: %w", err)
}
return i, nil
}
// CreateAccount creates a new IAM account. Returns an error if the account
// already exists.
func (s *IAMServiceInternal) CreateAccount(access string, account Account) error {
s.mu.Lock()
defer s.mu.Unlock()
return s.storer.StoreIAM(func(data []byte) ([]byte, error) {
var conf IAMConfig
if len(data) > 0 {
if err := json.Unmarshal(data, &conf); err != nil {
return nil, fmt.Errorf("failed to parse iam: %w", err)
}
} else {
conf = IAMConfig{AccessAccounts: map[string]Account{}}
}
_, ok := conf.AccessAccounts[access]
if ok {
return nil, fmt.Errorf("account already exists")
}
conf.AccessAccounts[access] = account
b, err := json.Marshal(conf)
if err != nil {
return nil, fmt.Errorf("failed to serialize iam: %w", err)
}
s.accts = conf
return b, nil
})
}
// GetUserAccount retrieves account info for the requested user. Returns
// ErrNoSuchUser if the account does not exist.
func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
s.mu.RLock()
defer s.mu.RUnlock()
data, err := s.storer.GetIAM()
if err != nil {
return Account{}, fmt.Errorf("get iam data: %w", err)
}
serial := crc32.ChecksumIEEE(data)
if serial != s.serial {
s.mu.RUnlock()
err := s.updateCache()
s.mu.RLock()
if err != nil {
return Account{}, fmt.Errorf("refresh iam cache: %w", err)
}
}
acct, ok := s.accts.AccessAccounts[access]
if !ok {
return Account{}, ErrNoSuchUser
}
return acct, nil
}
// updateCache must be called with no locks held
func (s *IAMServiceInternal) updateCache() error {
s.mu.Lock()
defer s.mu.Unlock()
data, err := s.storer.GetIAM()
if err != nil {
return fmt.Errorf("get iam data: %w", err)
}
serial := crc32.ChecksumIEEE(data)
if len(data) > 0 {
if err := json.Unmarshal(data, &s.accts); err != nil {
return fmt.Errorf("failed to parse the config file: %w", err)
}
} else {
s.accts.AccessAccounts = make(map[string]Account)
}
s.serial = serial
return nil
}
// DeleteUserAccount deletes the specified user account. Does not check if
// account exists.
func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
s.mu.Lock()
defer s.mu.Unlock()
return s.storer.StoreIAM(func(data []byte) ([]byte, error) {
if len(data) == 0 {
// empty config, do nothing
return data, nil
}
var conf IAMConfig
if err := json.Unmarshal(data, &conf); err != nil {
return nil, fmt.Errorf("failed to parse iam: %w", err)
}
delete(conf.AccessAccounts, access)
b, err := json.Marshal(conf)
if err != nil {
return nil, fmt.Errorf("failed to serialize iam: %w", err)
}
s.accts = conf
return b, nil
})
}
// ListUserAccounts lists all the user accounts stored.
func (s *IAMServiceInternal) ListUserAccounts() (accs []Account, err error) {
s.mu.RLock()
defer s.mu.RUnlock()
data, err := s.storer.GetIAM()
if err != nil {
return []Account{}, fmt.Errorf("get iam data: %w", err)
}
serial := crc32.ChecksumIEEE(data)
if serial != s.serial {
s.mu.RUnlock()
err := s.updateCache()
s.mu.RLock()
if err != nil {
return []Account{}, fmt.Errorf("refresh iam cache: %w", err)
}
}
for access, usr := range s.accts.AccessAccounts {
accs = append(accs, Account{
Access: access,
Secret: usr.Secret,
Role: usr.Role,
})
}
return accs, nil
}

View File

@@ -1,184 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package backend
import (
"context"
"fmt"
"io"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
//go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
type Backend interface {
fmt.Stringer
Shutdown()
// bucket operations
ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
CreateBucket(context.Context, *s3.CreateBucketInput) error
PutBucketAcl(_ context.Context, bucket string, data []byte) error
DeleteBucket(context.Context, *s3.DeleteBucketInput) error
// multipart operations
CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
// standard object operations
PutObject(context.Context, *s3.PutObjectInput) (string, error)
HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
DeleteObject(context.Context, *s3.DeleteObjectInput) error
DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
// special case object operations
RestoreObject(context.Context, *s3.RestoreObjectInput) error
SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
// object tags operations
GetTags(_ context.Context, bucket, object string) (map[string]string, error)
SetTags(_ context.Context, bucket, object string, tags map[string]string) error
RemoveTags(_ context.Context, bucket, object string) error
// non AWS actions
ChangeBucketOwner(_ context.Context, bucket, newOwner string) error
ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error)
}
type BackendUnsupported struct{}
var _ Backend = &BackendUnsupported{}
func New() Backend {
return &BackendUnsupported{}
}
func (BackendUnsupported) Shutdown() {}
func (BackendUnsupported) String() string {
return "Unsupported"
}
func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
return s3response.DeleteObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
return s3response.SelectObjectContentResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetTags(_ context.Context, bucket, object string) (map[string]string, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) SetTags(_ context.Context, bucket, object string, tags map[string]string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) RemoveTags(_ context.Context, bucket, object string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ChangeBucketOwner(_ context.Context, bucket, newOwner string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) ListBucketsAndOwners(context.Context) ([]s3response.Bucket, error) {
return []s3response.Bucket{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}

View File

@@ -1,122 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package backend
import (
"crypto/md5"
"encoding/hex"
"fmt"
"io/fs"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
var (
// RFC3339TimeFormat RFC3339 time format
RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
)
func IsValidBucketName(name string) bool { return true }
type ByBucketName []s3response.ListAllMyBucketsEntry
func (d ByBucketName) Len() int { return len(d) }
func (d ByBucketName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d ByBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name }
type ByObjectName []types.Object
func (d ByObjectName) Len() int { return len(d) }
func (d ByObjectName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
func GetStringPtr(s string) *string {
return &s
}
func GetTimePtr(t time.Time) *time.Time {
return &t
}
var (
errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
)
// ParseRange parses input range header and returns startoffset, length, and
// error. If no endoffset specified, then length is set to -1.
func ParseRange(file fs.FileInfo, acceptRange string) (int64, int64, error) {
if acceptRange == "" {
return 0, file.Size(), nil
}
rangeKv := strings.Split(acceptRange, "=")
if len(rangeKv) < 2 {
return 0, 0, errInvalidRange
}
bRange := strings.Split(rangeKv[1], "-")
if len(bRange) < 1 || len(bRange) > 2 {
return 0, 0, errInvalidRange
}
startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
if err != nil {
return 0, 0, errInvalidRange
}
endOffset := int64(-1)
if len(bRange) == 1 || bRange[1] == "" {
return startOffset, endOffset, nil
}
endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
if err != nil {
return 0, 0, errInvalidRange
}
if endOffset < startOffset {
return 0, 0, errInvalidRange
}
return startOffset, endOffset - startOffset + 1, nil
}
func GetMultipartMD5(parts []types.CompletedPart) string {
var partsEtagBytes []byte
for _, part := range parts {
partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
}
s3MD5 := fmt.Sprintf("%s-%d", md5String(partsEtagBytes), len(parts))
return s3MD5
}
func getEtagBytes(etag string) []byte {
decode, err := hex.DecodeString(strings.ReplaceAll(etag, string('"'), ""))
if err != nil {
return []byte(etag)
}
return decode
}
func md5String(data []byte) string {
sum := md5.Sum(data)
return hex.EncodeToString(sum[:])
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,89 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package posix
import (
"crypto/sha256"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
)
type tmpfile struct {
f *os.File
bucket string
objname string
size int64
}
func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
// Create a temp file for upload while in progress (see link comments below).
err := os.MkdirAll(dir, 0700)
if err != nil {
return nil, fmt.Errorf("make temp dir: %w", err)
}
f, err := os.CreateTemp(dir,
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
if err != nil {
return nil, err
}
return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
}
func (tmp *tmpfile) link() error {
tempname := tmp.f.Name()
// cleanup in case anything goes wrong, if rename succeeds then
// this will no longer exist
defer os.Remove(tempname)
// We use Rename as the atomic operation for object puts. The upload is
// written to a temp file to not conflict with any other simultaneous
// uploads. The final operation is to move the temp file into place for
// the object. This ensures the object semantics of last upload completed
// wins and is not some combination of writes from simultaneous uploads.
objPath := filepath.Join(tmp.bucket, tmp.objname)
err := os.Remove(objPath)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove stale path: %w", err)
}
err = tmp.f.Close()
if err != nil {
return fmt.Errorf("close tmpfile: %w", err)
}
err = os.Rename(tempname, objPath)
if err != nil {
return fmt.Errorf("rename tmpfile: %w", err)
}
return nil
}
func (tmp *tmpfile) Write(b []byte) (int, error) {
if int64(len(b)) > tmp.size {
return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
}
n, err := tmp.f.Write(b)
tmp.size -= int64(n)
return n, err
}
func (tmp *tmpfile) cleanup() {
tmp.f.Close()
}

View File

@@ -1,164 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package posix
import (
"crypto/sha256"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strconv"
"syscall"
"golang.org/x/sys/unix"
)
const procfddir = "/proc/self/fd"
type tmpfile struct {
f *os.File
bucket string
objname string
isOTmp bool
size int64
}
func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
// This can help reduce contention within the namespace (parent directories),
// etc. And will auto cleanup the inode on close if we never link this
// file descriptor into the namespace.
// Not all filesystems support this, so fallback to CreateTemp for when
// this is not supported.
fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
if err != nil {
// O_TMPFILE not supported, try fallback
err := os.MkdirAll(dir, 0700)
if err != nil {
return nil, fmt.Errorf("make temp dir: %w", err)
}
f, err := os.CreateTemp(dir,
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
if err != nil {
return nil, err
}
tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
// falloc is best effort, its fine if this fails
if size > 0 {
tmp.falloc()
}
return tmp, nil
}
// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
// later to link file into namespace
f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
// falloc is best effort, its fine if this fails
if size > 0 {
tmp.falloc()
}
return tmp, nil
}
func (tmp *tmpfile) falloc() error {
err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
if err != nil {
return fmt.Errorf("fallocate: %v", err)
}
return nil
}
func (tmp *tmpfile) link() error {
// We use Linkat/Rename as the atomic operation for object puts. The
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
// with any other simultaneous uploads. The final operation is to move the
// temp file into place for the object. This ensures the object semantics
// of last upload completed wins and is not some combination of writes
// from simultaneous uploads.
objPath := filepath.Join(tmp.bucket, tmp.objname)
err := os.Remove(objPath)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove stale path: %w", err)
}
if !tmp.isOTmp {
// O_TMPFILE not suported, use fallback
return tmp.fallbackLink()
}
procdir, err := os.Open(procfddir)
if err != nil {
return fmt.Errorf("open proc dir: %w", err)
}
defer procdir.Close()
dir, err := os.Open(filepath.Dir(objPath))
if err != nil {
return fmt.Errorf("open parent dir: %w", err)
}
defer dir.Close()
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
if err != nil {
return fmt.Errorf("link tmpfile (%q in %q): %w",
filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
}
err = tmp.f.Close()
if err != nil {
return fmt.Errorf("close tmpfile: %w", err)
}
return nil
}
func (tmp *tmpfile) fallbackLink() error {
tempname := tmp.f.Name()
// cleanup in case anything goes wrong, if rename succeeds then
// this will no longer exist
defer os.Remove(tempname)
err := tmp.f.Close()
if err != nil {
return fmt.Errorf("close tmpfile: %w", err)
}
objPath := filepath.Join(tmp.bucket, tmp.objname)
err = os.Rename(tempname, objPath)
if err != nil {
return fmt.Errorf("rename tmpfile: %w", err)
}
return nil
}
func (tmp *tmpfile) Write(b []byte) (int, error) {
if int64(len(b)) > tmp.size {
return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
}
n, err := tmp.f.Write(b)
tmp.size -= int64(n)
return n, err
}
func (tmp *tmpfile) cleanup() {
tmp.f.Close()
}

View File

@@ -1,89 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package posix
import (
"crypto/sha256"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
)
type tmpfile struct {
f *os.File
bucket string
objname string
size int64
}
func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
// Create a temp file for upload while in progress (see link comments below).
err := os.MkdirAll(dir, 0700)
if err != nil {
return nil, fmt.Errorf("make temp dir: %w", err)
}
f, err := os.CreateTemp(dir,
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
if err != nil {
return nil, err
}
return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
}
func (tmp *tmpfile) link() error {
tempname := tmp.f.Name()
// cleanup in case anything goes wrong, if rename succeeds then
// this will no longer exist
defer os.Remove(tempname)
// We use Rename as the atomic operation for object puts. The upload is
// written to a temp file to not conflict with any other simultaneous
// uploads. The final operation is to move the temp file into place for
// the object. This ensures the object semantics of last upload completed
// wins and is not some combination of writes from simultaneous uploads.
objPath := filepath.Join(tmp.bucket, tmp.objname)
err := os.Remove(objPath)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove stale path: %w", err)
}
err = tmp.f.Close()
if err != nil {
return fmt.Errorf("close tmpfile: %w", err)
}
err = os.Rename(tempname, objPath)
if err != nil {
return fmt.Errorf("rename tmpfile: %w", err)
}
return nil
}
func (tmp *tmpfile) Write(b []byte) (int, error) {
if int64(len(b)) > tmp.size {
return 0, fmt.Errorf("write exceeds content length")
}
n, err := tmp.f.Write(b)
tmp.size -= int64(n)
return n, err
}
func (tmp *tmpfile) cleanup() {
tmp.f.Close()
}

View File

@@ -1,777 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package scoutfs
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/pkg/xattr"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/backend/posix"
"github.com/versity/versitygw/s3err"
)
type ScoutFS struct {
*posix.Posix
rootfd *os.File
rootdir string
// glaciermode enables the following behavior:
// GET object: if file offline, return invalid object state
// HEAD object: if file offline, set obj storage class to GLACIER
// if file offline and staging, x-amz-restore: ongoing-request="true"
// if file offline and not staging, x-amz-restore: ongoing-request="false"
// if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
// note: this expiry-date is not used but provided for client glacier compatibility
// ListObjects: if file offline, set obj storage class to GLACIER
// RestoreObject: add batch stage request to file
glaciermode bool
}
var _ backend.Backend = &ScoutFS{}
const (
metaTmpDir = ".sgwtmp"
metaTmpMultipartDir = metaTmpDir + "/multipart"
tagHdr = "X-Amz-Tagging"
emptyMD5 = "d41d8cd98f00b204e9800998ecf8427e"
etagkey = "user.etag"
)
var (
stageComplete = "ongoing-request=\"false\", expiry-date=\"Fri, 2 Dec 2050 00:00:00 GMT\""
stageInProgress = "true"
stageNotInProgress = "false"
)
const (
// ScoutFS special xattr types
systemPrefix = "scoutfs.hide."
onameAttr = systemPrefix + "objname"
flagskey = systemPrefix + "sam_flags"
stagecopykey = systemPrefix + "sam_stagereq"
)
const (
// ScoutAM Flags
// Staging - file requested stage
Staging uint64 = 1 << iota
// StageFail - all copies failed to stage
StageFail
// NoArchive - no archive copies of file should be made
NoArchive
// ExtCacheRequested means file policy requests Ext Cache
ExtCacheRequested
// ExtCacheDone means this file ext cache copy has been
// created already (and possibly pruned, so may not exist)
ExtCacheDone
)
// Option sets various options for scoutfs
type Option func(s *ScoutFS)
// WithGlacierEmulation sets glacier mode emulation
func WithGlacierEmulation() Option {
return func(s *ScoutFS) { s.glaciermode = true }
}
func (s *ScoutFS) Shutdown() {
s.Posix.Shutdown()
s.rootfd.Close()
_ = s.rootdir
}
func (*ScoutFS) String() string {
return "ScoutFS Gateway"
}
// CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
// ioctl to not have to read and copy the part data to the final object. This
// saves a read and write cycle for all mutlipart uploads.
func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
parts := input.MultipartUpload.Parts
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return nil, fmt.Errorf("stat bucket: %w", err)
}
sum, err := s.checkUploadIDExists(bucket, object, uploadID)
if err != nil {
return nil, err
}
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
// check all parts ok
last := len(parts) - 1
partsize := int64(0)
var totalsize int64
for i, p := range parts {
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))
fi, err := os.Lstat(partPath)
if err != nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
if i == 0 {
partsize = fi.Size()
}
totalsize += fi.Size()
// all parts except the last need to be the same size
if i < last && partsize != fi.Size() {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
// non-last part sizes need to be multiples of 4k for move blocks
// TODO: fallback to no move blocks if not 4k aligned?
if i == 0 && i < last && fi.Size()%4096 != 0 {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
b, err := xattr.Get(partPath, "user.etag")
etag := string(b)
if err != nil {
etag = ""
}
if etag != *parts[i].ETag {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
}
}
// use totalsize=0 because we wont be writing to the file, only moving
// extents around. so we dont want to fallocate this.
f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0)
if err != nil {
return nil, fmt.Errorf("open temp file: %w", err)
}
defer f.cleanup()
for _, p := range parts {
pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)))
if err != nil {
return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err)
}
// scoutfs move data is a metadata only operation that moves the data
// extent references from the source, appeding to the destination.
// this needs to be 4k aligned.
err = moveData(pf, f.f)
pf.Close()
if err != nil {
return nil, fmt.Errorf("move blocks part %v: %v", p.PartNumber, err)
}
}
userMetaData := make(map[string]string)
upiddir := filepath.Join(objdir, uploadID)
loadUserMetaData(upiddir, userMetaData)
objname := filepath.Join(bucket, object)
dir := filepath.Dir(objname)
if dir != "" {
if err = mkdirAll(dir, os.FileMode(0755), bucket, object); err != nil {
if err != nil {
return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
}
}
}
err = f.link()
if err != nil {
return nil, fmt.Errorf("link object in namespace: %w", err)
}
for k, v := range userMetaData {
err = xattr.Set(objname, "user."+k, []byte(v))
if err != nil {
// cleanup object if returning error
os.Remove(objname)
return nil, fmt.Errorf("set user attr %q: %w", k, err)
}
}
// Calculate s3 compatible md5sum for complete multipart.
s3MD5 := backend.GetMultipartMD5(parts)
err = xattr.Set(objname, "user.etag", []byte(s3MD5))
if err != nil {
// cleanup object if returning error
os.Remove(objname)
return nil, fmt.Errorf("set etag attr: %w", err)
}
// cleanup tmp dirs
os.RemoveAll(upiddir)
// use Remove for objdir in case there are still other uploads
// for same object name outstanding
os.Remove(objdir)
return &s3.CompleteMultipartUploadOutput{
Bucket: &bucket,
ETag: &s3MD5,
Key: &object,
}, nil
}
func (s *ScoutFS) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, error) {
sum := sha256.Sum256([]byte(object))
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
_, err := os.Stat(filepath.Join(objdir, uploadID))
if errors.Is(err, fs.ErrNotExist) {
return [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
if err != nil {
return [32]byte{}, fmt.Errorf("stat upload: %w", err)
}
return sum, nil
}
func loadUserMetaData(path string, m map[string]string) (contentType, contentEncoding string) {
ents, err := xattr.List(path)
if err != nil || len(ents) == 0 {
return
}
for _, e := range ents {
if !isValidMeta(e) {
continue
}
b, err := xattr.Get(path, e)
if err == syscall.ENODATA {
m[strings.TrimPrefix(e, "user.")] = ""
continue
}
if err != nil {
continue
}
m[strings.TrimPrefix(e, "user.")] = string(b)
}
b, err := xattr.Get(path, "user.content-type")
contentType = string(b)
if err != nil {
contentType = ""
}
if contentType != "" {
m["content-type"] = contentType
}
b, err = xattr.Get(path, "user.content-encoding")
contentEncoding = string(b)
if err != nil {
contentEncoding = ""
}
if contentEncoding != "" {
m["content-encoding"] = contentEncoding
}
return
}
func isValidMeta(val string) bool {
if strings.HasPrefix(val, "user.X-Amz-Meta") {
return true
}
if strings.EqualFold(val, "user.Expires") {
return true
}
return false
}
// mkdirAll is similar to os.MkdirAll but it will return ErrObjectParentIsFile
// when appropriate
func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
// Fast path: if we can tell whether path is a directory or file, stop with success or error.
dir, err := os.Stat(path)
if err == nil {
if dir.IsDir() {
return nil
}
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
}
// Slow path: make sure parent exists and then call Mkdir for path.
i := len(path)
for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
i--
}
j := i
for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
j--
}
if j > 1 {
// Create parent.
err = mkdirAll(path[:j-1], perm, bucket, object)
if err != nil {
return err
}
}
// Parent now exists; invoke Mkdir and use its result.
err = os.Mkdir(path, perm)
if err != nil {
// Handle arguments like "foo/." by
// double-checking that directory doesn't exist.
dir, err1 := os.Lstat(path)
if err1 == nil && dir.IsDir() {
return nil
}
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
}
return nil
}
func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
bucket := *input.Bucket
object := *input.Key
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return nil, fmt.Errorf("stat bucket: %w", err)
}
objPath := filepath.Join(bucket, object)
fi, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return nil, fmt.Errorf("stat object: %w", err)
}
userMetaData := make(map[string]string)
contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
b, err := xattr.Get(objPath, etagkey)
etag := string(b)
if err != nil {
etag = ""
}
stclass := types.StorageClassStandard
requestOngoing := ""
if s.glaciermode {
requestOngoing = stageComplete
// Check if there are any offline exents associated with this file.
// If so, we will set storage class to glacier.
st, err := statMore(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return nil, fmt.Errorf("stat more: %w", err)
}
if st.Offline_blocks != 0 {
stclass = types.StorageClassGlacier
requestOngoing = stageNotInProgress
ok, err := isStaging(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return nil, fmt.Errorf("check stage status: %w", err)
}
if ok {
requestOngoing = stageInProgress
}
}
}
return &s3.HeadObjectOutput{
ContentLength: fi.Size(),
ContentType: &contentType,
ContentEncoding: &contentEncoding,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
StorageClass: stclass,
Restore: &requestOngoing,
}, nil
}
func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
bucket := *input.Bucket
object := *input.Key
acceptRange := *input.Range
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return nil, fmt.Errorf("stat bucket: %w", err)
}
objPath := filepath.Join(bucket, object)
fi, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return nil, fmt.Errorf("stat object: %w", err)
}
startOffset, length, err := backend.ParseRange(fi, acceptRange)
if err != nil {
return nil, err
}
if length == -1 {
length = fi.Size() - startOffset + 1
}
if startOffset+length > fi.Size() {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
if s.glaciermode {
// Check if there are any offline exents associated with this file.
// If so, we will return the InvalidObjectState error.
st, err := statMore(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return nil, fmt.Errorf("stat more: %w", err)
}
if st.Offline_blocks != 0 {
return nil, s3err.GetAPIError(s3err.ErrInvalidObjectState)
}
}
f, err := os.Open(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return nil, fmt.Errorf("open object: %w", err)
}
defer f.Close()
rdr := io.NewSectionReader(f, startOffset, length)
_, err = io.Copy(writer, rdr)
if err != nil {
return nil, fmt.Errorf("copy data: %w", err)
}
userMetaData := make(map[string]string)
contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
b, err := xattr.Get(objPath, etagkey)
etag := string(b)
if err != nil {
etag = ""
}
tags, err := s.getXattrTags(bucket, object)
if err != nil {
return nil, fmt.Errorf("get object tags: %w", err)
}
return &s3.GetObjectOutput{
AcceptRanges: &acceptRange,
ContentLength: length,
ContentEncoding: &contentEncoding,
ContentType: &contentType,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
TagCount: int32(len(tags)),
StorageClass: types.StorageClassStandard,
}, nil
}
func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error) {
tags := make(map[string]string)
b, err := xattr.Get(filepath.Join(bucket, object), "user."+tagHdr)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if isNoAttr(err) {
return tags, nil
}
if err != nil {
return nil, fmt.Errorf("get tags: %w", err)
}
err = json.Unmarshal(b, &tags)
if err != nil {
return nil, fmt.Errorf("unmarshal tags: %w", err)
}
return tags, nil
}
func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
bucket := *input.Bucket
prefix := *input.Prefix
marker := *input.Marker
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return nil, fmt.Errorf("stat bucket: %w", err)
}
fileSystem := os.DirFS(bucket)
results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
s.fileToObj(bucket), []string{metaTmpDir})
if err != nil {
return nil, fmt.Errorf("walk %v: %w", bucket, err)
}
return &s3.ListObjectsOutput{
CommonPrefixes: results.CommonPrefixes,
Contents: results.Objects,
Delimiter: &delim,
IsTruncated: results.Truncated,
Marker: &marker,
MaxKeys: maxkeys,
Name: &bucket,
NextMarker: &results.NextMarker,
Prefix: &prefix,
}, nil
}
func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
bucket := *input.Bucket
prefix := *input.Prefix
marker := *input.ContinuationToken
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return nil, fmt.Errorf("stat bucket: %w", err)
}
fileSystem := os.DirFS(bucket)
results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
s.fileToObj(bucket), []string{metaTmpDir})
if err != nil {
return nil, fmt.Errorf("walk %v: %w", bucket, err)
}
return &s3.ListObjectsV2Output{
CommonPrefixes: results.CommonPrefixes,
Contents: results.Objects,
Delimiter: &delim,
IsTruncated: results.Truncated,
ContinuationToken: &marker,
MaxKeys: int32(maxkeys),
Name: &bucket,
NextContinuationToken: &results.NextMarker,
Prefix: &prefix,
}, nil
}
func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
return func(path string, d fs.DirEntry) (types.Object, error) {
objPath := filepath.Join(bucket, path)
if d.IsDir() {
// directory object only happens if directory empty
// check to see if this is a directory object by checking etag
etagBytes, err := xattr.Get(objPath, etagkey)
if isNoAttr(err) || errors.Is(err, fs.ErrNotExist) {
return types.Object{}, backend.ErrSkipObj
}
if err != nil {
return types.Object{}, fmt.Errorf("get etag: %w", err)
}
etag := string(etagBytes)
fi, err := d.Info()
if errors.Is(err, fs.ErrNotExist) {
return types.Object{}, backend.ErrSkipObj
}
if err != nil {
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
key := path + "/"
return types.Object{
ETag: &etag,
Key: &key,
LastModified: backend.GetTimePtr(fi.ModTime()),
}, nil
}
// file object, get object info and fill out object data
etagBytes, err := xattr.Get(objPath, etagkey)
if errors.Is(err, fs.ErrNotExist) {
return types.Object{}, backend.ErrSkipObj
}
if err != nil && !isNoAttr(err) {
return types.Object{}, fmt.Errorf("get etag: %w", err)
}
etag := string(etagBytes)
fi, err := d.Info()
if errors.Is(err, fs.ErrNotExist) {
return types.Object{}, backend.ErrSkipObj
}
if err != nil {
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
sc := types.ObjectStorageClassStandard
if s.glaciermode {
// Check if there are any offline exents associated with this file.
// If so, we will return the InvalidObjectState error.
st, err := statMore(objPath)
if errors.Is(err, fs.ErrNotExist) {
return types.Object{}, backend.ErrSkipObj
}
if err != nil {
return types.Object{}, fmt.Errorf("stat more: %w", err)
}
if st.Offline_blocks != 0 {
sc = types.ObjectStorageClassGlacier
}
}
return types.Object{
ETag: &etag,
Key: &path,
LastModified: backend.GetTimePtr(fi.ModTime()),
Size: fi.Size(),
StorageClass: sc,
}, nil
}
}
// RestoreObject will set stage request on file if offline and do nothing if
// file is online
func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) error {
bucket := *input.Bucket
object := *input.Key
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return fmt.Errorf("stat bucket: %w", err)
}
err = setStaging(filepath.Join(bucket, object))
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
return fmt.Errorf("stage object: %w", err)
}
return nil
}
func setStaging(objname string) error {
b, err := xattr.Get(objname, flagskey)
if err != nil && !isNoAttr(err) {
return err
}
var oldflags uint64
if !isNoAttr(err) {
err = json.Unmarshal(b, &oldflags)
if err != nil {
return err
}
}
newflags := oldflags | Staging
if newflags == oldflags {
// no flags change, just return
return nil
}
return fSetNewGlobalFlags(objname, newflags)
}
func isStaging(objname string) (bool, error) {
b, err := xattr.Get(objname, flagskey)
if err != nil && !isNoAttr(err) {
return false, err
}
var flags uint64
if !isNoAttr(err) {
err = json.Unmarshal(b, &flags)
if err != nil {
return false, err
}
}
return flags&Staging == Staging, nil
}
func fSetNewGlobalFlags(objname string, flags uint64) error {
b, err := json.Marshal(&flags)
if err != nil {
return err
}
return xattr.Set(objname, flagskey, b)
}
func isNoAttr(err error) bool {
if err == nil {
return false
}
xerr, ok := err.(*xattr.Error)
if ok && xerr.Err == xattr.ENOATTR {
return true
}
if err == syscall.ENODATA {
return true
}
return false
}

View File

@@ -1,209 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build linux && amd64
package scoutfs
import (
"crypto/sha256"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strconv"
"syscall"
"golang.org/x/sys/unix"
"github.com/versity/scoutfs-go"
"github.com/versity/versitygw/backend/posix"
)
func New(rootdir string, opts ...Option) (*ScoutFS, error) {
p, err := posix.New(rootdir)
if err != nil {
return nil, err
}
f, err := os.Open(rootdir)
if err != nil {
return nil, fmt.Errorf("open %v: %w", rootdir, err)
}
s := &ScoutFS{Posix: p, rootfd: f, rootdir: rootdir}
for _, opt := range opts {
opt(s)
}
return s, nil
}
const procfddir = "/proc/self/fd"
type tmpfile struct {
f *os.File
bucket string
objname string
isOTmp bool
size int64
}
func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
// This can help reduce contention within the namespace (parent directories),
// etc. And will auto cleanup the inode on close if we never link this
// file descriptor into the namespace.
// Not all filesystems support this, so fallback to CreateTemp for when
// this is not supported.
fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
if err != nil {
// O_TMPFILE not supported, try fallback
err := os.MkdirAll(dir, 0700)
if err != nil {
return nil, fmt.Errorf("make temp dir: %w", err)
}
f, err := os.CreateTemp(dir,
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
if err != nil {
return nil, err
}
tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
// falloc is best effort, its fine if this fails
if size > 0 {
tmp.falloc()
}
return tmp, nil
}
// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
// later to link file into namespace
f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
// falloc is best effort, its fine if this fails
if size > 0 {
tmp.falloc()
}
return tmp, nil
}
func (tmp *tmpfile) falloc() error {
err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
if err != nil {
return fmt.Errorf("fallocate: %v", err)
}
return nil
}
func (tmp *tmpfile) link() error {
// We use Linkat/Rename as the atomic operation for object puts. The
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
// with any other simultaneous uploads. The final operation is to move the
// temp file into place for the object. This ensures the object semantics
// of last upload completed wins and is not some combination of writes
// from simultaneous uploads.
objPath := filepath.Join(tmp.bucket, tmp.objname)
err := os.Remove(objPath)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove stale path: %w", err)
}
if !tmp.isOTmp {
// O_TMPFILE not suported, use fallback
return tmp.fallbackLink()
}
procdir, err := os.Open(procfddir)
if err != nil {
return fmt.Errorf("open proc dir: %w", err)
}
defer procdir.Close()
dir, err := os.Open(filepath.Dir(objPath))
if err != nil {
return fmt.Errorf("open parent dir: %w", err)
}
defer dir.Close()
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
if err != nil {
return fmt.Errorf("link tmpfile: %w", err)
}
err = tmp.f.Close()
if err != nil {
return fmt.Errorf("close tmpfile: %w", err)
}
return nil
}
func (tmp *tmpfile) fallbackLink() error {
tempname := tmp.f.Name()
// cleanup in case anything goes wrong, if rename succeeds then
// this will no longer exist
defer os.Remove(tempname)
err := tmp.f.Close()
if err != nil {
return fmt.Errorf("close tmpfile: %w", err)
}
objPath := filepath.Join(tmp.bucket, tmp.objname)
err = os.Rename(tempname, objPath)
if err != nil {
return fmt.Errorf("rename tmpfile: %w", err)
}
return nil
}
func (tmp *tmpfile) Write(b []byte) (int, error) {
if int64(len(b)) > tmp.size {
return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
}
n, err := tmp.f.Write(b)
tmp.size -= int64(n)
return n, err
}
func (tmp *tmpfile) cleanup() {
tmp.f.Close()
}
func moveData(from *os.File, to *os.File) error {
return scoutfs.MoveData(from, to)
}
func statMore(path string) (stat, error) {
st, err := scoutfs.StatMore(path)
if err != nil {
return stat{}, err
}
var s stat
s.Meta_seq = st.Meta_seq
s.Data_seq = st.Data_seq
s.Data_version = st.Data_version
s.Online_blocks = st.Online_blocks
s.Offline_blocks = st.Offline_blocks
s.Crtime_sec = st.Crtime_sec
s.Crtime_nsec = st.Crtime_nsec
return s, nil
}

View File

@@ -1,58 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build !(linux && amd64)
package scoutfs
import (
"errors"
"fmt"
"os"
)
func New(rootdir string, opts ...Option) (*ScoutFS, error) {
return nil, fmt.Errorf("scoutfs only available on linux")
}
type tmpfile struct {
f *os.File
}
var (
errNotSupported = errors.New("not supported")
)
func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
return nil, errNotSupported
}
func (tmp *tmpfile) link() error {
return errNotSupported
}
func (tmp *tmpfile) Write(b []byte) (int, error) {
return 0, errNotSupported
}
func (tmp *tmpfile) cleanup() {
}
func moveData(from *os.File, to *os.File) error {
return errNotSupported
}
func statMore(path string) (stat, error) {
return stat{}, errNotSupported
}

View File

@@ -1,25 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package scoutfs
type stat struct {
Meta_seq uint64
Data_seq uint64
Data_version uint64
Online_blocks uint64
Offline_blocks uint64
Crtime_sec uint64
Crtime_nsec uint32
}

View File

@@ -1,219 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package backend
import (
"errors"
"fmt"
"io/fs"
"os"
"sort"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
type WalkResults struct {
CommonPrefixes []types.CommonPrefix
Objects []types.Object
Truncated bool
NextMarker string
}
type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
var ErrSkipObj = errors.New("skip this object")
// Walk walks the supplied fs.FS and returns results compatible with list
// objects responses
func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
cpmap := make(map[string]struct{})
var objects []types.Object
var pastMarker bool
if marker == "" {
pastMarker = true
}
pastMax := max == 0
var newMarker string
var truncated bool
err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
// Ignore the root directory
if path == "." {
return nil
}
if contains(d.Name(), skipdirs) {
return fs.SkipDir
}
if pastMax {
if len(objects) != 0 {
newMarker = *objects[len(objects)-1].Key
truncated = true
}
return fs.SkipAll
}
if d.IsDir() {
// If prefix is defined and the directory does not match prefix,
// do not descend into the directory because nothing will
// match this prefix. Make sure to append the / at the end of
// directories since this is implied as a directory path name.
// If path is a prefix of prefix, then path could still be
// building to match. So only skip if path isn't a prefix of prefix
// and prefix isn't a prefix of path.
if prefix != "" &&
!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
return fs.SkipDir
}
// TODO: can we do better here rather than a second readdir
// per directory?
ents, err := fs.ReadDir(fileSystem, path)
if err != nil {
return fmt.Errorf("readdir %q: %w", path, err)
}
if len(ents) == 0 {
dirobj, err := getObj(path, d)
if err == ErrSkipObj {
return nil
}
if err != nil {
return fmt.Errorf("directory to object %q: %w", path, err)
}
objects = append(objects, dirobj)
}
return nil
}
if !pastMarker {
if path == marker {
pastMarker = true
}
return nil
}
// If object doesn't have prefix, don't include in results.
if prefix != "" && !strings.HasPrefix(path, prefix) {
return nil
}
if delimiter == "" {
// If no delimiter specified, then all files with matching
// prefix are included in results
obj, err := getObj(path, d)
if err == ErrSkipObj {
return nil
}
if err != nil {
return fmt.Errorf("file to object %q: %w", path, err)
}
objects = append(objects, obj)
if max > 0 && (len(objects)+len(cpmap)) == int(max) {
pastMax = true
}
return nil
}
// Since delimiter is specified, we only want results that
// do not contain the delimiter beyond the prefix. If the
// delimiter exists past the prefix, then the substring
// between the prefix and delimiter is part of common prefixes.
//
// For example:
// prefix = A/
// delimiter = /
// and objects:
// A/file
// A/B/file
// B/C
// would return:
// objects: A/file
// common prefix: A/B/
//
// Note: No objects are included past the common prefix since
// these are all rolled up into the common prefix.
// Note: The delimiter can be anything, so we have to operate on
// the full path without any assumptions on posix directory hierarchy
// here. Usually the delimiter will be "/", but thats not required.
suffix := strings.TrimPrefix(path, prefix)
before, _, found := strings.Cut(suffix, delimiter)
if !found {
obj, err := getObj(path, d)
if err == ErrSkipObj {
return nil
}
if err != nil {
return fmt.Errorf("file to object %q: %w", path, err)
}
objects = append(objects, obj)
if (len(objects) + len(cpmap)) == int(max) {
pastMax = true
}
return nil
}
// Common prefixes are a set, so should not have duplicates.
// These are abstractly a "directory", so need to include the
// delimiter at the end.
cpmap[prefix+before+delimiter] = struct{}{}
if (len(objects) + len(cpmap)) == int(max) {
pastMax = true
}
return nil
})
if err != nil {
return WalkResults{}, err
}
var commonPrefixStrings []string
for k := range cpmap {
commonPrefixStrings = append(commonPrefixStrings, k)
}
sort.Strings(commonPrefixStrings)
commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
for _, cp := range commonPrefixStrings {
pfx := cp
commonPrefixes = append(commonPrefixes, types.CommonPrefix{
Prefix: &pfx,
})
}
return WalkResults{
CommonPrefixes: commonPrefixes,
Objects: objects,
Truncated: truncated,
NextMarker: newMarker,
}, nil
}
func contains(a string, strs []string) bool {
for _, s := range strs {
if s == a {
return true
}
}
return false
}

View File

@@ -1,204 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package backend_test
import (
"crypto/md5"
"encoding/hex"
"fmt"
"io/fs"
"testing"
"testing/fstest"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/backend"
)
type walkTest struct {
fsys fs.FS
expected backend.WalkResults
getobj backend.GetObjFunc
}
func getObj(path string, d fs.DirEntry) (types.Object, error) {
if d.IsDir() {
etag := getMD5(path)
fi, err := d.Info()
if err != nil {
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
return types.Object{
ETag: &etag,
Key: &path,
LastModified: backend.GetTimePtr(fi.ModTime()),
}, nil
}
etag := getMD5(path)
fi, err := d.Info()
if err != nil {
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
return types.Object{
ETag: &etag,
Key: &path,
LastModified: backend.GetTimePtr(fi.ModTime()),
Size: fi.Size(),
}, nil
}
func getMD5(text string) string {
hash := md5.Sum([]byte(text))
return hex.EncodeToString(hash[:])
}
func TestWalk(t *testing.T) {
tests := []walkTest{
{
// test case from
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html
fsys: fstest.MapFS{
"sample.jpg": {},
"photos/2006/January/sample.jpg": {},
"photos/2006/February/sample2.jpg": {},
"photos/2006/February/sample3.jpg": {},
"photos/2006/February/sample4.jpg": {},
},
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetStringPtr("photos/"),
}},
Objects: []types.Object{{
Key: backend.GetStringPtr("sample.jpg"),
}},
},
getobj: getObj,
},
{
// test case single dir/single file
fsys: fstest.MapFS{
"test/file": {},
},
expected: backend.WalkResults{
CommonPrefixes: []types.CommonPrefix{{
Prefix: backend.GetStringPtr("test/"),
}},
Objects: []types.Object{},
},
getobj: getObj,
},
}
for _, tt := range tests {
res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
if err != nil {
t.Fatalf("walk: %v", err)
}
compareResults(res, tt.expected, t)
}
}
func compareResults(got, wanted backend.WalkResults, t *testing.T) {
if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
t.Errorf("unexpected common prefix, got %v wanted %v",
printCommonPrefixes(got.CommonPrefixes),
printCommonPrefixes(wanted.CommonPrefixes))
}
if !compareObjects(got.Objects, wanted.Objects) {
t.Errorf("unexpected object, got %v wanted %v",
printObjects(got.Objects),
printObjects(wanted.Objects))
}
}
func compareCommonPrefix(a, b []types.CommonPrefix) bool {
if len(a) == 0 && len(b) == 0 {
return true
}
if len(a) != len(b) {
return false
}
for _, cp := range a {
if containsCommonPrefix(cp, b) {
return true
}
}
return false
}
func containsCommonPrefix(c types.CommonPrefix, list []types.CommonPrefix) bool {
for _, cp := range list {
if *c.Prefix == *cp.Prefix {
return true
}
}
return false
}
func printCommonPrefixes(list []types.CommonPrefix) string {
res := "["
for _, cp := range list {
if res == "[" {
res = res + *cp.Prefix
} else {
res = res + ", " + *cp.Prefix
}
}
return res + "]"
}
func compareObjects(a, b []types.Object) bool {
if len(a) == 0 && len(b) == 0 {
return true
}
if len(a) != len(b) {
return false
}
for _, cp := range a {
if containsObject(cp, b) {
return true
}
}
return false
}
func containsObject(c types.Object, list []types.Object) bool {
for _, cp := range list {
if *c.Key == *cp.Key {
return true
}
}
return false
}
func printObjects(list []types.Object) string {
res := "["
for _, cp := range list {
if res == "[" {
res = res + *cp.Key
} else {
res = res + ", " + *cp.Key
}
}
return res + "]"
}

View File

@@ -1,384 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package main
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"text/tabwriter"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3response"
)
var (
adminAccess string
adminSecret string
adminEndpoint string
)
func adminCommand() *cli.Command {
return &cli.Command{
Name: "admin",
Usage: "admin CLI tool",
Description: `Admin CLI tool for interacting with admin APIs.`,
Subcommands: []*cli.Command{
{
Name: "create-user",
Usage: "Create a new user",
Action: createUser,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "access",
Usage: "access key id for the new user",
Required: true,
Aliases: []string{"a"},
},
&cli.StringFlag{
Name: "secret",
Usage: "secret access key for the new user",
Required: true,
Aliases: []string{"s"},
},
&cli.StringFlag{
Name: "role",
Usage: "role for the new user",
Required: true,
Aliases: []string{"r"},
},
},
},
{
Name: "delete-user",
Usage: "Delete a user",
Action: deleteUser,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "access",
Usage: "access key id of the user to be deleted",
Required: true,
Aliases: []string{"a"},
},
},
},
{
Name: "list-users",
Usage: "List all the gateway users",
Action: listUsers,
},
{
Name: "change-bucket-owner",
Usage: "Changes the bucket owner",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "bucket",
Usage: "the bucket name to change the owner",
Required: true,
Aliases: []string{"b"},
},
&cli.StringFlag{
Name: "owner",
Usage: "the user access key id, who should be the bucket owner",
Required: true,
Aliases: []string{"o"},
},
},
Action: changeBucketOwner,
},
{
Name: "list-buckets",
Usage: "Lists all the gateway buckets and owners.",
Action: listBuckets,
},
},
Flags: []cli.Flag{
// TODO: create a configuration file for this
&cli.StringFlag{
Name: "access",
Usage: "admin access key id",
EnvVars: []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"},
Aliases: []string{"a"},
Required: true,
Destination: &adminAccess,
},
&cli.StringFlag{
Name: "secret",
Usage: "admin secret access key",
EnvVars: []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"},
Aliases: []string{"s"},
Required: true,
Destination: &adminSecret,
},
&cli.StringFlag{
Name: "endpoint-url",
Usage: "admin apis endpoint url",
EnvVars: []string{"ADMIN_ENDPOINT_URL"},
Aliases: []string{"er"},
Required: true,
Destination: &adminEndpoint,
},
},
}
}
func createUser(ctx *cli.Context) error {
access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
if access == "" || secret == "" {
return fmt.Errorf("invalid input parameters for the new user")
}
if role != "admin" && role != "user" {
return fmt.Errorf("invalid input parameter for role")
}
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/create-user?access=%v&secret=%v&role=%v", adminEndpoint, access, secret, role), nil)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
defer resp.Body.Close()
fmt.Printf("%s\n", body)
return nil
}
func deleteUser(ctx *cli.Context) error {
access := ctx.String("access")
if access == "" {
return fmt.Errorf("invalid input parameter for the new user")
}
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/delete-user?access=%v", adminEndpoint, access), nil)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
defer resp.Body.Close()
fmt.Printf("%s\n", body)
return nil
}
func listUsers(ctx *cli.Context) error {
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-users", adminEndpoint), nil)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
defer resp.Body.Close()
var accs []auth.Account
if err := json.Unmarshal(body, &accs); err != nil {
return err
}
printAcctTable(accs)
return nil
}
const (
// account table formatting
minwidth int = 2 // minimal cell width including any padding
tabwidth int = 0 // width of tab characters (equivalent number of spaces)
padding int = 2 // padding added to a cell before computing its width
padchar byte = ' ' // ASCII char used for padding
flags uint = 0 // formatting control flags
)
func printAcctTable(accs []auth.Account) {
w := new(tabwriter.Writer)
w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
fmt.Fprintln(w, "Account\tRole")
fmt.Fprintln(w, "-------\t----")
for _, acc := range accs {
fmt.Fprintf(w, "%v\t%v\n", acc.Access, acc.Role)
}
fmt.Fprintln(w)
w.Flush()
}
func changeBucketOwner(ctx *cli.Context) error {
bucket, owner := ctx.String("bucket"), ctx.String("owner")
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", adminEndpoint, bucket, owner), nil)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
defer resp.Body.Close()
fmt.Println(string(body))
return nil
}
func printBuckets(buckets []s3response.Bucket) {
w := new(tabwriter.Writer)
w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
fmt.Fprintln(w, "Bucket\tOwner")
fmt.Fprintln(w, "-------\t----")
for _, acc := range buckets {
fmt.Fprintf(w, "%v\t%v\n", acc.Name, acc.Owner)
}
fmt.Fprintln(w)
w.Flush()
}
func listBuckets(ctx *cli.Context) error {
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", adminEndpoint), nil)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return fmt.Errorf(string(body))
}
var buckets []s3response.Bucket
if err := json.Unmarshal(body, &buckets); err != nil {
return err
}
printBuckets(buckets)
return nil
}

View File

@@ -1,345 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package main
import (
"context"
"crypto/tls"
"fmt"
"log"
"os"
"strconv"
"github.com/gofiber/fiber/v2"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3log"
)
var (
port, admPort string
rootUserAccess string
rootUserSecret string
region string
admCertFile, admKeyFile string
certFile, keyFile string
kafkaURL, kafkaTopic, kafkaKey string
natsURL, natsTopic string
logWebhookURL string
accessLog string
debug bool
)
var (
// Version is the latest tag (set within Makefile)
Version = "git"
// Build is the commit hash (set within Makefile)
Build = "norev"
// BuildTime is the date/time of build (set within Makefile)
BuildTime = "none"
)
func main() {
setupSignalHandler()
app := initApp()
app.Commands = []*cli.Command{
posixCommand(),
scoutfsCommand(),
adminCommand(),
testCommand(),
}
ctx, cancel := context.WithCancel(context.Background())
go func() {
<-sigDone
fmt.Fprintf(os.Stderr, "terminating signal caught, shutting down\n")
cancel()
}()
if err := app.RunContext(ctx, os.Args); err != nil {
log.Fatal(err)
}
}
func initApp() *cli.App {
return &cli.App{
Name: "versitygw",
Usage: "Start S3 gateway service with specified backend storage.",
Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
to access the supported backend storage as if it was a native S3 service.`,
Action: func(ctx *cli.Context) error {
return ctx.App.Command("help").Run(ctx)
},
Flags: initFlags(),
}
}
func initFlags() []cli.Flag {
return []cli.Flag{
&cli.BoolFlag{
Name: "version",
Usage: "list versitygw version",
Aliases: []string{"v"},
Action: func(*cli.Context, bool) error {
fmt.Println("Version :", Version)
fmt.Println("Build :", Build)
fmt.Println("BuildTime:", BuildTime)
os.Exit(0)
return nil
},
},
&cli.StringFlag{
Name: "port",
Usage: "gateway listen address <ip>:<port> or :<port>",
Value: ":7070",
Destination: &port,
Aliases: []string{"p"},
},
&cli.StringFlag{
Name: "access",
Usage: "root user access key",
EnvVars: []string{"ROOT_ACCESS_KEY_ID", "ROOT_ACCESS_KEY"},
Aliases: []string{"a"},
Destination: &rootUserAccess,
},
&cli.StringFlag{
Name: "secret",
Usage: "root user secret access key",
EnvVars: []string{"ROOT_SECRET_ACCESS_KEY", "ROOT_SECRET_KEY"},
Aliases: []string{"s"},
Destination: &rootUserSecret,
},
&cli.StringFlag{
Name: "region",
Usage: "s3 region string",
Value: "us-east-1",
Destination: &region,
Aliases: []string{"r"},
},
&cli.StringFlag{
Name: "cert",
Usage: "TLS cert file",
Destination: &certFile,
},
&cli.StringFlag{
Name: "key",
Usage: "TLS key file",
Destination: &keyFile,
},
&cli.StringFlag{
Name: "admin-port",
Usage: "gateway admin server listen address <ip>:<port> or :<port>",
Destination: &admPort,
Aliases: []string{"ap"},
},
&cli.StringFlag{
Name: "admin-cert",
Usage: "TLS cert file for admin server",
Destination: &admCertFile,
},
&cli.StringFlag{
Name: "admin-cert-key",
Usage: "TLS key file for admin server",
Destination: &admKeyFile,
},
&cli.BoolFlag{
Name: "debug",
Usage: "enable debug output",
Destination: &debug,
},
&cli.StringFlag{
Name: "access-log",
Usage: "enable server access logging to specified file",
EnvVars: []string{"LOGFILE"},
Destination: &accessLog,
},
&cli.StringFlag{
Name: "log-webhook-url",
Usage: "webhook url to send the audit logs",
EnvVars: []string{"WEBHOOK"},
Destination: &logWebhookURL,
},
&cli.StringFlag{
Name: "event-kafka-url",
Usage: "kafka server url to send the bucket notifications.",
Destination: &kafkaURL,
Aliases: []string{"eku"},
},
&cli.StringFlag{
Name: "event-kafka-topic",
Usage: "kafka server pub-sub topic to send the bucket notifications to",
Destination: &kafkaTopic,
Aliases: []string{"ekt"},
},
&cli.StringFlag{
Name: "event-kafka-key",
Usage: "kafka server put-sub topic key to send the bucket notifications to",
Destination: &kafkaKey,
Aliases: []string{"ekk"},
},
&cli.StringFlag{
Name: "event-nats-url",
Usage: "nats server url to send the bucket notifications",
Destination: &natsURL,
Aliases: []string{"enu"},
},
&cli.StringFlag{
Name: "event-nats-topic",
Usage: "nats server pub-sub topic to send the bucket notifications to",
Destination: &natsTopic,
Aliases: []string{"ent"},
},
}
}
func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error {
// int32 max for 32 bit arch
blimit := int64(2*1024*1024*1024 - 1)
if strconv.IntSize > 32 {
// 5GB max for 64 bit arch
blimit = int64(5 * 1024 * 1024 * 1024)
}
app := fiber.New(fiber.Config{
AppName: "versitygw",
ServerHeader: "VERSITYGW",
BodyLimit: int(blimit),
})
var opts []s3api.Option
if certFile != "" || keyFile != "" {
if certFile == "" {
return fmt.Errorf("TLS key specified without cert file")
}
if keyFile == "" {
return fmt.Errorf("TLS cert specified without key file")
}
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
return fmt.Errorf("tls: load certs: %v", err)
}
opts = append(opts, s3api.WithTLS(cert))
}
if debug {
opts = append(opts, s3api.WithDebug())
}
if admPort == "" {
opts = append(opts, s3api.WithAdminServer())
}
admApp := fiber.New(fiber.Config{
AppName: "versitygw",
ServerHeader: "VERSITYGW",
})
var admOpts []s3api.AdminOpt
if admCertFile != "" || admKeyFile != "" {
if admCertFile == "" {
return fmt.Errorf("TLS key specified without cert file")
}
if admKeyFile == "" {
return fmt.Errorf("TLS cert specified without key file")
}
cert, err := tls.LoadX509KeyPair(admCertFile, admKeyFile)
if err != nil {
return fmt.Errorf("tls: load certs: %v", err)
}
admOpts = append(admOpts, s3api.WithAdminSrvTLS(cert))
}
err := s.InitIAM()
if err != nil {
return fmt.Errorf("init iam: %w", err)
}
iam, err := auth.NewInternal(s)
if err != nil {
return fmt.Errorf("setup internal iam service: %w", err)
}
logger, err := s3log.InitLogger(&s3log.LogConfig{
LogFile: accessLog,
WebhookURL: logWebhookURL,
})
if err != nil {
return fmt.Errorf("setup logger: %w", err)
}
evSender, err := s3event.InitEventSender(&s3event.EventConfig{
KafkaURL: kafkaURL,
KafkaTopic: kafkaTopic,
KafkaTopicKey: kafkaKey,
NatsURL: natsURL,
NatsTopic: natsTopic,
})
if err != nil {
return fmt.Errorf("unable to connect to the message broker: %w", err)
}
srv, err := s3api.New(app, be, middlewares.RootUserConfig{
Access: rootUserAccess,
Secret: rootUserSecret,
}, port, region, iam, logger, evSender, opts...)
if err != nil {
return fmt.Errorf("init gateway: %v", err)
}
admSrv := s3api.NewAdminServer(admApp, be, middlewares.RootUserConfig{Access: rootUserAccess, Secret: rootUserSecret}, admPort, region, iam, admOpts...)
c := make(chan error, 2)
go func() { c <- srv.Serve() }()
if admPort != "" {
go func() { c <- admSrv.Serve() }()
}
// for/select blocks until shutdown
Loop:
for {
select {
case <-ctx.Done():
err = ctx.Err()
break Loop
case err = <-c:
break Loop
case <-sigHup:
if logger != nil {
err = logger.HangUp()
if err != nil {
err = fmt.Errorf("HUP logger: %w", err)
break Loop
}
}
}
}
be.Shutdown()
if logger != nil {
lerr := logger.Shutdown()
if lerr != nil {
fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", lerr)
}
}
return err
}

View File

@@ -1,53 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package main
import (
"fmt"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/posix"
)
func posixCommand() *cli.Command {
return &cli.Command{
Name: "posix",
Usage: "posix filesystem storage backend",
Description: `Any posix filesystem that supports extended attributes. The top level
directory for the gateway must be provided. All sub directories of the
top level directory are treated as buckets, and all files/directories
below the "bucket directory" are treated as the objects. The object
name is split on "/" separator to translate to posix storage.
For example:
top level: /mnt/fs/gwroot
bucket: mybucket
object: a/b/c/myobject
will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
Action: runPosix,
}
}
func runPosix(ctx *cli.Context) error {
if ctx.NArg() == 0 {
return fmt.Errorf("no directory provided for operation")
}
be, err := posix.New(ctx.Args().Get(0))
if err != nil {
return fmt.Errorf("init posix: %v", err)
}
return runGateway(ctx, be, be)
}

View File

@@ -1,73 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package main
import (
"fmt"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/scoutfs"
)
var (
glacier bool
)
func scoutfsCommand() *cli.Command {
return &cli.Command{
Name: "scoutfs",
Usage: "scoutfs filesystem storage backend",
Description: `Support for ScoutFS.
The top level directory for the gateway must be provided. All sub directories
of the top level directory are treated as buckets, and all files/directories
below the "bucket directory" are treated as the objects. The object name is
split on "/" separator to translate to posix storage.
For example:
top level: /mnt/fs/gwroot
bucket: mybucket
object: a/b/c/myobject
will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
ScoutFS contains optimizations for multipart uploads using extent
move interfaces as well as support for tiered filesystems.`,
Action: runScoutfs,
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "glacier",
Usage: "enable glacier emulation mode",
Aliases: []string{"g"},
Destination: &glacier,
},
},
}
}
func runScoutfs(ctx *cli.Context) error {
if ctx.NArg() == 0 {
return fmt.Errorf("no directory provided for operation")
}
var opts []scoutfs.Option
if glacier {
opts = append(opts, scoutfs.WithGlacierEmulation())
}
be, err := scoutfs.New(ctx.Args().Get(0), opts...)
if err != nil {
return fmt.Errorf("init scoutfs: %v", err)
}
return runGateway(ctx, be, be)
}

View File

@@ -1,44 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package main
import (
"fmt"
"os"
"os/signal"
"syscall"
)
var (
sigDone = make(chan bool, 1)
sigHup = make(chan bool, 1)
)
func setupSignalHandler() {
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
go func() {
for sig := range sigs {
fmt.Fprintf(os.Stderr, "caught signal %v\n", sig)
switch sig {
case syscall.SIGINT, syscall.SIGTERM:
sigDone <- true
case syscall.SIGHUP:
sigHup <- true
}
}
}()
}

View File

@@ -1,203 +0,0 @@
package main
import (
"fmt"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/integration"
)
var (
awsID string
awsSecret string
endpoint string
prefix string
dstBucket string
partSize int64
objSize int64
concurrency int
files int
upload bool
download bool
pathStyle bool
checksumDisable bool
)
func testCommand() *cli.Command {
return &cli.Command{
Name: "test",
Usage: "Client side testing command for the gateway",
Description: `The testing CLI is used to test group of versitygw actions.
It also includes some performance and stress testing`,
Subcommands: initTestCommands(),
Flags: initTestFlags(),
}
}
func initTestFlags() []cli.Flag {
return []cli.Flag{
&cli.StringFlag{
Name: "access",
Usage: "aws user access key",
EnvVars: []string{"AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY"},
Aliases: []string{"a"},
Destination: &awsID,
},
&cli.StringFlag{
Name: "secret",
Usage: "aws user secret access key",
EnvVars: []string{"AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY"},
Aliases: []string{"s"},
Destination: &awsSecret,
},
&cli.StringFlag{
Name: "endpoint",
Usage: "s3 server endpoint",
Destination: &endpoint,
Aliases: []string{"e"},
},
&cli.BoolFlag{
Name: "debug",
Usage: "enable debug mode",
Aliases: []string{"d"},
Destination: &debug,
},
}
}
func initTestCommands() []*cli.Command {
return []*cli.Command{
{
Name: "full-flow",
Usage: "Tests the full flow of gateway.",
Description: `Runs all the available tests to test the full flow of the gateway.`,
Action: getAction(integration.TestFullFlow),
},
{
Name: "bench",
Usage: "Runs download/upload performance test on the gateway",
Description: `Uploads/downloads some number(specified by flags) of files with some capacity(bytes).
Logs the results to the console`,
Flags: []cli.Flag{
&cli.IntFlag{
Name: "files",
Usage: "Number of objects to read/write",
Value: 1,
Destination: &files,
},
&cli.Int64Flag{
Name: "objsize",
Usage: "Uploading object size",
Value: 0,
Destination: &objSize,
},
&cli.StringFlag{
Name: "prefix",
Usage: "Object name prefix",
Destination: &prefix,
},
&cli.BoolFlag{
Name: "upload",
Usage: "Upload data to the gateway",
Value: false,
Destination: &upload,
},
&cli.BoolFlag{
Name: "download",
Usage: "Download data to the gateway",
Value: false,
Destination: &download,
},
&cli.StringFlag{
Name: "bucket",
Usage: "Destination bucket name to read/write data",
Destination: &dstBucket,
},
&cli.Int64Flag{
Name: "partSize",
Usage: "Upload/download size per thread",
Value: 64 * 1024 * 1024,
Destination: &partSize,
},
&cli.IntFlag{
Name: "concurrency",
Usage: "Upload/download threads per object",
Value: 1,
Destination: &concurrency,
},
&cli.BoolFlag{
Name: "pathStyle",
Usage: "Use Pathstyle bucket addressing",
Value: false,
Destination: &pathStyle,
},
&cli.BoolFlag{
Name: "checksumDis",
Usage: "Disable server checksum",
Value: false,
Destination: &checksumDisable,
},
},
Action: func(ctx *cli.Context) error {
if upload && download {
return fmt.Errorf("must only specify one of upload or download")
}
if !upload && !download {
return fmt.Errorf("must specify one of upload or download")
}
if dstBucket == "" {
return fmt.Errorf("must specify bucket")
}
opts := []integration.Option{
integration.WithAccess(awsID),
integration.WithSecret(awsSecret),
integration.WithRegion(region),
integration.WithEndpoint(endpoint),
integration.WithConcurrency(concurrency),
integration.WithPartSize(partSize),
}
if debug {
opts = append(opts, integration.WithDebug())
}
if pathStyle {
opts = append(opts, integration.WithPathStyle())
}
if checksumDisable {
opts = append(opts, integration.WithDisableChecksum())
}
s3conf := integration.NewS3Conf(opts...)
return integration.TestPerformance(s3conf, upload, download, files, objSize, dstBucket, prefix)
},
},
}
}
type testFunc func(*integration.S3Conf)
func getAction(tf testFunc) func(*cli.Context) error {
return func(ctx *cli.Context) error {
opts := []integration.Option{
integration.WithAccess(awsID),
integration.WithSecret(awsSecret),
integration.WithRegion(region),
integration.WithEndpoint(endpoint),
}
if debug {
opts = append(opts, integration.WithDebug())
}
s := integration.NewS3Conf(opts...)
tf(s)
fmt.Println()
fmt.Println("RAN:", integration.RunCount, "PASS:", integration.PassCount, "FAIL:", integration.FailCount)
if integration.FailCount > 0 {
return fmt.Errorf("test failed with %v errors", integration.FailCount)
}
return nil
}
}

60
go.mod
View File

@@ -1,60 +0,0 @@
module github.com/versity/versitygw
go 1.20
require (
github.com/aws/aws-sdk-go-v2 v1.21.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5
github.com/aws/smithy-go v1.14.2
github.com/gofiber/fiber/v2 v2.49.2
github.com/google/uuid v1.3.1
github.com/nats-io/nats.go v1.29.0
github.com/pkg/xattr v0.4.9
github.com/segmentio/kafka-go v0.4.42
github.com/urfave/cli/v2 v2.25.7
github.com/valyala/fasthttp v1.50.0
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
golang.org/x/sys v0.12.0
)
require (
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.14.0 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.16.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/nats-io/nats-server/v2 v2.9.20 // indirect
github.com/nats-io/nkeys v0.4.4 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/stretchr/testify v1.8.1 // indirect
golang.org/x/crypto v0.11.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
)
require (
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.40
github.com/aws/aws-sdk-go-v2/credentials v1.13.38
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.84
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
)

167
go.sum
View File

@@ -1,167 +0,0 @@
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc=
github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM=
github.com/aws/aws-sdk-go-v2/config v1.18.40 h1:dbu1llI/nTIL+r6sYHMeVLl99DM8J8/o1I4EPurnhLg=
github.com/aws/aws-sdk-go-v2/config v1.18.40/go.mod h1:JjrCZQwSPGCoZRQzKHyZNNueaKO+kFaEy2sR6mCzd90=
github.com/aws/aws-sdk-go-v2/credentials v1.13.38 h1:gDAuCdVlA4lmmgQhvpZlscwicloCqH44vkxLklGkQLA=
github.com/aws/aws-sdk-go-v2/credentials v1.13.38/go.mod h1:sD4G/Ybgp6s89mWIES3Xn97CsRLpxvz9uVSdv0UxY8I=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.84 h1:LENrVcqnWTyI8fbIUCvxAMe+fXbREIaXzcR8WPwco1U=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.84/go.mod h1:LHxCiYAStsgps4srke7HujyADd504MSkNXjLpOtICTc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk=
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM=
github.com/aws/aws-sdk-go-v2/service/sso v1.14.0 h1:AR/hlTsCyk1CwlyKnPFvIMvnONydRjDDRT9OGb0i+/g=
github.com/aws/aws-sdk-go-v2/service/sso v1.14.0/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.16.0 h1:vbgiXuhtn49+erlPrgIvQ+J32rg1HseaPf8lEpKbkxQ=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.16.0/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4=
github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 h1:s4bioTgjSFRwOoyEFzAVCmFmoowBgjTR8gkrF/sQ4wk=
github.com/aws/aws-sdk-go-v2/service/sts v1.22.0/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU=
github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ=
github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gofiber/fiber/v2 v2.49.2 h1:ONEN3/Vc+dUCxxDgZZwpqvhISgHqb+bu+isBiEyKEQs=
github.com/gofiber/fiber/v2 v2.49.2/go.mod h1:gNsKnyrmfEWFpJxQAV0qvW6l70K1dZGno12oLtukcts=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4=
github.com/nats-io/nats-server/v2 v2.9.20 h1:bt1dW6xsL1hWWwv7Hovm+EJt5L6iplyqlgEFkoEUk0k=
github.com/nats-io/nats-server/v2 v2.9.20/go.mod h1:aTb/xtLCGKhfTFLxP591CMWfkdgBmcUUSkiSOe5A3gw=
github.com/nats-io/nats.go v1.29.0 h1:dSXZ+SZeGyTdHVYeXimeq12FsIpb9dM8CJ2IZFiHcyE=
github.com/nats-io/nats.go v1.29.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc=
github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA=
github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU=
github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -1,210 +0,0 @@
package integration
func TestAuthentication(s *S3Conf) {
Authentication_empty_auth_header(s)
Authentication_invalid_auth_header(s)
Authentication_unsupported_signature_version(s)
Authentication_malformed_credentials(s)
Authentication_malformed_credentials_invalid_parts(s)
Authentication_credentials_terminated_string(s)
Authentication_credentials_incorrect_service(s)
Authentication_credentials_incorrect_region(s)
Authentication_credentials_invalid_date(s)
Authentication_credentials_future_date(s)
Authentication_credentials_past_date(s)
Authentication_credentials_non_existing_access_key(s)
Authentication_invalid_signed_headers(s)
Authentication_missing_date_header(s)
Authentication_invalid_date_header(s)
Authentication_date_mismatch(s)
Authentication_incorrect_payload_hash(s)
Authentication_incorrect_md5(s)
Authentication_signature_error_incorrect_secret_key(s)
}
func TestCreateBucket(s *S3Conf) {
CreateBucket_invalid_bucket_name(s)
CreateBucket_existing_bucket(s)
CreateBucket_as_user(s)
CreateDeleteBucket_success(s)
}
func TestHeadBucket(s *S3Conf) {
HeadBucket_non_existing_bucket(s)
HeadBucket_success(s)
}
func TestListBuckets(s *S3Conf) {
ListBuckets_as_user(s)
ListBuckets_as_admin(s)
ListBuckets_success(s)
}
func TestDeleteBucket(s *S3Conf) {
DeleteBucket_non_existing_bucket(s)
DeleteBucket_non_empty_bucket(s)
}
func TestPutObject(s *S3Conf) {
PutObject_non_existing_bucket(s)
PutObject_special_chars(s)
PutObject_existing_dir_obj(s)
PutObject_obj_parent_is_file(s)
PutObject_success(s)
}
func TestHeadObject(s *S3Conf) {
HeadObject_non_existing_object(s)
HeadObject_success(s)
}
func TestGetObject(s *S3Conf) {
GetObject_non_existing_key(s)
GetObject_invalid_ranges(s)
GetObject_with_meta(s)
GetObject_success(s)
GetObject_by_range_success(s)
}
func TestListObjects(s *S3Conf) {
ListObjects_non_existing_bucket(s)
ListObjects_with_prefix(s)
ListObject_truncated(s)
ListObjects_invalid_max_keys(s)
ListObjects_max_keys_0(s)
ListObjects_delimiter(s)
}
func TestDeleteObject(s *S3Conf) {
DeleteObject_non_existing_object(s)
DeleteObject_success(s)
}
func TestDeleteObjects(s *S3Conf) {
DeleteObjects_empty_input(s)
DeleteObjects_non_existing_objects(s)
DeleteObjects_success(s)
}
func TestCopyObject(s *S3Conf) {
CopyObject_non_existing_dst_bucket(s)
CopyObject_not_owned_source_bucket(s)
CopyObject_copy_to_itself(s)
CopyObject_success(s)
}
func TestPutObjectTagging(s *S3Conf) {
PutObjectTagging_non_existing_object(s)
PutObjectTagging_success(s)
}
func TestGetObjectTagging(s *S3Conf) {
GetObjectTagging_non_existing_object(s)
GetObjectTagging_success(s)
}
func TestDeleteObjectTagging(s *S3Conf) {
DeleteObjectTagging_non_existing_object(s)
DeleteObjectTagging_success(s)
}
func TestCreateMultipartUpload(s *S3Conf) {
CreateMultipartUpload_non_existing_bucket(s)
CreateMultipartUpload_success(s)
}
func TestUploadPart(s *S3Conf) {
UploadPart_non_existing_bucket(s)
UploadPart_invalid_part_number(s)
UploadPart_non_existing_key(s)
UploadPart_non_existing_mp_upload(s)
UploadPart_success(s)
}
func TestUploadPartCopy(s *S3Conf) {
UploadPartCopy_non_existing_bucket(s)
UploadPartCopy_incorrect_uploadId(s)
UploadPartCopy_incorrect_object_key(s)
UploadPartCopy_invalid_part_number(s)
UploadPartCopy_invalid_copy_source(s)
UploadPartCopy_non_existing_source_bucket(s)
UploadPartCopy_non_existing_source_object_key(s)
UploadPartCopy_success(s)
UploadPartCopy_by_range_invalid_range(s)
UploadPartCopy_greater_range_than_obj_size(s)
UploadPartCopy_by_range_success(s)
}
func TestListParts(s *S3Conf) {
ListParts_incorrect_uploadId(s)
ListParts_incorrect_object_key(s)
ListParts_success(s)
}
func TestListMultipartUploads(s *S3Conf) {
ListMultipartUploads_non_existing_bucket(s)
ListMultipartUploads_empty_result(s)
ListMultipartUploads_invalid_max_uploads(s)
ListMultipartUploads_max_uploads(s)
ListMultipartUploads_incorrect_next_key_marker(s)
ListMultipartUploads_ignore_upload_id_marker(s)
ListMultipartUploads_success(s)
}
func TestAbortMultipartUpload(s *S3Conf) {
AbortMultipartUpload_non_existing_bucket(s)
AbortMultipartUpload_incorrect_uploadId(s)
AbortMultipartUpload_incorrect_object_key(s)
AbortMultipartUpload_success(s)
}
func TestCompleteMultipartUpload(s *S3Conf) {
CompletedMultipartUpload_non_existing_bucket(s)
CompleteMultipartUpload_invalid_part_number(s)
CompleteMultipartUpload_invalid_ETag(s)
CompleteMultipartUpload_success(s)
}
func TestPutBucketAcl(s *S3Conf) {
PutBucketAcl_non_existing_bucket(s)
PutBucketAcl_invalid_acl_canned_and_acp(s)
PutBucketAcl_invalid_acl_canned_and_grants(s)
PutBucketAcl_invalid_acl_acp_and_grants(s)
PutBucketAcl_invalid_owner(s)
PutBucketAcl_success_access_denied(s)
PutBucketAcl_success_grants(s)
PutBucketAcl_success_canned_acl(s)
PutBucketAcl_success_acp(s)
}
func TestGetBucketAcl(s *S3Conf) {
GetBucketAcl_non_existing_bucket(s)
GetBucketAcl_access_denied(s)
GetBucketAcl_success(s)
}
func TestFullFlow(s *S3Conf) {
TestAuthentication(s)
TestCreateBucket(s)
TestHeadBucket(s)
TestListBuckets(s)
TestDeleteBucket(s)
TestPutObject(s)
TestHeadObject(s)
TestGetObject(s)
TestListObjects(s)
TestDeleteObject(s)
TestDeleteObjects(s)
TestCopyObject(s)
TestPutObjectTagging(s)
TestDeleteObjectTagging(s)
TestCreateMultipartUpload(s)
TestUploadPart(s)
TestUploadPartCopy(s)
TestListParts(s)
TestListMultipartUploads(s)
TestAbortMultipartUpload(s)
TestCompleteMultipartUpload(s)
TestPutBucketAcl(s)
TestGetBucketAcl(s)
}

View File

@@ -1,84 +0,0 @@
package integration
import (
"crypto/rand"
"crypto/sha256"
"hash"
"io"
)
type RReader struct {
buf []byte
dataleft int
hash hash.Hash
}
func NewDataReader(totalsize, bufsize int) *RReader {
b := make([]byte, bufsize)
rand.Read(b)
return &RReader{
buf: b,
dataleft: totalsize,
hash: sha256.New(),
}
}
func (r *RReader) Read(p []byte) (int, error) {
n := min(len(p), len(r.buf), r.dataleft)
r.dataleft -= n
err := error(nil)
if n == 0 {
err = io.EOF
}
r.hash.Write(r.buf[:n])
return copy(p, r.buf[:n]), err
}
func (r *RReader) Sum() []byte {
return r.hash.Sum(nil)
}
type ZReader struct {
buf []byte
dataleft int
}
func NewZeroReader(totalsize, bufsize int) *ZReader {
b := make([]byte, bufsize)
return &ZReader{buf: b, dataleft: totalsize}
}
func (r *ZReader) Read(p []byte) (int, error) {
n := min(len(p), len(r.buf), r.dataleft)
r.dataleft -= n
err := error(nil)
if n == 0 {
err = io.EOF
}
return copy(p, r.buf[:n]), err
}
func min(values ...int) int {
if len(values) == 0 {
return 0
}
min := values[0]
for _, v := range values {
if v < min {
min = v
}
}
return min
}
type NW struct{}
func NewNullWriter() NW {
return NW{}
}
func (NW) WriteAt(p []byte, off int64) (n int, err error) {
return len(p), nil
}

View File

@@ -1,31 +0,0 @@
package integration
import "fmt"
var (
colorReset = "\033[0m"
colorRed = "\033[31m"
colorGreen = "\033[32m"
colorCyan = "\033[36m"
)
var (
RunCount = 0
PassCount = 0
FailCount = 0
)
func runF(format string, a ...interface{}) {
RunCount++
fmt.Printf(colorCyan+"RUN "+colorReset+format+"\n", a...)
}
func failF(format string, a ...interface{}) {
FailCount++
fmt.Printf(colorRed+"FAIL "+colorReset+format+"\n", a...)
}
func passF(format string, a ...interface{}) {
PassCount++
fmt.Printf(colorGreen+"PASS "+colorReset+format+"\n", a...)
}

View File

@@ -1,153 +0,0 @@
package integration
import (
"context"
"io"
"log"
"net/http"
"os"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/smithy-go/middleware"
)
type S3Conf struct {
awsID string
awsSecret string
awsRegion string
endpoint string
checksumDisable bool
pathStyle bool
PartSize int64
Concurrency int
debug bool
}
func NewS3Conf(opts ...Option) *S3Conf {
s := &S3Conf{}
for _, opt := range opts {
opt(s)
}
return s
}
type Option func(*S3Conf)
func WithAccess(ak string) Option {
return func(s *S3Conf) { s.awsID = ak }
}
func WithSecret(sk string) Option {
return func(s *S3Conf) { s.awsSecret = sk }
}
func WithRegion(r string) Option {
return func(s *S3Conf) { s.awsRegion = r }
}
func WithEndpoint(e string) Option {
return func(s *S3Conf) { s.endpoint = e }
}
func WithDisableChecksum() Option {
return func(s *S3Conf) { s.checksumDisable = true }
}
func WithPathStyle() Option {
return func(s *S3Conf) { s.pathStyle = true }
}
func WithPartSize(p int64) Option {
return func(s *S3Conf) { s.PartSize = p }
}
func WithConcurrency(c int) Option {
return func(s *S3Conf) { s.Concurrency = c }
}
func WithDebug() Option {
return func(s *S3Conf) { s.debug = true }
}
func (c *S3Conf) getCreds() credentials.StaticCredentialsProvider {
// TODO support token/IAM
if c.awsSecret == "" {
c.awsSecret = os.Getenv("AWS_SECRET_ACCESS_KEY")
}
if c.awsSecret == "" {
log.Fatal("no AWS_SECRET_ACCESS_KEY found")
}
return credentials.NewStaticCredentialsProvider(c.awsID, c.awsSecret, "")
}
func (c *S3Conf) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
URL: c.endpoint,
SigningRegion: c.awsRegion,
HostnameImmutable: true,
}, nil
}
func (c *S3Conf) Config() aws.Config {
creds := c.getCreds()
tr := &http.Transport{}
client := &http.Client{Transport: tr}
opts := []func(*config.LoadOptions) error{
config.WithRegion(c.awsRegion),
config.WithCredentialsProvider(creds),
config.WithHTTPClient(client),
}
if c.endpoint != "" && c.endpoint != "aws" {
opts = append(opts,
config.WithEndpointResolverWithOptions(c))
}
if c.checksumDisable {
opts = append(opts,
config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
}
if c.debug {
opts = append(opts,
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
}
cfg, err := config.LoadDefaultConfig(
context.TODO(), opts...)
if err != nil {
log.Fatalln("error:", err)
}
return cfg
}
func (c *S3Conf) UploadData(r io.Reader, bucket, object string) error {
uploader := manager.NewUploader(s3.NewFromConfig(c.Config()))
uploader.PartSize = c.PartSize
uploader.Concurrency = c.Concurrency
upinfo := &s3.PutObjectInput{
Body: r,
Bucket: &bucket,
Key: &object,
}
_, err := uploader.Upload(context.Background(), upinfo)
return err
}
func (c *S3Conf) DownloadData(w io.WriterAt, bucket, object string) (int64, error) {
downloader := manager.NewDownloader(s3.NewFromConfig(c.Config()))
downloader.PartSize = c.PartSize
downloader.Concurrency = c.Concurrency
downinfo := &s3.GetObjectInput{
Bucket: &bucket,
Key: &object,
}
return downloader.Download(context.Background(), w, downinfo)
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,528 +0,0 @@
package integration
import (
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
var (
bcktCount = 0
succUsrCrt = "The user has been created successfully"
failUsrCrt = "failed to create a user: update iam data: account already exists"
)
func getBucketName() string {
bcktCount++
return fmt.Sprintf("test-bucket-%v", bcktCount)
}
func setup(s *S3Conf, bucket string) error {
s3client := s3.NewFromConfig(s.Config())
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: &bucket,
})
cancel()
return err
}
func teardown(s *S3Conf, bucket string) error {
s3client := s3.NewFromConfig(s.Config())
deleteObject := func(bucket, key, versionId *string) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: bucket,
Key: key,
VersionId: versionId,
})
cancel()
if err != nil {
return fmt.Errorf("failed to delete object %v: %v", *key, err)
}
return nil
}
in := &s3.ListObjectsV2Input{Bucket: &bucket}
for {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.ListObjectsV2(ctx, in)
cancel()
if err != nil {
return fmt.Errorf("failed to list objects: %v", err)
}
for _, item := range out.Contents {
err = deleteObject(&bucket, item.Key, nil)
if err != nil {
return err
}
}
if out.IsTruncated {
in.ContinuationToken = out.ContinuationToken
} else {
break
}
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{
Bucket: &bucket,
})
cancel()
return err
}
func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) {
runF(testName)
bucketName := getBucketName()
err := setup(s, bucketName)
if err != nil {
failF("%v: failed to create a bucket: %v", testName, err.Error())
return
}
client := s3.NewFromConfig(s.Config())
handlerErr := handler(client, bucketName)
if handlerErr != nil {
failF("%v: %v", testName, handlerErr.Error())
}
err = teardown(s, bucketName)
if err != nil {
if handlerErr == nil {
failF("%v: failed to delete the bucket: %v", testName, err.Error())
} else {
fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err.Error())
}
}
if handlerErr == nil {
passF(testName)
}
}
type authConfig struct {
testName string
path string
method string
body []byte
service string
date time.Time
}
func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) {
runF(cfg.testName)
req, err := http.NewRequest(cfg.method, fmt.Sprintf("%v/%v", s.endpoint, cfg.path), bytes.NewReader(cfg.body))
if err != nil {
failF("%v: failed to send the request: %v", cfg.testName, err.Error())
return
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.awsID, SecretAccessKey: s.awsSecret}, req, hexPayload, cfg.service, s.awsRegion, cfg.date)
if signErr != nil {
failF("%v: failed to sign the request: %v", cfg.testName, err.Error())
return
}
err = handler(req)
if err != nil {
failF("%v: %v", cfg.testName, err.Error())
return
}
passF(cfg.testName)
}
func checkAuthErr(resp *http.Response, apiErr s3err.APIError) error {
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
var errResp s3err.APIErrorResponse
err = xml.Unmarshal(body, &errResp)
if err != nil {
return err
}
if resp.StatusCode != apiErr.HTTPStatusCode {
return fmt.Errorf("expected response status code to be %v, instead got %v", apiErr.HTTPStatusCode, resp.StatusCode)
}
if errResp.Code != apiErr.Code {
return fmt.Errorf("expected error code to be %v, instead got %v", apiErr.Code, errResp.Code)
}
if errResp.Message != apiErr.Description {
return fmt.Errorf("expected error message to be %v, instead got %v", apiErr.Description, errResp.Message)
}
return nil
}
func checkApiErr(err error, apiErr s3err.APIError) error {
if err == nil {
return fmt.Errorf("expected %v, instead got nil", apiErr.Code)
}
var ae smithy.APIError
if errors.As(err, &ae) {
if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description {
return nil
}
return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
} else {
return fmt.Errorf("expected aws api error, instead got: %v", err.Error())
}
}
func checkSdkApiErr(err error, code string) error {
var ae smithy.APIError
if errors.As(err, &ae) {
if ae.ErrorCode() != code {
return fmt.Errorf("expected %v, instead got %v", ae.ErrorCode(), code)
}
return nil
}
return err
}
func putObjects(client *s3.Client, objs []string, bucket string) error {
for _, key := range objs {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := client.PutObject(ctx, &s3.PutObjectInput{
Key: &key,
Bucket: &bucket,
})
cancel()
if err != nil {
return err
}
}
return nil
}
func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) {
data = make([]byte, lgth)
rand.Read(data)
csum = sha256.Sum256(data)
r := bytes.NewReader(data)
input.Body = r
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err = client.PutObject(ctx, input)
cancel()
return
}
func CreateMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: &key,
})
cancel()
return out, err
}
func isEqual(a, b []byte) bool {
if len(a) != len(b) {
return false
}
for i, d := range a {
if d != b[i] {
return false
}
}
return true
}
func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool {
if len(list1) != len(list2) {
return false
}
for i, item := range list1 {
if *item.Key != *list2[i].Key || *item.UploadId != *list2[i].UploadId {
return false
}
}
return true
}
func compareParts(parts1, parts2 []types.Part) bool {
if len(parts1) != len(parts2) {
return false
}
for i, prt := range parts1 {
if prt.PartNumber != parts2[i].PartNumber {
return false
}
if *prt.ETag != *parts2[i].ETag {
return false
}
}
return true
}
func areTagsSame(tags1, tags2 []types.Tag) bool {
if len(tags1) != len(tags2) {
return false
}
for _, tag := range tags1 {
if !containsTag(tag, tags2) {
return false
}
}
return true
}
func containsTag(tag types.Tag, list []types.Tag) bool {
for _, item := range list {
if *item.Key == *tag.Key && *item.Value == *tag.Value {
return true
}
}
return false
}
func compareGrants(grts1, grts2 []types.Grant) bool {
if len(grts1) != len(grts2) {
return false
}
for i, grt := range grts1 {
if grt.Permission != grts2[i].Permission {
return false
}
if *grt.Grantee.ID != *grts2[i].Grantee.ID {
return false
}
}
return true
}
func execCommand(args ...string) ([]byte, error) {
cmd := exec.Command("./versitygw", args...)
return cmd.CombinedOutput()
}
func getString(str *string) string {
if str == nil {
return ""
}
return *str
}
func getPtr(str string) *string {
return &str
}
func areMapsSame(mp1, mp2 map[string]string) bool {
if len(mp1) != len(mp2) {
return false
}
for key, val := range mp1 {
if mp2[key] != val {
return false
}
}
return true
}
func compareBuckets(list1 []types.Bucket, list2 []s3response.ListAllMyBucketsEntry) bool {
if len(list1) != len(list2) {
return false
}
elementMap := make(map[string]bool)
for _, elem := range list1 {
elementMap[*elem.Name] = true
}
for _, elem := range list2 {
if _, found := elementMap[elem.Name]; !found {
return false
}
}
return true
}
func compareObjects(list1 []string, list2 []types.Object) bool {
if len(list1) != len(list2) {
return false
}
elementMap := make(map[string]bool)
for _, elem := range list1 {
elementMap[elem] = true
}
for _, elem := range list2 {
if _, found := elementMap[*elem.Key]; !found {
return false
}
}
return true
}
func comparePrefixes(list1 []string, list2 []types.CommonPrefix) bool {
if len(list1) != len(list2) {
return false
}
elementMap := make(map[string]bool)
for _, elem := range list1 {
elementMap[elem] = true
}
for _, elem := range list2 {
if _, found := elementMap[*elem.Prefix]; !found {
return false
}
}
return true
}
func compareDelObjects(list1 []string, list2 []types.DeletedObject) bool {
if len(list1) != len(list2) {
return false
}
elementMap := make(map[string]bool)
for _, elem := range list1 {
elementMap[elem] = true
}
for _, elem := range list2 {
if _, found := elementMap[*elem.Key]; !found {
return false
}
}
return true
}
func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId string) (parts []types.Part, err error) {
dr := NewDataReader(size, size)
datafile := "rand.data"
w, err := os.Create(datafile)
if err != nil {
return parts, err
}
defer w.Close()
_, err = io.Copy(w, dr)
if err != nil {
return parts, err
}
fileInfo, err := w.Stat()
if err != nil {
return parts, err
}
partSize := fileInfo.Size() / int64(partCount)
var offset int64
for partNumber := int64(1); partNumber <= int64(partCount); partNumber++ {
partStart := (partNumber - 1) * partSize
partEnd := partStart + partSize - 1
if partEnd > fileInfo.Size()-1 {
partEnd = fileInfo.Size() - 1
}
partBuffer := make([]byte, partEnd-partStart+1)
_, err := w.ReadAt(partBuffer, partStart)
if err != nil {
return parts, err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
Body: bytes.NewReader(partBuffer),
PartNumber: int32(partNumber),
})
cancel()
if err != nil {
return parts, err
} else {
parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)})
offset += partSize
}
}
return parts, err
}
type user struct {
access string
secret string
role string
}
func createUsers(s *S3Conf, users []user) error {
for _, usr := range users {
out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role)
if err != nil {
return err
}
if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) {
return fmt.Errorf("failed to create a user account")
}
}
return nil
}
func changeBucketsOwner(s *S3Conf, buckets []string, owner string) error {
for _, bucket := range buckets {
out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "change-bucket-owner", "-b", bucket, "-o", owner)
if err != nil {
return err
}
if !strings.Contains(string(out), "Bucket owner has been updated successfully") {
return fmt.Errorf(string(out))
}
}
return nil
}

View File

@@ -1,37 +0,0 @@
#!/bin/bash
# make temp dirs
mkdir /tmp/gw
rm -rf /tmp/covdata
mkdir /tmp/covdata
# run server in background
GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass posix /tmp/gw &
GW_PID=$!
# wait a second for server to start up
sleep 1
# check if server is still running
if ! kill -0 $GW_PID; then
echo "server no longer running"
exit 1
fi
# run tests
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow; then
echo "tests failed"
kill $GW_PID
exit 1
fi
# kill off server
kill $GW_PID
exit 0
# if the above binary was built with -cover enabled (make testbin),
# then the following can be used for code coverage reports:
# go tool covdata percent -i=/tmp/covdata
# go tool covdata textfmt -i=/tmp/covdata -o profile.txt
# go tool cover -html=profile.txt

View File

@@ -1,43 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3api
import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/controllers"
)
type S3AdminRouter struct{}
func (ar *S3AdminRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService) {
controller := controllers.NewAdminController(iam, be)
// CreateUser admin api
app.Patch("/create-user", controller.CreateUser)
// DeleteUsers admin api
app.Patch("/delete-user", controller.DeleteUser)
// ListUsers admin api
app.Patch("/list-users", controller.ListUsers)
// ChangeBucketOwner admin api
app.Patch("/change-bucket-owner", controller.ChangeBucketOwner)
// ListBucketsAndOwners admin api
app.Patch("/list-buckets", controller.ListBuckets)
}

View File

@@ -1,72 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3api
import (
"crypto/tls"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/logger"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/middlewares"
)
type S3AdminServer struct {
app *fiber.App
backend backend.Backend
router *S3AdminRouter
port string
cert *tls.Certificate
}
func NewAdminServer(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, opts ...AdminOpt) *S3AdminServer {
server := &S3AdminServer{
app: app,
backend: be,
router: new(S3AdminRouter),
port: port,
}
for _, opt := range opts {
opt(server)
}
// Logging middlewares
app.Use(logger.New())
app.Use(middlewares.DecodeURL(nil))
// Authentication middlewares
app.Use(middlewares.VerifyV4Signature(root, iam, nil, region, false))
app.Use(middlewares.VerifyMD5Body(nil))
app.Use(middlewares.AclParser(be, nil))
server.router.Init(app, be, iam)
return server
}
type AdminOpt func(s *S3AdminServer)
func WithAdminSrvTLS(cert tls.Certificate) AdminOpt {
return func(s *S3AdminServer) { s.cert = &cert }
}
func (sa *S3AdminServer) Serve() (err error) {
if sa.cert != nil {
return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
}
return sa.app.Listen(sa.port)
}

View File

@@ -1,119 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package controllers
import (
"fmt"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
)
type AdminController struct {
iam auth.IAMService
be backend.Backend
}
func NewAdminController(iam auth.IAMService, be backend.Backend) AdminController {
return AdminController{iam: iam, be: be}
}
func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
access, secret, role := ctx.Query("access"), ctx.Query("secret"), ctx.Query("role")
requesterRole := ctx.Locals("role").(string)
if requesterRole != "admin" {
return fmt.Errorf("access denied: only admin users have access to this resource")
}
if role != "user" && role != "admin" {
return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin'")
}
user := auth.Account{Secret: secret, Role: role}
err := c.iam.CreateAccount(access, user)
if err != nil {
return fmt.Errorf("failed to create a user: %w", err)
}
return ctx.SendString("The user has been created successfully")
}
func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
access := ctx.Query("access")
requesterRole := ctx.Locals("role").(string)
if requesterRole != "admin" {
return fmt.Errorf("access denied: only admin users have access to this resource")
}
err := c.iam.DeleteUserAccount(access)
if err != nil {
return err
}
return ctx.SendString("The user has been deleted successfully")
}
func (c AdminController) ListUsers(ctx *fiber.Ctx) error {
role := ctx.Locals("role").(string)
if role != "admin" {
return fmt.Errorf("access denied: only admin users have access to this resource")
}
accs, err := c.iam.ListUserAccounts()
if err != nil {
return err
}
return ctx.JSON(accs)
}
func (c AdminController) ChangeBucketOwner(ctx *fiber.Ctx) error {
role := ctx.Locals("role").(string)
if role != "admin" {
return fmt.Errorf("access denied: only admin users have access to this resource")
}
owner := ctx.Query("owner")
bucket := ctx.Query("bucket")
accs, err := auth.CheckIfAccountsExist([]string{owner}, c.iam)
if err != nil {
return err
}
if len(accs) > 0 {
return fmt.Errorf("user specified as the new bucket owner does not exist")
}
err = c.be.ChangeBucketOwner(ctx.Context(), bucket, owner)
if err != nil {
return err
}
return ctx.Status(201).SendString("Bucket owner has been updated successfully")
}
func (c AdminController) ListBuckets(ctx *fiber.Ctx) error {
role := ctx.Locals("role").(string)
if role != "admin" {
return fmt.Errorf("access denied: only admin users have access to this resource")
}
buckets, err := c.be.ListBucketsAndOwners(ctx.Context())
if err != nil {
return err
}
return ctx.JSON(buckets)
}

View File

@@ -1,467 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package controllers
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3response"
)
func TestAdminController_CreateUser(t *testing.T) {
type args struct {
req *http.Request
}
adminController := AdminController{
iam: &IAMServiceMock{
CreateAccountFunc: func(access string, account auth.Account) error {
return nil
},
},
}
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("role", "admin")
return ctx.Next()
})
app.Patch("/create-user", adminController.CreateUser)
appErr := fiber.New()
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("role", "user")
return ctx.Next()
})
appErr.Patch("/create-user", adminController.CreateUser)
tests := []struct {
name string
app *fiber.App
args args
wantErr bool
statusCode int
}{
{
name: "Admin-create-user-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/create-user?access=test&secret=test&role=user", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Admin-create-user-invalid-user-role",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/create-user?access=test&secret=test&role=invalid", nil),
},
wantErr: false,
statusCode: 500,
},
{
name: "Admin-create-user-invalid-requester-role",
app: appErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/create-user?access=test&secret=test&role=admin", nil),
},
wantErr: false,
statusCode: 500,
},
}
for _, tt := range tests {
resp, err := tt.app.Test(tt.args.req)
if (err != nil) != tt.wantErr {
t.Errorf("AdminController.CreateUser() error = %v, wantErr %v", err, tt.wantErr)
}
if resp.StatusCode != tt.statusCode {
t.Errorf("AdminController.CreateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
}
}
}
func TestAdminController_DeleteUser(t *testing.T) {
type args struct {
req *http.Request
}
adminController := AdminController{
iam: &IAMServiceMock{
DeleteUserAccountFunc: func(access string) error {
return nil
},
},
}
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("role", "admin")
return ctx.Next()
})
app.Patch("/delete-user", adminController.DeleteUser)
appErr := fiber.New()
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("role", "user")
return ctx.Next()
})
appErr.Patch("/delete-user", adminController.DeleteUser)
tests := []struct {
name string
app *fiber.App
args args
wantErr bool
statusCode int
}{
{
name: "Admin-delete-user-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Admin-delete-user-invalid-requester-role",
app: appErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
},
wantErr: false,
statusCode: 500,
},
}
for _, tt := range tests {
resp, err := tt.app.Test(tt.args.req)
if (err != nil) != tt.wantErr {
t.Errorf("AdminController.DeleteUser() error = %v, wantErr %v", err, tt.wantErr)
}
if resp.StatusCode != tt.statusCode {
t.Errorf("AdminController.DeleteUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
}
}
}
func TestAdminController_ListUsers(t *testing.T) {
type args struct {
req *http.Request
}
adminController := AdminController{
iam: &IAMServiceMock{
ListUserAccountsFunc: func() ([]auth.Account, error) {
return []auth.Account{}, nil
},
},
}
adminControllerErr := AdminController{
iam: &IAMServiceMock{
ListUserAccountsFunc: func() ([]auth.Account, error) {
return []auth.Account{}, fmt.Errorf("server error")
},
},
}
appErr := fiber.New()
appErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("role", "admin")
return ctx.Next()
})
appErr.Patch("/list-users", adminControllerErr.ListUsers)
appRoleErr := fiber.New()
appRoleErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("role", "user")
return ctx.Next()
})
appRoleErr.Patch("/list-users", adminController.ListUsers)
appSucc := fiber.New()
appSucc.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("role", "admin")
return ctx.Next()
})
appSucc.Patch("/list-users", adminController.ListUsers)
tests := []struct {
name string
app *fiber.App
args args
wantErr bool
statusCode int
}{
{
name: "Admin-list-users-access-denied",
app: appRoleErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
},
wantErr: false,
statusCode: 500,
},
{
name: "Admin-list-users-iam-error",
app: appErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
},
wantErr: false,
statusCode: 500,
},
{
name: "Admin-list-users-success",
app: appSucc,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/list-users", nil),
},
wantErr: false,
statusCode: 200,
},
}
for _, tt := range tests {
resp, err := tt.app.Test(tt.args.req)
if (err != nil) != tt.wantErr {
t.Errorf("AdminController.ListUsers() error = %v, wantErr %v", err, tt.wantErr)
}
if resp.StatusCode != tt.statusCode {
t.Errorf("AdminController.ListUsers() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
}
}
}
func TestAdminController_ChangeBucketOwner(t *testing.T) {
type args struct {
req *http.Request
}
adminController := AdminController{
be: &BackendMock{
ChangeBucketOwnerFunc: func(contextMoqParam context.Context, bucket, newOwner string) error {
return nil
},
},
iam: &IAMServiceMock{
GetUserAccountFunc: func(access string) (auth.Account, error) {
return auth.Account{}, nil
},
},
}
adminControllerIamErr := AdminController{
iam: &IAMServiceMock{
GetUserAccountFunc: func(access string) (auth.Account, error) {
return auth.Account{}, fmt.Errorf("unknown server error")
},
},
}
adminControllerIamAccDoesNotExist := AdminController{
iam: &IAMServiceMock{
GetUserAccountFunc: func(access string) (auth.Account, error) {
return auth.Account{}, auth.ErrNoSuchUser
},
},
}
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("role", "admin")
return ctx.Next()
})
app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
appRoleErr := fiber.New()
appRoleErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("role", "user")
return ctx.Next()
})
appRoleErr.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
appIamErr := fiber.New()
appIamErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("role", "admin")
return ctx.Next()
})
appIamErr.Patch("/change-bucket-owner", adminControllerIamErr.ChangeBucketOwner)
appIamNoSuchUser := fiber.New()
appIamNoSuchUser.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("role", "admin")
return ctx.Next()
})
appIamNoSuchUser.Patch("/change-bucket-owner", adminControllerIamAccDoesNotExist.ChangeBucketOwner)
tests := []struct {
name string
app *fiber.App
args args
wantErr bool
statusCode int
}{
{
name: "Change-bucket-owner-access-denied",
app: appRoleErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
},
wantErr: false,
statusCode: 500,
},
{
name: "Change-bucket-owner-check-account-server-error",
app: appIamErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
},
wantErr: false,
statusCode: 500,
},
{
name: "Change-bucket-owner-acc-does-not-exist",
app: appIamNoSuchUser,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner", nil),
},
wantErr: false,
statusCode: 500,
},
{
name: "Change-bucket-owner-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/change-bucket-owner?bucket=bucket&owner=owner", nil),
},
wantErr: false,
statusCode: 201,
},
}
for _, tt := range tests {
resp, err := tt.app.Test(tt.args.req)
if (err != nil) != tt.wantErr {
t.Errorf("AdminController.ChangeBucketOwner() error = %v, wantErr %v", err, tt.wantErr)
}
if resp.StatusCode != tt.statusCode {
t.Errorf("AdminController.ChangeBucketOwner() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
}
}
}
func TestAdminController_ListBuckets(t *testing.T) {
type args struct {
req *http.Request
}
adminController := AdminController{
be: &BackendMock{
ListBucketsAndOwnersFunc: func(contextMoqParam context.Context) ([]s3response.Bucket, error) {
return []s3response.Bucket{}, nil
},
},
}
app := fiber.New()
app.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("role", "admin")
return ctx.Next()
})
app.Patch("/list-buckets", adminController.ListBuckets)
appRoleErr := fiber.New()
appRoleErr.Use(func(ctx *fiber.Ctx) error {
ctx.Locals("role", "user")
return ctx.Next()
})
appRoleErr.Patch("/list-buckets", adminController.ListBuckets)
tests := []struct {
name string
app *fiber.App
args args
wantErr bool
statusCode int
}{
{
name: "List-buckets-incorrect-role",
app: appRoleErr,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
},
wantErr: false,
statusCode: 500,
},
{
name: "List-buckets-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPatch, "/list-buckets", nil),
},
wantErr: false,
statusCode: 200,
},
}
for _, tt := range tests {
resp, err := tt.app.Test(tt.args.req)
if (err != nil) != tt.wantErr {
t.Errorf("AdminController.ListBuckets() error = %v, wantErr %v", err, tt.wantErr)
}
if resp.StatusCode != tt.statusCode {
t.Errorf("AdminController.ListBuckets() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
}
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,206 +0,0 @@
// Code generated by moq; DO NOT EDIT.
// github.com/matryer/moq
package controllers
import (
"github.com/versity/versitygw/auth"
"sync"
)
// Ensure, that IAMServiceMock does implement auth.IAMService.
// If this is not the case, regenerate this file with moq.
var _ auth.IAMService = &IAMServiceMock{}
// IAMServiceMock is a mock implementation of auth.IAMService.
//
// func TestSomethingThatUsesIAMService(t *testing.T) {
//
// // make and configure a mocked auth.IAMService
// mockedIAMService := &IAMServiceMock{
// CreateAccountFunc: func(access string, account auth.Account) error {
// panic("mock out the CreateAccount method")
// },
// DeleteUserAccountFunc: func(access string) error {
// panic("mock out the DeleteUserAccount method")
// },
// GetUserAccountFunc: func(access string) (auth.Account, error) {
// panic("mock out the GetUserAccount method")
// },
// ListUserAccountsFunc: func() ([]auth.Account, error) {
// panic("mock out the ListUserAccounts method")
// },
// }
//
// // use mockedIAMService in code that requires auth.IAMService
// // and then make assertions.
//
// }
type IAMServiceMock struct {
// CreateAccountFunc mocks the CreateAccount method.
CreateAccountFunc func(access string, account auth.Account) error
// DeleteUserAccountFunc mocks the DeleteUserAccount method.
DeleteUserAccountFunc func(access string) error
// GetUserAccountFunc mocks the GetUserAccount method.
GetUserAccountFunc func(access string) (auth.Account, error)
// ListUserAccountsFunc mocks the ListUserAccounts method.
ListUserAccountsFunc func() ([]auth.Account, error)
// calls tracks calls to the methods.
calls struct {
// CreateAccount holds details about calls to the CreateAccount method.
CreateAccount []struct {
// Access is the access argument value.
Access string
// Account is the account argument value.
Account auth.Account
}
// DeleteUserAccount holds details about calls to the DeleteUserAccount method.
DeleteUserAccount []struct {
// Access is the access argument value.
Access string
}
// GetUserAccount holds details about calls to the GetUserAccount method.
GetUserAccount []struct {
// Access is the access argument value.
Access string
}
// ListUserAccounts holds details about calls to the ListUserAccounts method.
ListUserAccounts []struct {
}
}
lockCreateAccount sync.RWMutex
lockDeleteUserAccount sync.RWMutex
lockGetUserAccount sync.RWMutex
lockListUserAccounts sync.RWMutex
}
// CreateAccount calls CreateAccountFunc.
func (mock *IAMServiceMock) CreateAccount(access string, account auth.Account) error {
if mock.CreateAccountFunc == nil {
panic("IAMServiceMock.CreateAccountFunc: method is nil but IAMService.CreateAccount was just called")
}
callInfo := struct {
Access string
Account auth.Account
}{
Access: access,
Account: account,
}
mock.lockCreateAccount.Lock()
mock.calls.CreateAccount = append(mock.calls.CreateAccount, callInfo)
mock.lockCreateAccount.Unlock()
return mock.CreateAccountFunc(access, account)
}
// CreateAccountCalls gets all the calls that were made to CreateAccount.
// Check the length with:
//
// len(mockedIAMService.CreateAccountCalls())
func (mock *IAMServiceMock) CreateAccountCalls() []struct {
Access string
Account auth.Account
} {
var calls []struct {
Access string
Account auth.Account
}
mock.lockCreateAccount.RLock()
calls = mock.calls.CreateAccount
mock.lockCreateAccount.RUnlock()
return calls
}
// DeleteUserAccount calls DeleteUserAccountFunc.
func (mock *IAMServiceMock) DeleteUserAccount(access string) error {
if mock.DeleteUserAccountFunc == nil {
panic("IAMServiceMock.DeleteUserAccountFunc: method is nil but IAMService.DeleteUserAccount was just called")
}
callInfo := struct {
Access string
}{
Access: access,
}
mock.lockDeleteUserAccount.Lock()
mock.calls.DeleteUserAccount = append(mock.calls.DeleteUserAccount, callInfo)
mock.lockDeleteUserAccount.Unlock()
return mock.DeleteUserAccountFunc(access)
}
// DeleteUserAccountCalls gets all the calls that were made to DeleteUserAccount.
// Check the length with:
//
// len(mockedIAMService.DeleteUserAccountCalls())
func (mock *IAMServiceMock) DeleteUserAccountCalls() []struct {
Access string
} {
var calls []struct {
Access string
}
mock.lockDeleteUserAccount.RLock()
calls = mock.calls.DeleteUserAccount
mock.lockDeleteUserAccount.RUnlock()
return calls
}
// GetUserAccount calls GetUserAccountFunc.
func (mock *IAMServiceMock) GetUserAccount(access string) (auth.Account, error) {
if mock.GetUserAccountFunc == nil {
panic("IAMServiceMock.GetUserAccountFunc: method is nil but IAMService.GetUserAccount was just called")
}
callInfo := struct {
Access string
}{
Access: access,
}
mock.lockGetUserAccount.Lock()
mock.calls.GetUserAccount = append(mock.calls.GetUserAccount, callInfo)
mock.lockGetUserAccount.Unlock()
return mock.GetUserAccountFunc(access)
}
// GetUserAccountCalls gets all the calls that were made to GetUserAccount.
// Check the length with:
//
// len(mockedIAMService.GetUserAccountCalls())
func (mock *IAMServiceMock) GetUserAccountCalls() []struct {
Access string
} {
var calls []struct {
Access string
}
mock.lockGetUserAccount.RLock()
calls = mock.calls.GetUserAccount
mock.lockGetUserAccount.RUnlock()
return calls
}
// ListUserAccounts calls ListUserAccountsFunc.
func (mock *IAMServiceMock) ListUserAccounts() ([]auth.Account, error) {
if mock.ListUserAccountsFunc == nil {
panic("IAMServiceMock.ListUserAccountsFunc: method is nil but IAMService.ListUserAccounts was just called")
}
callInfo := struct {
}{}
mock.lockListUserAccounts.Lock()
mock.calls.ListUserAccounts = append(mock.calls.ListUserAccounts, callInfo)
mock.lockListUserAccounts.Unlock()
return mock.ListUserAccountsFunc()
}
// ListUserAccountsCalls gets all the calls that were made to ListUserAccounts.
// Check the length with:
//
// len(mockedIAMService.ListUserAccountsCalls())
func (mock *IAMServiceMock) ListUserAccountsCalls() []struct {
} {
var calls []struct {
}
mock.lockListUserAccounts.RLock()
calls = mock.calls.ListUserAccounts
mock.lockListUserAccounts.RUnlock()
return calls
}

View File

@@ -1,61 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"net/http"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3log"
)
func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
isRoot, access := ctx.Locals("isRoot").(bool), ctx.Locals("access").(string)
path := ctx.Path()
pathParts := strings.Split(path, "/")
bucket := pathParts[1]
if path == "/" && ctx.Method() == http.MethodGet {
return ctx.Next()
}
if ctx.Method() == http.MethodPatch {
return ctx.Next()
}
if len(pathParts) == 2 && pathParts[1] != "" && ctx.Method() == http.MethodPut && !ctx.Request().URI().QueryArgs().Has("acl") {
if err := auth.IsAdmin(access, isRoot); err != nil {
return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
}
return ctx.Next()
}
//TODO: provide correct action names for the logger, after implementing DetectAction middleware
data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
if err != nil {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}
parsedAcl, err := auth.ParseACL(data)
if err != nil {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}
ctx.Locals("parsedAcl", parsedAcl)
return ctx.Next()
}
}

View File

@@ -1,245 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"math"
"net/http"
"os"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/smithy-go/logging"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
const (
iso8601Format = "20060102T150405Z"
YYYYMMDD = "20060102"
)
type RootUserConfig struct {
Access string
Secret string
}
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
ctx.Locals("region", region)
ctx.Locals("startTime", time.Now())
authorization := ctx.Get("Authorization")
if authorization == "" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), &controllers.MetaOpts{Logger: logger})
}
// Check the signature version
authParts := strings.Split(authorization, ",")
for i, el := range authParts {
authParts[i] = strings.TrimSpace(el)
}
if len(authParts) != 3 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
}
startParts := strings.Split(authParts[0], " ")
if startParts[0] != "AWS4-HMAC-SHA256" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), &controllers.MetaOpts{Logger: logger})
}
credKv := strings.Split(startParts[1], "=")
if len(credKv) != 2 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
}
// Credential variables validation
creds := strings.Split(credKv[1], "/")
if len(creds) != 5 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
}
if creds[4] != "aws4_request" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureTerminationStr), &controllers.MetaOpts{Logger: logger})
}
if creds[3] != "s3" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureIncorrService), &controllers.MetaOpts{Logger: logger})
}
if creds[2] != region {
return controllers.SendResponse(ctx, s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),
HTTPStatusCode: http.StatusForbidden,
}, &controllers.MetaOpts{Logger: logger})
}
ctx.Locals("access", creds[0])
ctx.Locals("isRoot", creds[0] == root.Access)
_, err := time.Parse(YYYYMMDD, creds[1])
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
}
signHdrKv := strings.Split(authParts[1], "=")
if len(signHdrKv) != 2 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
}
signedHdrs := strings.Split(signHdrKv[1], ";")
account, err := acct.getAccount(creds[0])
if err == auth.ErrNoSuchUser {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), &controllers.MetaOpts{Logger: logger})
}
if err != nil {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}
ctx.Locals("role", account.Role)
// Check X-Amz-Date header
date := ctx.Get("X-Amz-Date")
if date == "" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), &controllers.MetaOpts{Logger: logger})
}
// Parse the date and check the date validity
tdate, err := time.Parse(iso8601Format, date)
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), &controllers.MetaOpts{Logger: logger})
}
if date[:8] != creds[1] {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
}
// Validate the dates difference
err = validateDate(tdate)
if err != nil {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}
hashPayloadHeader := ctx.Get("X-Amz-Content-Sha256")
ok := isSpecialPayload(hashPayloadHeader)
if !ok {
// Calculate the hash of the request payload
hashedPayload := sha256.Sum256(ctx.Body())
hexPayload := hex.EncodeToString(hashedPayload[:])
// Compare the calculated hash with the hash provided
if hashPayloadHeader != hexPayload {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), &controllers.MetaOpts{Logger: logger})
}
}
// Create a new http request instance from fasthttp request
req, err := utils.CreateHttpRequestFromCtx(ctx, signedHdrs)
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
}
signer := v4.NewSigner()
signErr := signer.SignHTTP(req.Context(), aws.Credentials{
AccessKeyID: creds[0],
SecretAccessKey: account.Secret,
}, req, hashPayloadHeader, creds[3], region, tdate, func(options *v4.SignerOptions) {
options.DisableURIPathEscaping = true
if debug {
options.LogSigning = true
options.Logger = logging.NewStandardLogger(os.Stderr)
}
})
if signErr != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
}
parts := strings.Split(req.Header.Get("Authorization"), " ")
if len(parts) < 4 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
}
calculatedSign := strings.Split(parts[3], "=")[1]
expectedSign := strings.Split(authParts[2], "=")[1]
if expectedSign != calculatedSign {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch), &controllers.MetaOpts{Logger: logger})
}
return ctx.Next()
}
}
type accounts struct {
root RootUserConfig
iam auth.IAMService
}
func (a accounts) getAccount(access string) (auth.Account, error) {
if access == a.root.Access {
return auth.Account{
Secret: a.root.Secret,
Role: "admin",
}, nil
}
return a.iam.GetUserAccount(access)
}
func isSpecialPayload(str string) bool {
specialValues := map[string]bool{
"UNSIGNED-PAYLOAD": true,
"STREAMING-UNSIGNED-PAYLOAD-TRAILER": true,
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD": true,
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER": true,
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD": true,
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
}
return specialValues[str]
}
func validateDate(date time.Time) error {
now := time.Now().UTC()
diff := date.Unix() - now.Unix()
// Checks the dates difference to be less than a minute
if math.Abs(float64(diff)) > 60 {
if diff > 0 {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
}
} else {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
}
}
}
return nil
}

View File

@@ -1,44 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"fmt"
"log"
"github.com/gofiber/fiber/v2"
)
func RequestLogger(isDebug bool) fiber.Handler {
return func(ctx *fiber.Ctx) error {
ctx.Locals("isDebug", isDebug)
if isDebug {
log.Println("Request headers: ")
ctx.Request().Header.VisitAll(func(key, val []byte) {
log.Printf("%s: %s", key, val)
})
if ctx.Request().URI().QueryArgs().Len() != 0 {
fmt.Println()
log.Println("Request query arguments: ")
ctx.Request().URI().QueryArgs().VisitAll(func(key, val []byte) {
log.Printf("%s: %s", key, val)
})
}
}
return ctx.Next()
}
}

View File

@@ -1,43 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"crypto/md5"
"encoding/base64"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
incomingSum := ctx.Get("Content-Md5")
if incomingSum == "" {
return ctx.Next()
}
sum := md5.Sum(ctx.Body())
calculatedSum := base64.StdEncoding.EncodeToString(sum[:])
if incomingSum != calculatedSum {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})
}
return ctx.Next()
}
}

View File

@@ -1,41 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"net/url"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func DecodeURL(logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
reqURL := ctx.Request().URI().String()
decoded, err := url.Parse(reqURL)
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
}
ctx.Path(decoded.Path)
decodedURL, err := url.QueryUnescape(reqURL)
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidURI), &controllers.MetaOpts{Logger: logger})
}
ctx.Request().SetRequestURI(decodedURL)
return ctx.Next()
}
}

View File

@@ -1,103 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3api
import (
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3log"
)
type S3ApiRouter struct {
WithAdmSrv bool
}
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender) {
s3ApiController := controllers.New(be, iam, logger, evs)
if sa.WithAdmSrv {
adminController := controllers.NewAdminController(iam, be)
// CreateUser admin api
app.Patch("/create-user", adminController.CreateUser)
// DeleteUsers admin api
app.Patch("/delete-user", adminController.DeleteUser)
// ListUsers admin api
app.Patch("/list-users", adminController.ListUsers)
// ChangeBucketOwner admin api
app.Patch("/change-bucket-owner", adminController.ChangeBucketOwner)
// ListBucketsAndOwners admin api
app.Patch("/list-buckets", adminController.ListBuckets)
}
// ListBuckets action
app.Get("/", s3ApiController.ListBuckets)
// CreateBucket action
// PutBucketAcl action
app.Put("/:bucket", s3ApiController.PutBucketActions)
// DeleteBucket action
app.Delete("/:bucket", s3ApiController.DeleteBucket)
// HeadBucket
app.Head("/:bucket", s3ApiController.HeadBucket)
// GetBucketAcl action
// ListMultipartUploads action
// ListObjects action
// ListObjectsV2 action
app.Get("/:bucket", s3ApiController.ListActions)
// HeadObject action
app.Head("/:bucket/:key/*", s3ApiController.HeadObject)
// GetObjectAcl action
// GetObject action
// ListObjectParts action
// GetTags action
// ListParts action
// GetObjectAttributes action
app.Get("/:bucket/:key/*", s3ApiController.GetActions)
// DeleteObject action
// AbortMultipartUpload action
// RemoveTags action
app.Delete("/:bucket/:key/*", s3ApiController.DeleteActions)
// DeleteObjects action
app.Post("/:bucket", s3ApiController.DeleteObjects)
// CompleteMultipartUpload action
// CreateMultipartUpload
// RestoreObject action
// SelectObjectContent action
app.Post("/:bucket/:key/*", s3ApiController.CreateActions)
// CopyObject action
// PutObject action
// UploadPart action
// UploadPartCopy action
// SetTags action
// PutObjectAcl action
app.Put("/:bucket/:key/*", s3ApiController.PutActions)
}

View File

@@ -1,51 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3api
import (
"testing"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
)
func TestS3ApiRouter_Init(t *testing.T) {
type args struct {
app *fiber.App
be backend.Backend
iam auth.IAMService
}
tests := []struct {
name string
sa *S3ApiRouter
args args
}{
{
name: "Initialize S3 api router",
sa: &S3ApiRouter{},
args: args{
app: fiber.New(),
be: backend.BackendUnsupported{},
iam: &auth.IAMServiceInternal{},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil)
})
}
}

View File

@@ -1,88 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3api
import (
"crypto/tls"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/logger"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/middlewares"
"github.com/versity/versitygw/s3event"
"github.com/versity/versitygw/s3log"
)
type S3ApiServer struct {
app *fiber.App
backend backend.Backend
router *S3ApiRouter
port string
cert *tls.Certificate
debug bool
}
func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, evs s3event.S3EventSender, opts ...Option) (*S3ApiServer, error) {
server := &S3ApiServer{
app: app,
backend: be,
router: new(S3ApiRouter),
port: port,
}
for _, opt := range opts {
opt(server)
}
// Logging middlewares
app.Use(logger.New())
app.Use(middlewares.DecodeURL(l))
app.Use(middlewares.RequestLogger(server.debug))
// Authentication middlewares
app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
app.Use(middlewares.VerifyMD5Body(l))
app.Use(middlewares.AclParser(be, l))
server.router.Init(app, be, iam, l, evs)
return server, nil
}
// Option sets various options for New()
type Option func(*S3ApiServer)
// WithTLS sets TLS Credentials
func WithTLS(cert tls.Certificate) Option {
return func(s *S3ApiServer) { s.cert = &cert }
}
// WithAdminServer runs admin endpoints with the gateway in the same network
func WithAdminServer() Option {
return func(s *S3ApiServer) { s.router.WithAdmSrv = true }
}
// WithDebug sets debug output
func WithDebug() Option {
return func(s *S3ApiServer) { s.debug = true }
}
func (sa *S3ApiServer) Serve() (err error) {
if sa.cert != nil {
return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
}
return sa.app.Listen(sa.port)
}

View File

@@ -1,114 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3api
import (
"crypto/tls"
"reflect"
"testing"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3api/middlewares"
)
func TestNew(t *testing.T) {
type args struct {
app *fiber.App
be backend.Backend
port string
root middlewares.RootUserConfig
}
app := fiber.New()
be := backend.BackendUnsupported{}
router := S3ApiRouter{}
port := ":7070"
tests := []struct {
name string
args args
wantS3ApiServer *S3ApiServer
wantErr bool
}{
{
name: "Create S3 api server",
args: args{
app: app,
be: be,
port: port,
root: middlewares.RootUserConfig{},
},
wantS3ApiServer: &S3ApiServer{
app: app,
port: port,
router: &router,
backend: be,
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil)
if (err != nil) != tt.wantErr {
t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(gotS3ApiServer, tt.wantS3ApiServer) {
t.Errorf("New() = %v, want %v", gotS3ApiServer, tt.wantS3ApiServer)
}
})
}
}
func TestS3ApiServer_Serve(t *testing.T) {
tests := []struct {
name string
sa *S3ApiServer
wantErr bool
}{
{
name: "Serve-invalid-address",
wantErr: true,
sa: &S3ApiServer{
app: fiber.New(),
backend: backend.BackendUnsupported{},
port: "Invalid address",
router: &S3ApiRouter{},
},
},
{
name: "Serve-invalid-address-with-certificate",
wantErr: true,
sa: &S3ApiServer{
app: fiber.New(),
backend: backend.BackendUnsupported{},
port: "Invalid address",
router: &S3ApiRouter{},
cert: &tls.Certificate{},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := tt.sa.Serve(); (err != nil) != tt.wantErr {
t.Errorf("S3ApiServer.Serve() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

View File

@@ -1,55 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
"fmt"
"log"
"github.com/gofiber/fiber/v2"
)
func LogCtxDetails(ctx *fiber.Ctx, respBody []byte) {
isDebug, ok := ctx.Locals("isDebug").(bool)
_, notLogReqBody := ctx.Locals("logReqBody").(bool)
_, notLogResBody := ctx.Locals("logResBody").(bool)
if isDebug && ok {
// Log request body
if !notLogReqBody {
fmt.Println()
log.Printf("Request Body: %s", ctx.Request().Body())
}
// Log path parameters
fmt.Println()
log.Println("Path parameters: ")
for key, val := range ctx.AllParams() {
log.Printf("%s: %s", key, val)
}
// Log response headers
fmt.Println()
log.Println("Response Headers: ")
ctx.Response().Header.VisitAll(func(key, val []byte) {
log.Printf("%s: %s", key, val)
})
// Log response body
if !notLogResBody && len(respBody) > 0 {
fmt.Println()
log.Printf("Response body %s", ctx.Response().Body())
}
}
}

View File

@@ -1,128 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
"bytes"
"errors"
"fmt"
"net/http"
"regexp"
"strconv"
"strings"
"github.com/gofiber/fiber/v2"
"github.com/valyala/fasthttp"
"github.com/versity/versitygw/s3err"
)
var (
bucketNameRegexp = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]+[a-z0-9]$`)
bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
)
func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
metadata = make(map[string]string)
headers.VisitAll(func(key, value []byte) {
if strings.HasPrefix(string(key), "X-Amz-Meta-") {
trimmedKey := strings.TrimPrefix(string(key), "X-Amz-Meta-")
headerValue := string(value)
metadata[trimmedKey] = headerValue
}
})
return
}
func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Request, error) {
req := ctx.Request()
httpReq, err := http.NewRequest(string(req.Header.Method()), req.URI().String(), bytes.NewReader(req.Body()))
if err != nil {
return nil, errors.New("error in creating an http request")
}
// Set the request headers
req.Header.VisitAll(func(key, value []byte) {
keyStr := string(key)
if includeHeader(keyStr, signedHdrs) {
httpReq.Header.Add(keyStr, string(value))
}
})
// Check if Content-Length in signed headers
// If content length is non 0, then the header will be included
if !includeHeader("Content-Length", signedHdrs) {
httpReq.ContentLength = 0
}
// Set the Host header
httpReq.Host = string(req.Header.Host())
return httpReq, nil
}
func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
for key, val := range meta {
ctx.Set(fmt.Sprintf("X-Amz-Meta-%s", key), val)
}
}
func ParseUint(str string) (int32, error) {
if str == "" {
return -1, nil
}
num, err := strconv.ParseUint(str, 10, 16)
if err != nil {
return -1, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)
}
return int32(num), nil
}
type CustomHeader struct {
Key string
Value string
}
func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) {
for _, header := range headers {
ctx.Set(header.Key, header.Value)
}
}
func IsValidBucketName(bucket string) bool {
if len(bucket) < 3 || len(bucket) > 63 {
return false
}
// Checks to contain only digits, lowercase letters, dot, hyphen.
// Checks to start and end with only digits and lowercase letters.
if !bucketNameRegexp.MatchString(bucket) {
return false
}
// Checks not to be a valid IP address
if bucketNameIpRegexp.MatchString(bucket) {
return false
}
return true
}
func includeHeader(hdr string, signedHdrs []string) bool {
for _, shdr := range signedHdrs {
if strings.EqualFold(hdr, shdr) {
return true
}
}
return false
}

View File

@@ -1,272 +0,0 @@
package utils
import (
"bytes"
"net/http"
"reflect"
"testing"
"github.com/gofiber/fiber/v2"
"github.com/valyala/fasthttp"
)
func TestCreateHttpRequestFromCtx(t *testing.T) {
type args struct {
ctx *fiber.Ctx
}
app := fiber.New()
// Expected output, Case 1
ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
req := ctx.Request()
request, _ := http.NewRequest(string(req.Header.Method()), req.URI().String(), bytes.NewReader(req.Body()))
// Case 2
ctx2 := app.AcquireCtx(&fasthttp.RequestCtx{})
req2 := ctx2.Request()
req2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
request2, _ := http.NewRequest(string(req2.Header.Method()), req2.URI().String(), bytes.NewReader(req2.Body()))
request2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
tests := []struct {
name string
args args
want *http.Request
wantErr bool
}{
{
name: "Success-response",
args: args{
ctx: ctx,
},
want: request,
wantErr: false,
},
{
name: "Success-response-With-Headers",
args: args{
ctx: ctx2,
},
want: request2,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := CreateHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"})
if (err != nil) != tt.wantErr {
t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got.Header, tt.want.Header) {
t.Errorf("CreateHttpRequestFromCtx() got = %v, want %v", got, tt.want)
}
})
}
}
func TestGetUserMetaData(t *testing.T) {
type args struct {
headers *fasthttp.RequestHeader
}
app := fiber.New()
// Case 1
ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
req := ctx.Request()
// Case 2
ctx2 := app.AcquireCtx(&fasthttp.RequestCtx{})
req2 := ctx2.Request()
req2.Header.Add("X-Amz-Meta-Name", "Nick")
req2.Header.Add("X-Amz-Meta-Age", "27")
tests := []struct {
name string
args args
wantMetadata map[string]string
}{
{
name: "Success-empty-response",
args: args{
headers: &req.Header,
},
wantMetadata: map[string]string{},
},
{
name: "Success-non-empty-response",
args: args{
headers: &req2.Header,
},
wantMetadata: map[string]string{
"Age": "27",
"Name": "Nick",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if gotMetadata := GetUserMetaData(tt.args.headers); !reflect.DeepEqual(gotMetadata, tt.wantMetadata) {
t.Errorf("GetUserMetaData() = %v, want %v", gotMetadata, tt.wantMetadata)
}
})
}
}
func Test_includeHeader(t *testing.T) {
type args struct {
hdr string
signedHdrs []string
}
tests := []struct {
name string
args args
want bool
}{
{
name: "include-header-falsy-case",
args: args{
hdr: "Content-Type",
signedHdrs: []string{"X-Amz-Acl", "Content-Encoding"},
},
want: false,
},
{
name: "include-header-falsy-case",
args: args{
hdr: "Content-Type",
signedHdrs: []string{"X-Amz-Acl", "Content-Type"},
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := includeHeader(tt.args.hdr, tt.args.signedHdrs); got != tt.want {
t.Errorf("includeHeader() = %v, want %v", got, tt.want)
}
})
}
}
func TestIsValidBucketName(t *testing.T) {
type args struct {
bucket string
}
tests := []struct {
name string
args args
want bool
}{
{
name: "IsValidBucketName-short-name",
args: args{
bucket: "a",
},
want: false,
},
{
name: "IsValidBucketName-start-with-hyphen",
args: args{
bucket: "-bucket",
},
want: false,
},
{
name: "IsValidBucketName-start-with-dot",
args: args{
bucket: ".bucket",
},
want: false,
},
{
name: "IsValidBucketName-contain-invalid-character",
args: args{
bucket: "my@bucket",
},
want: false,
},
{
name: "IsValidBucketName-end-with-hyphen",
args: args{
bucket: "bucket-",
},
want: false,
},
{
name: "IsValidBucketName-end-with-dot",
args: args{
bucket: "bucket.",
},
want: false,
},
{
name: "IsValidBucketName-valid-bucket-name",
args: args{
bucket: "my-bucket",
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := IsValidBucketName(tt.args.bucket); got != tt.want {
t.Errorf("IsValidBucketName() = %v, want %v", got, tt.want)
}
})
}
}
func TestParseUint(t *testing.T) {
type args struct {
str string
}
tests := []struct {
name string
args args
want int32
wantErr bool
}{
{
name: "Parse-uint-empty-string",
args: args{
str: "",
},
want: -1,
wantErr: false,
},
{
name: "Parse-uint-invalid-number-string",
args: args{
str: "bla",
},
want: -1,
wantErr: true,
},
{
name: "Parse-uint-invalid-negative-number",
args: args{
str: "-5",
},
want: -1,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ParseUint(tt.args.str)
if (err != nil) != tt.wantErr {
t.Errorf("ParseMaxKeys() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("ParseMaxKeys() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -1,440 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3err
import (
"bytes"
"encoding/xml"
"net/http"
)
// APIError structure
type APIError struct {
Code string
Description string
HTTPStatusCode int
}
// APIErrorResponse - error response format
type APIErrorResponse struct {
XMLName xml.Name `xml:"Error" json:"-"`
Code string
Message string
Key string `xml:"Key,omitempty" json:"Key,omitempty"`
BucketName string `xml:"BucketName,omitempty" json:"BucketName,omitempty"`
Resource string
Region string `xml:"Region,omitempty" json:"Region,omitempty"`
RequestID string `xml:"RequestId" json:"RequestId"`
HostID string `xml:"HostId" json:"HostId"`
}
func (A APIError) Error() string {
var bytesBuffer bytes.Buffer
bytesBuffer.WriteString(xml.Header)
e := xml.NewEncoder(&bytesBuffer)
_ = e.Encode(A)
return bytesBuffer.String()
}
// ErrorCode type of error status.
type ErrorCode int
// Error codes, see full list at http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
const (
ErrNone ErrorCode = iota
ErrAccessDenied
ErrMethodNotAllowed
ErrBucketNotEmpty
ErrBucketAlreadyExists
ErrBucketAlreadyOwnedByYou
ErrNoSuchBucket
ErrNoSuchKey
ErrNoSuchUpload
ErrInvalidBucketName
ErrInvalidDigest
ErrInvalidMaxKeys
ErrInvalidMaxUploads
ErrInvalidMaxParts
ErrInvalidPartNumberMarker
ErrInvalidPart
ErrInternalError
ErrInvalidCopyDest
ErrInvalidCopySource
ErrInvalidTag
ErrAuthHeaderEmpty
ErrSignatureVersionNotSupported
ErrMalformedPOSTRequest
ErrPOSTFileRequired
ErrPostPolicyConditionInvalidFormat
ErrEntityTooSmall
ErrEntityTooLarge
ErrMissingFields
ErrMissingCredTag
ErrCredMalformed
ErrMalformedXML
ErrMalformedDate
ErrMalformedPresignedDate
ErrMalformedCredentialDate
ErrMissingSignHeadersTag
ErrMissingSignTag
ErrUnsignedHeaders
ErrInvalidQueryParams
ErrInvalidQuerySignatureAlgo
ErrExpiredPresignRequest
ErrMalformedExpires
ErrNegativeExpires
ErrMaximumExpires
ErrSignatureDoesNotMatch
ErrSignatureDateDoesNotMatch
ErrSignatureTerminationStr
ErrSignatureIncorrService
ErrContentSHA256Mismatch
ErrInvalidAccessKeyID
ErrRequestNotReadyYet
ErrMissingDateHeader
ErrInvalidRequest
ErrAuthNotSetup
ErrNotImplemented
ErrPreconditionFailed
ErrInvalidObjectState
ErrInvalidRange
ErrInvalidURI
// Non-AWS errors
ErrExistingObjectIsDirectory
ErrObjectParentIsFile
)
var errorCodeResponse = map[ErrorCode]APIError{
ErrAccessDenied: {
Code: "AccessDenied",
Description: "Access Denied.",
HTTPStatusCode: http.StatusForbidden,
},
ErrMethodNotAllowed: {
Code: "MethodNotAllowed",
Description: "The specified method is not allowed against this resource.",
HTTPStatusCode: http.StatusMethodNotAllowed,
},
ErrBucketNotEmpty: {
Code: "BucketNotEmpty",
Description: "The bucket you tried to delete is not empty",
HTTPStatusCode: http.StatusConflict,
},
ErrBucketAlreadyExists: {
Code: "BucketAlreadyExists",
Description: "The requested bucket name is not available. The bucket name can not be an existing collection, and the bucket namespace is shared by all users of the system. Please select a different name and try again.",
HTTPStatusCode: http.StatusConflict,
},
ErrBucketAlreadyOwnedByYou: {
Code: "BucketAlreadyOwnedByYou",
Description: "Your previous request to create the named bucket succeeded and you already own it.",
HTTPStatusCode: http.StatusConflict,
},
ErrInvalidBucketName: {
Code: "InvalidBucketName",
Description: "The specified bucket is not valid.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidDigest: {
Code: "InvalidDigest",
Description: "The Content-Md5 you specified is not valid.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMaxUploads: {
Code: "InvalidArgument",
Description: "Argument max-uploads must be an integer between 0 and 2147483647",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMaxKeys: {
Code: "InvalidArgument",
Description: "Argument maxKeys must be an integer between 0 and 2147483647",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMaxParts: {
Code: "InvalidArgument",
Description: "Argument max-parts must be an integer between 0 and 2147483647",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidPartNumberMarker: {
Code: "InvalidArgument",
Description: "Argument partNumberMarker must be an integer.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrNoSuchBucket: {
Code: "NoSuchBucket",
Description: "The specified bucket does not exist",
HTTPStatusCode: http.StatusNotFound,
},
ErrNoSuchKey: {
Code: "NoSuchKey",
Description: "The specified key does not exist.",
HTTPStatusCode: http.StatusNotFound,
},
ErrNoSuchUpload: {
Code: "NoSuchUpload",
Description: "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
HTTPStatusCode: http.StatusNotFound,
},
ErrInternalError: {
Code: "InternalError",
Description: "We encountered an internal error, please try again.",
HTTPStatusCode: http.StatusInternalServerError,
},
ErrInvalidPart: {
Code: "InvalidPart",
Description: "One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidCopyDest: {
Code: "InvalidRequest",
Description: "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidCopySource: {
Code: "InvalidArgument",
Description: "Copy Source must mention the source bucket and key: sourcebucket/sourcekey.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidTag: {
Code: "InvalidArgument",
Description: "The Tag value you have provided is invalid",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMalformedXML: {
Code: "MalformedXML",
Description: "The XML you provided was not well-formed or did not validate against our published schema.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrAuthHeaderEmpty: {
Code: "InvalidArgument",
Description: "Authorization header is invalid -- one and only one ' ' (space) required.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrSignatureVersionNotSupported: {
Code: "InvalidRequest",
Description: "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMalformedPOSTRequest: {
Code: "MalformedPOSTRequest",
Description: "The body of your POST request is not well-formed multipart/form-data.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrPOSTFileRequired: {
Code: "InvalidArgument",
Description: "POST requires exactly one file upload per request.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrPostPolicyConditionInvalidFormat: {
Code: "PostPolicyInvalidKeyName",
Description: "Invalid according to Policy: Policy Condition failed",
HTTPStatusCode: http.StatusForbidden,
},
ErrEntityTooSmall: {
Code: "EntityTooSmall",
Description: "Your proposed upload is smaller than the minimum allowed object size.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrEntityTooLarge: {
Code: "EntityTooLarge",
Description: "Your proposed upload exceeds the maximum allowed object size.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMissingFields: {
Code: "MissingFields",
Description: "Missing fields in request.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMissingCredTag: {
Code: "InvalidRequest",
Description: "Missing Credential field for this request.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrCredMalformed: {
Code: "AuthorizationQueryParametersError",
Description: "Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting \"<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request\".",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMalformedDate: {
Code: "MalformedDate",
Description: "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMalformedPresignedDate: {
Code: "AuthorizationQueryParametersError",
Description: "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\"",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMissingSignHeadersTag: {
Code: "InvalidArgument",
Description: "Signature header missing SignedHeaders field.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMissingSignTag: {
Code: "AccessDenied",
Description: "Signature header missing Signature field.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrUnsignedHeaders: {
Code: "AccessDenied",
Description: "There were headers present in the request which were not signed",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidQueryParams: {
Code: "AuthorizationQueryParametersError",
Description: "Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidQuerySignatureAlgo: {
Code: "AuthorizationQueryParametersError",
Description: "X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\".",
HTTPStatusCode: http.StatusBadRequest,
},
ErrExpiredPresignRequest: {
Code: "AccessDenied",
Description: "Request has expired",
HTTPStatusCode: http.StatusForbidden,
},
ErrMalformedExpires: {
Code: "AuthorizationQueryParametersError",
Description: "X-Amz-Expires should be a number",
HTTPStatusCode: http.StatusBadRequest,
},
ErrNegativeExpires: {
Code: "AuthorizationQueryParametersError",
Description: "X-Amz-Expires must be non-negative",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMaximumExpires: {
Code: "AuthorizationQueryParametersError",
Description: "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidAccessKeyID: {
Code: "InvalidAccessKeyId",
Description: "The access key ID you provided does not exist in our records.",
HTTPStatusCode: http.StatusForbidden,
},
ErrRequestNotReadyYet: {
Code: "AccessDenied",
Description: "Request is not valid yet",
HTTPStatusCode: http.StatusForbidden,
},
ErrSignatureDoesNotMatch: {
Code: "SignatureDoesNotMatch",
Description: "The request signature we calculated does not match the signature you provided. Check your key and signing method.",
HTTPStatusCode: http.StatusForbidden,
},
ErrSignatureDateDoesNotMatch: {
Code: "SignatureDoesNotMatch",
Description: "Date in Credential scope does not match YYYYMMDD from ISO-8601 version of date from HTTP",
HTTPStatusCode: http.StatusForbidden,
},
ErrSignatureTerminationStr: {
Code: "SignatureDoesNotMatch",
Description: "Credential should be scoped with a valid terminator: 'aws4_request'",
HTTPStatusCode: http.StatusForbidden,
},
ErrSignatureIncorrService: {
Code: "SignatureDoesNotMatch",
Description: "Credential should be scoped to correct service: s3",
HTTPStatusCode: http.StatusForbidden,
},
ErrContentSHA256Mismatch: {
Code: "XAmzContentSHA256Mismatch",
Description: "The provided 'x-amz-content-sha256' header does not match what was computed.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMissingDateHeader: {
Code: "AccessDenied",
Description: "AWS authentication requires a valid Date or x-amz-date header",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidRequest: {
Code: "InvalidRequest",
Description: "Invalid Request",
HTTPStatusCode: http.StatusBadRequest,
},
ErrAuthNotSetup: {
Code: "InvalidRequest",
Description: "Signed request requires setting up SeaweedFS S3 authentication",
HTTPStatusCode: http.StatusBadRequest,
},
ErrNotImplemented: {
Code: "NotImplemented",
Description: "A header you provided implies functionality that is not implemented",
HTTPStatusCode: http.StatusNotImplemented,
},
ErrPreconditionFailed: {
Code: "PreconditionFailed",
Description: "At least one of the pre-conditions you specified did not hold",
HTTPStatusCode: http.StatusPreconditionFailed,
},
ErrInvalidObjectState: {
Code: "InvalidObjectState",
Description: "The operation is not valid for the current state of the object",
HTTPStatusCode: http.StatusForbidden,
},
ErrInvalidRange: {
Code: "InvalidRange",
Description: "The requested range is not valid for the request. Try another range.",
HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
},
ErrInvalidURI: {
Code: "InvalidURI",
Description: "The specified URI couldn't be parsed.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrExistingObjectIsDirectory: {
Code: "ExistingObjectIsDirectory",
Description: "Existing Object is a directory.",
HTTPStatusCode: http.StatusConflict,
},
ErrObjectParentIsFile: {
Code: "ObjectParentIsFile",
Description: "Object parent already exists as a file.",
HTTPStatusCode: http.StatusConflict,
},
}
// GetAPIError provides API Error for input API error code.
func GetAPIError(code ErrorCode) APIError {
return errorCodeResponse[code]
}
// getErrorResponse gets in standard error and resource value and
// provides a encodable populated response values
func GetAPIErrorResponse(err APIError, resource, requestID, hostID string) []byte {
return encodeResponse(APIErrorResponse{
Code: err.Code,
Message: err.Description,
BucketName: "",
Key: "",
Resource: resource,
Region: "",
RequestID: requestID,
HostID: hostID,
})
}
// Encodes the response headers into XML format.
func encodeResponse(response interface{}) []byte {
var bytesBuffer bytes.Buffer
bytesBuffer.WriteString(xml.Header)
e := xml.NewEncoder(&bytesBuffer)
e.Encode(response)
return bytesBuffer.Bytes()
}

View File

@@ -1,130 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3event
import (
"fmt"
"github.com/gofiber/fiber/v2"
)
type S3EventSender interface {
SendEvent(ctx *fiber.Ctx, meta EventMeta)
}
type EventMeta struct {
BucketOwner string
EventName EventType
ObjectSize int64
ObjectETag *string
VersionId *string
}
type EventFields struct {
Records []EventSchema
}
type EventType string
const (
EventObjectPut EventType = "s3:ObjectCreated:Put"
EventObjectCopy EventType = "s3:ObjectCreated:Copy"
EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
EventObjectDelete EventType = "s3:ObjectRemoved:Delete"
EventObjectRestoreCompleted EventType = "s3:ObjectRestore:Completed"
EventObjectTaggingPut EventType = "s3:ObjectTagging:Put"
EventObjectTaggingDelete EventType = "s3:ObjectTagging:Delete"
EventObjectAclPut EventType = "s3:ObjectAcl:Put"
// Not supported
// EventObjectRestorePost EventType = "s3:ObjectRestore:Post"
// EventObjectRestoreDelete EventType = "s3:ObjectRestore:Delete"
)
type EventSchema struct {
EventVersion string `json:"eventVersion"`
EventSource string `json:"eventSource"`
AwsRegion string `json:"awsRegion"`
EventTime string `json:"eventTime"`
EventName EventType `json:"eventName"`
UserIdentity EventUserIdentity `json:"userIdentity"`
RequestParameters EventRequestParams `json:"requestParameters"`
ResponseElements EventResponseElements `json:"responseElements"`
S3 EventS3Data `json:"s3"`
GlacierEventData EventGlacierData `json:"glacierEventData"`
}
type EventUserIdentity struct {
PrincipalId string `json:"PrincipalId"`
}
type EventRequestParams struct {
SourceIPAddress string `json:"sourceIPAddress"`
}
type EventResponseElements struct {
RequestId string `json:"x-amz-request-id"`
HostId string `json:"x-amz-id-2"`
}
type EventS3Data struct {
S3SchemaVersion string `json:"s3SchemaVersion"`
ConfigurationId string `json:"configurationId"`
Bucket EventS3BucketData `json:"bucket"`
Object EventObjectData `json:"object"`
}
type EventGlacierData struct {
RestoreEventData EventRestoreData `json:"restoreEventData"`
}
type EventRestoreData struct {
LifecycleRestorationExpiryTime string `json:"lifecycleRestorationExpiryTime"`
LifecycleRestoreStorageClass string `json:"lifecycleRestoreStorageClass"`
}
type EventS3BucketData struct {
Name string `json:"name"`
OwnerIdentity EventUserIdentity `json:"ownerIdentity"`
Arn string `json:"arn"`
}
type EventObjectData struct {
Key string `json:"key"`
Size int64 `json:"size"`
ETag *string `json:"eTag"`
VersionId *string `json:"versionId"`
Sequencer string `json:"sequencer"`
}
type EventConfig struct {
KafkaURL string
KafkaTopic string
KafkaTopicKey string
NatsURL string
NatsTopic string
}
func InitEventSender(cfg *EventConfig) (S3EventSender, error) {
if cfg.KafkaURL != "" && cfg.NatsURL != "" {
return nil, fmt.Errorf("there should be specified one of the following: kafka, nats")
}
if cfg.NatsURL != "" {
return InitNatsEventService(cfg.NatsURL, cfg.NatsTopic)
}
if cfg.KafkaURL != "" {
return InitKafkaEventService(cfg.KafkaURL, cfg.KafkaTopic, cfg.KafkaTopicKey)
}
return nil, nil
}

View File

@@ -1,153 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3event
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"time"
"github.com/gofiber/fiber/v2"
"github.com/segmentio/kafka-go"
)
var sequencer = 0
type Kafka struct {
key string
writer *kafka.Writer
mu sync.Mutex
}
func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
if topic == "" {
return nil, fmt.Errorf("kafka message topic should be specified")
}
w := kafka.NewWriter(kafka.WriterConfig{
Brokers: []string{url},
Topic: topic,
Balancer: &kafka.LeastBytes{},
BatchTimeout: 5 * time.Millisecond,
})
msg := map[string]string{
"Service": "S3",
"Event": "s3:TestEvent",
"Time": time.Now().Format(time.RFC3339),
"Bucket": "Test-Bucket",
}
msgJSON, err := json.Marshal(msg)
if err != nil {
return nil, err
}
message := kafka.Message{
Key: []byte(key),
Value: msgJSON,
}
ctx := context.Background()
err = w.WriteMessages(ctx, message)
if err != nil {
return nil, err
}
return &Kafka{
key: key,
writer: w,
}, nil
}
func (ks *Kafka) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
ks.mu.Lock()
defer ks.mu.Unlock()
path := strings.Split(ctx.Path(), "/")
bucket, object := path[1], strings.Join(path[2:], "/")
schema := EventSchema{
EventVersion: "2.2",
EventSource: "aws:s3",
AwsRegion: ctx.Locals("region").(string),
EventTime: time.Now().Format(time.RFC3339),
EventName: meta.EventName,
UserIdentity: EventUserIdentity{
PrincipalId: ctx.Locals("access").(string),
},
RequestParameters: EventRequestParams{
SourceIPAddress: ctx.IP(),
},
ResponseElements: EventResponseElements{
RequestId: ctx.Get("X-Amz-Request-Id"),
HostId: ctx.Get("X-Amx-Id-2"),
},
S3: EventS3Data{
S3SchemaVersion: "1.0",
// This field will come up after implementing per bucket notifications
ConfigurationId: "kafka-global",
Bucket: EventS3BucketData{
Name: bucket,
OwnerIdentity: EventUserIdentity{
PrincipalId: ctx.Locals("access").(string),
},
Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
},
Object: EventObjectData{
Key: object,
Size: meta.ObjectSize,
ETag: meta.ObjectETag,
VersionId: meta.VersionId,
Sequencer: genSequencer(),
},
},
GlacierEventData: EventGlacierData{
// Not supported
RestoreEventData: EventRestoreData{},
},
}
ks.send([]EventSchema{schema})
}
func (ks *Kafka) send(evnt []EventSchema) {
msg, err := json.Marshal(evnt)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
return
}
message := kafka.Message{
Key: []byte(ks.key),
Value: msg,
}
ctx := context.Background()
err = ks.writer.WriteMessages(ctx, message)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to send kafka event: %v\n", err.Error())
}
}
func genSequencer() string {
sequencer = sequencer + 1
return fmt.Sprintf("%X", sequencer)
}

View File

@@ -1,112 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3event
import (
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"time"
"github.com/gofiber/fiber/v2"
"github.com/nats-io/nats.go"
)
type NatsEventSender struct {
topic string
client *nats.Conn
mu sync.Mutex
}
func InitNatsEventService(url, topic string) (S3EventSender, error) {
if topic == "" {
return nil, fmt.Errorf("nats message topic should be specified")
}
client, err := nats.Connect(url)
if err != nil {
return nil, err
}
return &NatsEventSender{
topic: topic,
client: client,
}, nil
}
func (ns *NatsEventSender) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
ns.mu.Lock()
defer ns.mu.Unlock()
path := strings.Split(ctx.Path(), "/")
bucket, object := path[1], strings.Join(path[2:], "/")
schema := EventSchema{
EventVersion: "2.2",
EventSource: "aws:s3",
AwsRegion: ctx.Locals("region").(string),
EventTime: time.Now().Format(time.RFC3339),
EventName: meta.EventName,
UserIdentity: EventUserIdentity{
PrincipalId: ctx.Locals("access").(string),
},
RequestParameters: EventRequestParams{
SourceIPAddress: ctx.IP(),
},
ResponseElements: EventResponseElements{
RequestId: ctx.Get("X-Amz-Request-Id"),
HostId: ctx.Get("X-Amx-Id-2"),
},
S3: EventS3Data{
S3SchemaVersion: "1.0",
// This field will come up after implementing per bucket notifications
ConfigurationId: "nats-global",
Bucket: EventS3BucketData{
Name: bucket,
OwnerIdentity: EventUserIdentity{
PrincipalId: ctx.Locals("access").(string),
},
Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
},
Object: EventObjectData{
Key: object,
Size: meta.ObjectSize,
ETag: meta.ObjectETag,
VersionId: meta.VersionId,
Sequencer: genSequencer(),
},
},
GlacierEventData: EventGlacierData{
// Not supported
RestoreEventData: EventRestoreData{},
},
}
ns.send([]EventSchema{schema})
}
func (ns *NatsEventSender) send(evnt []EventSchema) {
msg, err := json.Marshal(evnt)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
}
err = ns.client.Publish(ns.topic, msg)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to send nats event: %v\n", err.Error())
}
}

View File

@@ -1,112 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3log
import (
"crypto/tls"
"encoding/hex"
"fmt"
"math/rand"
"strings"
"time"
"github.com/gofiber/fiber/v2"
)
type AuditLogger interface {
Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta)
HangUp() error
Shutdown() error
}
type LogMeta struct {
BucketOwner string
ObjectSize int64
Action string
}
type LogConfig struct {
LogFile string
WebhookURL string
}
type LogFields struct {
BucketOwner string
Bucket string
Time time.Time
RemoteIP string
Requester string
RequestID string
Operation string
Key string
RequestURI string
HttpStatus int
ErrorCode string
BytesSent int
ObjectSize int64
TotalTime int64
TurnAroundTime int64
Referer string
UserAgent string
VersionID string
HostID string
SignatureVersion string
CipherSuite string
AuthenticationType string
HostHeader string
TLSVersion string
AccessPointARN string
AclRequired string
}
func InitLogger(cfg *LogConfig) (AuditLogger, error) {
if cfg.WebhookURL != "" && cfg.LogFile != "" {
return nil, fmt.Errorf("there should be specified one of the following: file, webhook")
}
if cfg.WebhookURL != "" {
return InitWebhookLogger(cfg.WebhookURL)
}
if cfg.LogFile != "" {
return InitFileLogger(cfg.LogFile)
}
return nil, nil
}
func genID() string {
src := rand.New(rand.NewSource(time.Now().UnixNano()))
b := make([]byte, 8)
if _, err := src.Read(b); err != nil {
panic(err)
}
return strings.ToUpper(hex.EncodeToString(b))
}
func getTLSVersionName(version uint16) string {
switch version {
case tls.VersionTLS10:
return "TLSv1.0"
case tls.VersionTLS11:
return "TLSv1.1"
case tls.VersionTLS12:
return "TLSv1.2"
case tls.VersionTLS13:
return "TLSv1.3"
default:
return ""
}
}

View File

@@ -1,230 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3log
import (
"crypto/tls"
"fmt"
"os"
"strings"
"sync"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3err"
)
const (
logFileMode = 0600
timeFormat = "02/January/2006:15:04:05 -0700"
)
// FileLogger is a local file audit log
type FileLogger struct {
logfile string
f *os.File
gotErr bool
mu sync.Mutex
}
var _ AuditLogger = &FileLogger{}
// InitFileLogger initializes audit logs to local file
func InitFileLogger(logname string) (AuditLogger, error) {
f, err := os.OpenFile(logname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return nil, fmt.Errorf("open log: %w", err)
}
f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
return &FileLogger{logfile: logname, f: f}, nil
}
// Log sends log message to file logger
func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
f.mu.Lock()
defer f.mu.Unlock()
if f.gotErr {
return
}
lf := LogFields{}
access := "-"
reqURI := ctx.OriginalURL()
path := strings.Split(ctx.Path(), "/")
bucket, object := path[1], strings.Join(path[2:], "/")
errorCode := ""
httpStatus := 200
startTime := ctx.Locals("startTime").(time.Time)
tlsConnState := ctx.Context().TLSConnectionState()
if tlsConnState != nil {
lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
}
if err != nil {
serr, ok := err.(s3err.APIError)
if ok {
errorCode = serr.Code
httpStatus = serr.HTTPStatusCode
} else {
errorCode = err.Error()
httpStatus = 500
}
}
switch ctx.Locals("access").(type) {
case string:
access = ctx.Locals("access").(string)
}
lf.BucketOwner = meta.BucketOwner
lf.Bucket = bucket
lf.Time = time.Now()
lf.RemoteIP = ctx.IP()
lf.Requester = access
lf.RequestID = genID()
lf.Operation = meta.Action
lf.Key = object
lf.RequestURI = reqURI
lf.HttpStatus = httpStatus
lf.ErrorCode = errorCode
lf.BytesSent = len(body)
lf.ObjectSize = meta.ObjectSize
lf.TotalTime = time.Since(startTime).Milliseconds()
lf.TurnAroundTime = time.Since(startTime).Milliseconds()
lf.Referer = ctx.Get("Referer")
lf.UserAgent = ctx.Get("User-Agent")
lf.VersionID = ctx.Query("versionId")
lf.HostID = ctx.Get("X-Amz-Id-2")
lf.SignatureVersion = "SigV4"
lf.AuthenticationType = "AuthHeader"
lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
lf.AclRequired = "Yes"
f.writeLog(lf)
}
func (f *FileLogger) writeLog(lf LogFields) {
if lf.BucketOwner == "" {
lf.BucketOwner = "-"
}
if lf.Bucket == "" {
lf.Bucket = "-"
}
if lf.RemoteIP == "" {
lf.RemoteIP = "-"
}
if lf.Requester == "" {
lf.Requester = "-"
}
if lf.Operation == "" {
lf.Operation = "-"
}
if lf.Key == "" {
lf.Key = "-"
}
if lf.RequestURI == "" {
lf.RequestURI = "-"
}
if lf.ErrorCode == "" {
lf.ErrorCode = "-"
}
if lf.Referer == "" {
lf.Referer = "-"
}
if lf.UserAgent == "" {
lf.UserAgent = "-"
}
if lf.VersionID == "" {
lf.VersionID = "-"
}
if lf.HostID == "" {
lf.HostID = "-"
}
if lf.CipherSuite == "" {
lf.CipherSuite = "-"
}
if lf.HostHeader == "" {
lf.HostHeader = "-"
}
if lf.TLSVersion == "" {
lf.TLSVersion = "-"
}
log := fmt.Sprintf("%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v\n",
lf.BucketOwner,
lf.Bucket,
fmt.Sprintf("[%v]", lf.Time.Format(timeFormat)),
lf.RemoteIP,
lf.Requester,
lf.RequestID,
lf.Operation,
lf.Key,
lf.RequestURI,
lf.HttpStatus,
lf.ErrorCode,
lf.BytesSent,
lf.ObjectSize,
lf.TotalTime,
lf.TurnAroundTime,
lf.Referer,
lf.UserAgent,
lf.VersionID,
lf.HostID,
lf.SignatureVersion,
lf.CipherSuite,
lf.AuthenticationType,
lf.HostHeader,
lf.TLSVersion,
lf.AccessPointARN,
lf.AclRequired,
)
_, err := f.f.WriteString(log)
if err != nil {
fmt.Fprintf(os.Stderr, "error writing to log file: %v\n", err)
// TODO: do we need to terminate on log error?
// set err for now so that we don't spew errors
f.gotErr = true
}
}
// HangUp closes current logfile handle and opens a new one
// typically needed for log rotations
func (f *FileLogger) HangUp() error {
err := f.f.Close()
if err != nil {
return fmt.Errorf("close log: %w", err)
}
f.f, err = os.OpenFile(f.logfile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return fmt.Errorf("open log: %w", err)
}
f.f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
return nil
}
// Shutdown closes logfile handle
func (f *FileLogger) Shutdown() error {
return f.f.Close()
}

View File

@@ -1,156 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3log
import (
"bytes"
"crypto/tls"
"encoding/json"
"fmt"
"net"
"net/http"
"os"
"strings"
"sync"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3err"
)
// WebhookLogger is a webhook URL audit log
type WebhookLogger struct {
mu sync.Mutex
url string
}
var _ AuditLogger = &WebhookLogger{}
// InitWebhookLogger initializes audit logs to webhook URL
func InitWebhookLogger(url string) (AuditLogger, error) {
client := &http.Client{
Timeout: 3 * time.Second,
}
_, err := client.Post(url, "application/json", nil)
if err != nil {
if err, ok := err.(net.Error); ok && !err.Timeout() {
return nil, fmt.Errorf("unreachable webhook url: %w", err)
}
}
return &WebhookLogger{
url: url,
}, nil
}
// Log sends log message to webhook
func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
wl.mu.Lock()
defer wl.mu.Unlock()
lf := LogFields{}
access := "-"
reqURI := ctx.OriginalURL()
path := strings.Split(ctx.Path(), "/")
bucket, object := path[1], strings.Join(path[2:], "/")
errorCode := ""
httpStatus := 200
startTime := ctx.Locals("startTime").(time.Time)
tlsConnState := ctx.Context().TLSConnectionState()
if tlsConnState != nil {
lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
}
if err != nil {
serr, ok := err.(s3err.APIError)
if ok {
errorCode = serr.Code
httpStatus = serr.HTTPStatusCode
} else {
errorCode = err.Error()
httpStatus = 500
}
}
switch ctx.Locals("access").(type) {
case string:
access = ctx.Locals("access").(string)
}
lf.BucketOwner = meta.BucketOwner
lf.Bucket = bucket
lf.Time = time.Now()
lf.RemoteIP = ctx.IP()
lf.Requester = access
lf.RequestID = genID()
lf.Operation = meta.Action
lf.Key = object
lf.RequestURI = reqURI
lf.HttpStatus = httpStatus
lf.ErrorCode = errorCode
lf.BytesSent = len(body)
lf.ObjectSize = meta.ObjectSize
lf.TotalTime = time.Since(startTime).Milliseconds()
lf.TurnAroundTime = time.Since(startTime).Milliseconds()
lf.Referer = ctx.Get("Referer")
lf.UserAgent = ctx.Get("User-Agent")
lf.VersionID = ctx.Query("versionId")
lf.HostID = ctx.Get("X-Amz-Id-2")
lf.SignatureVersion = "SigV4"
lf.AuthenticationType = "AuthHeader"
lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
lf.AclRequired = "Yes"
wl.sendLog(lf)
}
func (wl *WebhookLogger) sendLog(lf LogFields) {
jsonLog, err := json.Marshal(lf)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to parse the log data: %v\n", err.Error())
}
req, err := http.NewRequest(http.MethodPost, wl.url, bytes.NewReader(jsonLog))
if err != nil {
fmt.Fprintln(os.Stderr, err)
}
req.Header.Set("Content-Type", "application/json; charset=utf-8")
go makeRequest(req)
}
func makeRequest(req *http.Request) {
client := &http.Client{
Timeout: 1 * time.Second,
}
_, err := client.Do(req)
if err != nil {
if err, ok := err.(net.Error); ok && !err.Timeout() {
fmt.Fprintf(os.Stderr, "error sending webhook log: %v\n", err)
}
}
}
// HangUp does nothing for webhooks
func (wl *WebhookLogger) HangUp() error {
return nil
}
// Shutdown does nothing for webhooks
func (wl *WebhookLogger) Shutdown() error {
return nil
}

View File

@@ -1,692 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<xsd:schema
xmlns:tns="http://s3.amazonaws.com/doc/2006-03-01/"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
elementFormDefault="qualified"
targetNamespace="http://s3.amazonaws.com/doc/2006-03-01/">
<xsd:element name="CreateBucket">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="MetadataEntry">
<xsd:sequence>
<xsd:element name="Name" type="xsd:string"/>
<xsd:element name="Value" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="CreateBucketResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="CreateBucketReturn" type="tns:CreateBucketResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="Status">
<xsd:sequence>
<xsd:element name="Code" type="xsd:int"/>
<xsd:element name="Description" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="Result">
<xsd:sequence>
<xsd:element name="Status" type="tns:Status"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="CreateBucketResult">
<xsd:sequence>
<xsd:element name="BucketName" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="DeleteBucket">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="DeleteBucketResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="DeleteBucketResponse" type="tns:Status"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="BucketLoggingStatus">
<xsd:sequence>
<xsd:element name="LoggingEnabled" type="tns:LoggingSettings" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="LoggingSettings">
<xsd:sequence>
<xsd:element name="TargetBucket" type="xsd:string"/>
<xsd:element name="TargetPrefix" type="xsd:string"/>
<xsd:element name="TargetGrants" type="tns:AccessControlList" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="GetBucketLoggingStatus">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetBucketLoggingStatusResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="GetBucketLoggingStatusResponse" type="tns:BucketLoggingStatus"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="SetBucketLoggingStatus">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
<xsd:element name="BucketLoggingStatus" type="tns:BucketLoggingStatus"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="SetBucketLoggingStatusResponse">
<xsd:complexType>
<xsd:sequence/>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetObjectAccessControlPolicy">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetObjectAccessControlPolicyResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="GetObjectAccessControlPolicyResponse" type="tns:AccessControlPolicy"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetBucketAccessControlPolicy">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetBucketAccessControlPolicyResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="GetBucketAccessControlPolicyResponse" type="tns:AccessControlPolicy"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType abstract="true" name="Grantee"/>
<xsd:complexType name="User" abstract="true">
<xsd:complexContent>
<xsd:extension base="tns:Grantee"/>
</xsd:complexContent>
</xsd:complexType>
<xsd:complexType name="AmazonCustomerByEmail">
<xsd:complexContent>
<xsd:extension base="tns:User">
<xsd:sequence>
<xsd:element name="EmailAddress" type="xsd:string"/>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:complexType name="CanonicalUser">
<xsd:complexContent>
<xsd:extension base="tns:User">
<xsd:sequence>
<xsd:element name="ID" type="xsd:string"/>
<xsd:element name="DisplayName" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:complexType name="Group">
<xsd:complexContent>
<xsd:extension base="tns:Grantee">
<xsd:sequence>
<xsd:element name="URI" type="xsd:string"/>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:simpleType name="Permission">
<xsd:restriction base="xsd:string">
<xsd:enumeration value="READ"/>
<xsd:enumeration value="WRITE"/>
<xsd:enumeration value="READ_ACP"/>
<xsd:enumeration value="WRITE_ACP"/>
<xsd:enumeration value="FULL_CONTROL"/>
</xsd:restriction>
</xsd:simpleType>
<xsd:simpleType name="StorageClass">
<xsd:restriction base="xsd:string">
<xsd:enumeration value="STANDARD"/>
<xsd:enumeration value="REDUCED_REDUNDANCY"/>
<xsd:enumeration value="GLACIER"/>
<xsd:enumeration value="UNKNOWN"/>
</xsd:restriction>
</xsd:simpleType>
<xsd:complexType name="Grant">
<xsd:sequence>
<xsd:element name="Grantee" type="tns:Grantee"/>
<xsd:element name="Permission" type="tns:Permission"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="AccessControlList">
<xsd:sequence>
<xsd:element name="Grant" type="tns:Grant" minOccurs="0" maxOccurs="100"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="CreateBucketConfiguration">
<xsd:sequence>
<xsd:element name="LocationConstraint" type="tns:LocationConstraint"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="LocationConstraint">
<xsd:simpleContent>
<xsd:extension base="xsd:string"/>
</xsd:simpleContent>
</xsd:complexType>
<xsd:complexType name="AccessControlPolicy">
<xsd:sequence>
<xsd:element name="Owner" type="tns:CanonicalUser"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="SetObjectAccessControlPolicy">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="SetObjectAccessControlPolicyResponse">
<xsd:complexType>
<xsd:sequence/>
</xsd:complexType>
</xsd:element>
<xsd:element name="SetBucketAccessControlPolicy">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="SetBucketAccessControlPolicyResponse">
<xsd:complexType>
<xsd:sequence/>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetObject">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="GetMetadata" type="xsd:boolean"/>
<xsd:element name="GetData" type="xsd:boolean"/>
<xsd:element name="InlineData" type="xsd:boolean"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetObjectResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="GetObjectResponse" type="tns:GetObjectResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="GetObjectResult">
<xsd:complexContent>
<xsd:extension base="tns:Result">
<xsd:sequence>
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="unbounded"/>
<xsd:element name="Data" type="xsd:base64Binary" nillable="true"/>
<xsd:element name="LastModified" type="xsd:dateTime"/>
<xsd:element name="ETag" type="xsd:string"/>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:element name="GetObjectExtended">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="GetMetadata" type="xsd:boolean"/>
<xsd:element name="GetData" type="xsd:boolean"/>
<xsd:element name="InlineData" type="xsd:boolean"/>
<xsd:element name="ByteRangeStart" type="xsd:long" minOccurs="0"/>
<xsd:element name="ByteRangeEnd" type="xsd:long" minOccurs="0"/>
<xsd:element name="IfModifiedSince" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="IfUnmodifiedSince" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="IfMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
<xsd:element name="IfNoneMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
<xsd:element name="ReturnCompleteObjectOnConditionFailure" type="xsd:boolean" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetObjectExtendedResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="GetObjectResponse" type="tns:GetObjectResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="PutObject">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="100"/>
<xsd:element name="ContentLength" type="xsd:long"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
<xsd:element name="StorageClass" type="tns:StorageClass" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="PutObjectResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="PutObjectResponse" type="tns:PutObjectResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="PutObjectResult">
<xsd:sequence>
<xsd:element name="ETag" type="xsd:string"/>
<xsd:element name="LastModified" type="xsd:dateTime"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="PutObjectInline">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element minOccurs="0" maxOccurs="100" name="Metadata" type="tns:MetadataEntry"/>
<xsd:element name="Data" type="xsd:base64Binary"/>
<xsd:element name="ContentLength" type="xsd:long"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
<xsd:element name="StorageClass" type="tns:StorageClass" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="PutObjectInlineResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="PutObjectInlineResponse" type="tns:PutObjectResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="DeleteObject">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="DeleteObjectResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="DeleteObjectResponse" type="tns:Status"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="ListBucket">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Prefix" type="xsd:string" minOccurs="0"/>
<xsd:element name="Marker" type="xsd:string" minOccurs="0"/>
<xsd:element name="MaxKeys" type="xsd:int" minOccurs="0"/>
<xsd:element name="Delimiter" type="xsd:string" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="ListBucketResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="ListBucketResponse" type="tns:ListBucketResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="ListVersionsResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="ListVersionsResponse" type="tns:ListVersionsResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="ListEntry">
<xsd:sequence>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="LastModified" type="xsd:dateTime"/>
<xsd:element name="ETag" type="xsd:string"/>
<xsd:element name="Size" type="xsd:long"/>
<xsd:element name="Owner" type="tns:CanonicalUser" minOccurs="0"/>
<xsd:element name="StorageClass" type="tns:StorageClass"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="VersionEntry">
<xsd:sequence>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="VersionId" type="xsd:string"/>
<xsd:element name="IsLatest" type="xsd:boolean"/>
<xsd:element name="LastModified" type="xsd:dateTime"/>
<xsd:element name="ETag" type="xsd:string"/>
<xsd:element name="Size" type="xsd:long"/>
<xsd:element name="Owner" type="tns:CanonicalUser" minOccurs="0"/>
<xsd:element name="StorageClass" type="tns:StorageClass"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="DeleteMarkerEntry">
<xsd:sequence>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="VersionId" type="xsd:string"/>
<xsd:element name="IsLatest" type="xsd:boolean"/>
<xsd:element name="LastModified" type="xsd:dateTime"/>
<xsd:element name="Owner" type="tns:CanonicalUser" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="PrefixEntry">
<xsd:sequence>
<xsd:element name="Prefix" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="ListBucketResult">
<xsd:sequence>
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="unbounded"/>
<xsd:element name="Name" type="xsd:string"/>
<xsd:element name="Prefix" type="xsd:string"/>
<xsd:element name="Marker" type="xsd:string"/>
<xsd:element name="NextMarker" type="xsd:string" minOccurs="0"/>
<xsd:element name="MaxKeys" type="xsd:int"/>
<xsd:element name="Delimiter" type="xsd:string" minOccurs="0"/>
<xsd:element name="IsTruncated" type="xsd:boolean"/>
<xsd:element name="Contents" type="tns:ListEntry" minOccurs="0" maxOccurs="unbounded"/>
<xsd:element name="CommonPrefixes" type="tns:PrefixEntry" minOccurs="0" maxOccurs="unbounded"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="ListVersionsResult">
<xsd:sequence>
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="unbounded"/>
<xsd:element name="Name" type="xsd:string"/>
<xsd:element name="Prefix" type="xsd:string"/>
<xsd:element name="KeyMarker" type="xsd:string"/>
<xsd:element name="VersionIdMarker" type="xsd:string"/>
<xsd:element name="NextKeyMarker" type="xsd:string" minOccurs="0"/>
<xsd:element name="NextVersionIdMarker" type="xsd:string" minOccurs="0"/>
<xsd:element name="MaxKeys" type="xsd:int"/>
<xsd:element name="Delimiter" type="xsd:string" minOccurs="0"/>
<xsd:element name="IsTruncated" type="xsd:boolean"/>
<xsd:choice minOccurs="0" maxOccurs="unbounded">
<xsd:element name="Version" type="tns:VersionEntry"/>
<xsd:element name="DeleteMarker" type="tns:DeleteMarkerEntry"/>
</xsd:choice>
<xsd:element name="CommonPrefixes" type="tns:PrefixEntry" minOccurs="0" maxOccurs="unbounded"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="ListAllMyBuckets">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="ListAllMyBucketsResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="ListAllMyBucketsResponse" type="tns:ListAllMyBucketsResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="ListAllMyBucketsEntry">
<xsd:sequence>
<xsd:element name="Name" type="xsd:string"/>
<xsd:element name="CreationDate" type="xsd:dateTime"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="ListAllMyBucketsResult">
<xsd:sequence>
<xsd:element name="Owner" type="tns:CanonicalUser"/>
<xsd:element name="Buckets" type="tns:ListAllMyBucketsList"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="ListAllMyBucketsList">
<xsd:sequence>
<xsd:element name="Bucket" type="tns:ListAllMyBucketsEntry" minOccurs="0" maxOccurs="unbounded"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="PostResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Location" type="xsd:anyURI"/>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="ETag" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:simpleType name="MetadataDirective">
<xsd:restriction base="xsd:string">
<xsd:enumeration value="COPY"/>
<xsd:enumeration value="REPLACE"/>
</xsd:restriction>
</xsd:simpleType>
<xsd:element name="CopyObject">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="SourceBucket" type="xsd:string"/>
<xsd:element name="SourceKey" type="xsd:string"/>
<xsd:element name="DestinationBucket" type="xsd:string"/>
<xsd:element name="DestinationKey" type="xsd:string"/>
<xsd:element name="MetadataDirective" type="tns:MetadataDirective" minOccurs="0"/>
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="100"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
<xsd:element name="CopySourceIfModifiedSince" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="CopySourceIfUnmodifiedSince" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="CopySourceIfMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
<xsd:element name="CopySourceIfNoneMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
<xsd:element name="StorageClass" type="tns:StorageClass" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="CopyObjectResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="CopyObjectResult" type="tns:CopyObjectResult" />
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="CopyObjectResult">
<xsd:sequence>
<xsd:element name="LastModified" type="xsd:dateTime"/>
<xsd:element name="ETag" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="RequestPaymentConfiguration">
<xsd:sequence>
<xsd:element name="Payer" type="tns:Payer" minOccurs="1" maxOccurs="1"/>
</xsd:sequence>
</xsd:complexType>
<xsd:simpleType name="Payer">
<xsd:restriction base="xsd:string">
<xsd:enumeration value="BucketOwner"/>
<xsd:enumeration value="Requester"/>
</xsd:restriction>
</xsd:simpleType>
<xsd:complexType name="VersioningConfiguration">
<xsd:sequence>
<xsd:element name="Status" type="tns:VersioningStatus" minOccurs="0"/>
<xsd:element name="MfaDelete" type="tns:MfaDeleteStatus" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
<xsd:simpleType name="MfaDeleteStatus">
<xsd:restriction base="xsd:string">
<xsd:enumeration value="Enabled"/>
<xsd:enumeration value="Disabled"/>
</xsd:restriction>
</xsd:simpleType>
<xsd:simpleType name="VersioningStatus">
<xsd:restriction base="xsd:string">
<xsd:enumeration value="Enabled"/>
<xsd:enumeration value="Suspended"/>
</xsd:restriction>
</xsd:simpleType>
<xsd:complexType name="NotificationConfiguration">
<xsd:sequence>
<xsd:element name="TopicConfiguration" minOccurs="0" maxOccurs="unbounded" type="tns:TopicConfiguration"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="TopicConfiguration">
<xsd:sequence>
<xsd:element name="Topic" minOccurs="1" maxOccurs="1" type="xsd:string"/>
<xsd:element name="Event" minOccurs="1" maxOccurs="unbounded" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
</xsd:schema>

View File

@@ -1,6 +0,0 @@
https://doc.s3.amazonaws.com/2006-03-01/AmazonS3.xsd
see https://blog.aqwari.net/xml-schema-go/
go install aqwari.net/xml/cmd/xsdgen@latest
xsdgen -o s3api_xsd_generated.go -pkg s3response AmazonS3.xsd

File diff suppressed because it is too large Load Diff

View File

@@ -1,141 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3response
import (
"encoding/xml"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
// Part describes part metadata.
type Part struct {
PartNumber int
LastModified string
ETag string
Size int64
}
// ListPartsResponse - s3 api list parts response.
type ListPartsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
Bucket string
Key string
UploadID string `xml:"UploadId"`
Initiator Initiator
Owner Owner
// The class of storage used to store the object.
StorageClass string
PartNumberMarker int
NextPartNumberMarker int
MaxParts int
IsTruncated bool
// List of parts.
Parts []Part `xml:"Part"`
}
// ListMultipartUploadsResponse - s3 api list multipart uploads response.
type ListMultipartUploadsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
Bucket string
KeyMarker string
UploadIDMarker string `xml:"UploadIdMarker"`
NextKeyMarker string
NextUploadIDMarker string `xml:"NextUploadIdMarker"`
Delimiter string
Prefix string
EncodingType string `xml:"EncodingType,omitempty"`
MaxUploads int
IsTruncated bool
// List of pending uploads.
Uploads []Upload `xml:"Upload"`
// Delimed common prefixes.
CommonPrefixes []CommonPrefix
}
// Upload desribes in progress multipart upload
type Upload struct {
Key string
UploadID string `xml:"UploadId"`
Initiator Initiator
Owner Owner
StorageClass string
Initiated string
}
// CommonPrefix ListObjectsResponse common prefixes (directory abstraction)
type CommonPrefix struct {
Prefix string
}
// Initiator same fields as Owner
type Initiator Owner
// Owner bucket ownership
type Owner struct {
ID string
DisplayName string
}
type Tag struct {
Key string `xml:"Key"`
Value string `xml:"Value"`
}
type TagSet struct {
Tags []Tag `xml:"Tag"`
}
type Tagging struct {
TagSet TagSet `xml:"TagSet"`
}
type DeleteObjects struct {
Objects []types.ObjectIdentifier `xml:"Object"`
}
type DeleteObjectsResult struct {
Deleted []types.DeletedObject
Error []types.Error
}
type SelectObjectContentPayload struct {
Expression *string
ExpressionType types.ExpressionType
RequestProgress *types.RequestProgress
InputSerialization *types.InputSerialization
OutputSerialization *types.OutputSerialization
ScanRange *types.ScanRange
}
type SelectObjectContentResult struct {
Records *types.RecordsEvent
Stats *types.StatsEvent
Progress *types.ProgressEvent
Cont *string
End *string
}
type Bucket struct {
Name string `json:"name"`
Owner string `json:"owner"`
}

View File

@@ -1,31 +0,0 @@
%global debug_package %{nil}
%define pkg_version @@VERSION@@
Name: versitygw
Version: %{pkg_version}
Release: 1%{?dist}
Summary: Versity S3 Gateway
License: Apache-2.0
URL: https://github.com/versity/versitygw
Source0: %{name}-%{version}.tar.gz
%description
The S3 gateway is an S3 protocol translator that allows an S3 client
to access the supported backend storage as if it was a native S3 service.
BuildRequires: golang >= 1.20
%prep
%setup
%build
make
%install
mkdir -p %{buildroot}%{_bindir}
install -m 0755 %{name} %{buildroot}%{_bindir}/
%post
%files
%{_bindir}/%{name}