mirror of
https://github.com/versity/versitygw.git
synced 2026-01-24 20:12:01 +00:00
Compare commits
1 Commits
v0.5
...
api-unit-t
| Author | SHA1 | Date | |
|---|---|---|---|
|
8631e10714 |
27
.github/ISSUE_TEMPLATE/bug_report.md
vendored
27
.github/ISSUE_TEMPLATE/bug_report.md
vendored
@@ -1,27 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: bug
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior.
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Server Version**
|
||||
output of
|
||||
```
|
||||
./versitygw -version
|
||||
uname -a
|
||||
```
|
||||
|
||||
**Additional context**
|
||||
Describe s3 client and version if applicable.
|
||||
14
.github/ISSUE_TEMPLATE/feature_request.md
vendored
14
.github/ISSUE_TEMPLATE/feature_request.md
vendored
@@ -1,14 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: enhancement
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
||||
30
.github/workflows/functional.yml
vendored
30
.github/workflows/functional.yml
vendored
@@ -1,30 +0,0 @@
|
||||
name: functional tests
|
||||
on: pull_request
|
||||
jobs:
|
||||
|
||||
build:
|
||||
name: RunTests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: 'stable'
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Get Dependencies
|
||||
run: |
|
||||
go get -v -t -d ./...
|
||||
|
||||
- name: Build and Run
|
||||
run: |
|
||||
make testbin
|
||||
./runtests.sh
|
||||
|
||||
- name: Coverage Report
|
||||
run: |
|
||||
go tool covdata percent -i=/tmp/covdata
|
||||
19
.github/workflows/go.yml
vendored
19
.github/workflows/go.yml
vendored
@@ -8,13 +8,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 'stable'
|
||||
go-version: "1.20"
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v1
|
||||
|
||||
- name: Verify all files pass gofmt formatting
|
||||
run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then gofmt -s -d .; exit 1; fi
|
||||
@@ -23,16 +23,5 @@ jobs:
|
||||
run: |
|
||||
go get -v -t -d ./...
|
||||
|
||||
- name: Build
|
||||
run: make
|
||||
|
||||
- name: Test
|
||||
run: go test -coverprofile profile.txt -race -v -timeout 30s -tags=github ./...
|
||||
|
||||
- name: Install govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
shell: bash
|
||||
|
||||
- name: Run govulncheck
|
||||
run: govulncheck ./...
|
||||
shell: bash
|
||||
run: go test -v -timeout 30s -tags=github ./...
|
||||
|
||||
12
.gitignore
vendored
12
.gitignore
vendored
@@ -7,8 +7,6 @@
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
cmd/versitygw/versitygw
|
||||
/versitygw
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
@@ -24,13 +22,3 @@ go.work
|
||||
|
||||
# ignore IntelliJ directories
|
||||
.idea
|
||||
|
||||
# auto generated VERSION file
|
||||
VERSION
|
||||
|
||||
# build output
|
||||
/versitygw.spec
|
||||
*.tar
|
||||
*.tar.gz
|
||||
**/rand.data
|
||||
/profile.txt
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
We as members, contributors, and leaders pledge to make participation in our
|
||||
community a harassment-free experience for everyone, regardless of age, body
|
||||
size, visible or invisible disability, ethnicity, sex characteristics, gender
|
||||
identity and expression, level of experience, education, socio-economic status,
|
||||
nationality, personal appearance, race, religion, or sexual identity
|
||||
and orientation.
|
||||
|
||||
We pledge to act and interact in ways that contribute to an open, welcoming,
|
||||
diverse, inclusive, and healthy community.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to a positive environment for our
|
||||
community include:
|
||||
|
||||
* Demonstrating empathy and kindness toward other people
|
||||
* Being respectful of differing opinions, viewpoints, and experiences
|
||||
* Giving and gracefully accepting constructive feedback
|
||||
* Accepting responsibility and apologizing to those affected by our mistakes,
|
||||
and learning from the experience
|
||||
* Focusing on what is best not just for us as individuals, but for the
|
||||
overall community
|
||||
|
||||
Examples of unacceptable behavior include:
|
||||
|
||||
* The use of sexualized language or imagery, and sexual attention or
|
||||
advances of any kind
|
||||
* Trolling, insulting or derogatory comments, and personal or political attacks
|
||||
* Public or private harassment
|
||||
* Publishing others' private information, such as a physical or email
|
||||
address, without their explicit permission
|
||||
* Other conduct which could reasonably be considered inappropriate in a
|
||||
professional setting
|
||||
|
||||
## Enforcement Responsibilities
|
||||
|
||||
Community leaders are responsible for clarifying and enforcing our standards of
|
||||
acceptable behavior and will take appropriate and fair corrective action in
|
||||
response to any behavior that they deem inappropriate, threatening, offensive,
|
||||
or harmful.
|
||||
|
||||
Community leaders have the right and responsibility to remove, edit, or reject
|
||||
comments, commits, code, wiki edits, issues, and other contributions that are
|
||||
not aligned to this Code of Conduct, and will communicate reasons for moderation
|
||||
decisions when appropriate.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies within all community spaces, and also applies when
|
||||
an individual is officially representing the community in public spaces.
|
||||
Examples of representing our community include using an official e-mail address,
|
||||
posting via an official social media account, or acting as an appointed
|
||||
representative at an online or offline event.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported to the community leaders responsible for enforcement at
|
||||
versitygw@versity.com.
|
||||
All complaints will be reviewed and investigated promptly and fairly.
|
||||
|
||||
All community leaders are obligated to respect the privacy and security of the
|
||||
reporter of any incident.
|
||||
|
||||
## Enforcement Guidelines
|
||||
|
||||
Community leaders will follow these Community Impact Guidelines in determining
|
||||
the consequences for any action they deem in violation of this Code of Conduct:
|
||||
|
||||
### 1. Correction
|
||||
|
||||
**Community Impact**: Use of inappropriate language or other behavior deemed
|
||||
unprofessional or unwelcome in the community.
|
||||
|
||||
**Consequence**: A private, written warning from community leaders, providing
|
||||
clarity around the nature of the violation and an explanation of why the
|
||||
behavior was inappropriate. A public apology may be requested.
|
||||
|
||||
### 2. Warning
|
||||
|
||||
**Community Impact**: A violation through a single incident or series
|
||||
of actions.
|
||||
|
||||
**Consequence**: A warning with consequences for continued behavior. No
|
||||
interaction with the people involved, including unsolicited interaction with
|
||||
those enforcing the Code of Conduct, for a specified period of time. This
|
||||
includes avoiding interactions in community spaces as well as external channels
|
||||
like social media. Violating these terms may lead to a temporary or
|
||||
permanent ban.
|
||||
|
||||
### 3. Temporary Ban
|
||||
|
||||
**Community Impact**: A serious violation of community standards, including
|
||||
sustained inappropriate behavior.
|
||||
|
||||
**Consequence**: A temporary ban from any sort of interaction or public
|
||||
communication with the community for a specified period of time. No public or
|
||||
private interaction with the people involved, including unsolicited interaction
|
||||
with those enforcing the Code of Conduct, is allowed during this period.
|
||||
Violating these terms may lead to a permanent ban.
|
||||
|
||||
### 4. Permanent Ban
|
||||
|
||||
**Community Impact**: Demonstrating a pattern of violation of community
|
||||
standards, including sustained inappropriate behavior, harassment of an
|
||||
individual, or aggression toward or disparagement of classes of individuals.
|
||||
|
||||
**Consequence**: A permanent ban from any sort of public interaction within
|
||||
the community.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
|
||||
version 2.0, available at
|
||||
https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
|
||||
|
||||
Community Impact Guidelines were inspired by [Mozilla's code of conduct
|
||||
enforcement ladder](https://github.com/mozilla/diversity).
|
||||
|
||||
[homepage]: https://www.contributor-covenant.org
|
||||
|
||||
For answers to common questions about this code of conduct, see the FAQ at
|
||||
https://www.contributor-covenant.org/faq. Translations are available at
|
||||
https://www.contributor-covenant.org/translations.
|
||||
76
Makefile
76
Makefile
@@ -1,76 +0,0 @@
|
||||
# Copyright 2023 Versity Software
|
||||
# This file is licensed under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Go parameters
|
||||
GOCMD=go
|
||||
GOBUILD=$(GOCMD) build
|
||||
GOCLEAN=$(GOCMD) clean
|
||||
GOTEST=$(GOCMD) test
|
||||
|
||||
BIN=versitygw
|
||||
|
||||
VERSION := $(shell if test -e VERSION; then cat VERSION; else git describe --abbrev=0 --tags HEAD; fi)
|
||||
BUILD := $(shell git rev-parse --short HEAD || echo release-rpm)
|
||||
TIME := `date -u '+%Y-%m-%d_%I:%M:%S%p'`
|
||||
|
||||
LDFLAGS=-ldflags "-X=main.Build=$(BUILD) -X=main.BuildTime=$(TIME) -X=main.Version=$(VERSION)"
|
||||
|
||||
all: build
|
||||
|
||||
build: $(BIN)
|
||||
|
||||
.PHONY: $(BIN)
|
||||
$(BIN):
|
||||
$(GOBUILD) $(LDFLAGS) -o $(BIN) cmd/$(BIN)/*.go
|
||||
|
||||
testbin:
|
||||
$(GOBUILD) $(LDFLAGS) -o $(BIN) -cover -race cmd/$(BIN)/*.go
|
||||
|
||||
.PHONY: test
|
||||
test:
|
||||
$(GOTEST) ./...
|
||||
|
||||
.PHONY: check
|
||||
check:
|
||||
# note this requires staticcheck be in your PATH:
|
||||
# export PATH=$PATH:~/go/bin
|
||||
# go install honnef.co/go/tools/cmd/staticcheck@latest
|
||||
staticcheck ./...
|
||||
golint ./...
|
||||
gofmt -s -l .
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(GOCLEAN)
|
||||
|
||||
.PHONY: cleanall
|
||||
cleanall: clean
|
||||
rm -f $(BIN)
|
||||
rm -f versitygw-*.tar
|
||||
rm -f versitygw-*.tar.gz
|
||||
rm -f versitygw.spec
|
||||
|
||||
%.spec: %.spec.in
|
||||
sed -e 's/@@VERSION@@/$(VERSION)/g' < $< > $@+
|
||||
mv $@+ $@
|
||||
|
||||
TARFILE = $(BIN)-$(VERSION).tar
|
||||
|
||||
dist: $(BIN).spec
|
||||
echo $(VERSION) >VERSION
|
||||
git archive --format=tar --prefix $(BIN)-$(VERSION)/ HEAD > $(TARFILE)
|
||||
@ tar rf $(TARFILE) --transform="s@\(.*\)@$(BIN)-$(VERSION)/\1@" $(BIN).spec VERSION
|
||||
rm -f VERSION
|
||||
rm -f $(BIN).spec
|
||||
gzip -f $(TARFILE)
|
||||
63
README.md
63
README.md
@@ -1,63 +0,0 @@
|
||||
# The Versity Gateway:<br/>A High-Performance S3 to Storage System Translation Service
|
||||
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo-white.svg">
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://github.com/versity/versitygw/blob/assets/assets/logo.svg">
|
||||
<a href="https://www.versity.com"><img alt="Versity Software logo image." src="https://github.com/versity/versitygw/blob/assets/assets/logo.svg"></a>
|
||||
</picture>
|
||||
|
||||
[](https://github.com/versity/versitygw/blob/main/LICENSE)
|
||||
|
||||
**Current status:** Alpha, in development not yet suited for production use
|
||||
|
||||
See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
|
||||
|
||||
Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.
|
||||
|
||||
The server translates incoming S3 API requests and transforms them into equivalent operations to the backend service. By leveraging this gateway server, applications can interact with the S3-compatible API on top of already existing storage systems. This project enables leveraging existing infrastructure investments while seamlessly integrating with S3-compatible systems, offering increased flexibility and compatibility in managing data storage.
|
||||
|
||||
The Versity Gateway is focused on performance, simplicity, and expandability. The Versity Gateway is designed with modularity in mind, enabling future extensions to support additional backend storage systems. At present, the Versity Gateway supports any generic POSIX file backend storage and Versity’s open source ScoutFS filesystem.
|
||||
|
||||
The gateway is completely stateless. Multiple Versity Gateway instances may be deployed in a cluster to increase aggregate throughput. The Versity Gateway’s stateless architecture allows any request to be serviced by any gateway thereby distributing workloads and enhancing performance. Load balancers may be used to evenly distribute requests across the cluster of gateways for optimal performance.
|
||||
|
||||
The S3 HTTP(S) server and routing is implemented using the [Fiber](https://gofiber.io) web framework. This framework is actively developed with a focus on performance. S3 API compatibility leverages the official [aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) whenever possible for maximum service compatibility with AWS S3.
|
||||
|
||||
## Getting Started
|
||||
See the [Quickstart](https://github.com/versity/versitygw/wiki/Quickstart) documentation.
|
||||
|
||||
### Run the gateway with posix backend:
|
||||
|
||||
```
|
||||
mkdir /tmp/vgw
|
||||
ROOT_ACCESS_KEY="testuser" ROOT_SECRET_KEY="secret" ./versitygw --port :10000 posix /tmp/vgw
|
||||
```
|
||||
This will enable an S3 server on the current host listening on port 10000 and hosting the directory `/tmp/vgw`.
|
||||
|
||||
To get the usage output, run the following:
|
||||
|
||||
```
|
||||
./versitygw --help
|
||||
```
|
||||
|
||||
The command format is
|
||||
|
||||
```
|
||||
versitygw [global options] command [command options] [arguments...]
|
||||
```
|
||||
The global options are specified before the backend type and the backend options are specified after.
|
||||
|
||||
***
|
||||
|
||||
#### Versity gives you clarity and control over your archival storage, so you can allocate more resources to your core mission.
|
||||
|
||||
### Contact
|
||||
|
||||
info@versity.com <br />
|
||||
+1 844 726 8826
|
||||
|
||||
### @versitysoftware
|
||||
[](https://www.linkedin.com/company/versity/)
|
||||
[](https://twitter.com/VersitySoftware)
|
||||
[](https://www.facebook.com/versitysoftware)
|
||||
[](https://www.instagram.com/versitysoftware/)
|
||||
|
||||
265
auth/acl.go
265
auth/acl.go
@@ -1,265 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
type ACL struct {
|
||||
ACL types.BucketCannedACL
|
||||
Owner string
|
||||
Grantees []Grantee
|
||||
}
|
||||
|
||||
type Grantee struct {
|
||||
Permission types.Permission
|
||||
Access string
|
||||
}
|
||||
|
||||
type GetBucketAclOutput struct {
|
||||
Owner *types.Owner
|
||||
AccessControlList AccessControlList
|
||||
}
|
||||
|
||||
type AccessControlList struct {
|
||||
Grants []types.Grant `xml:"Grant"`
|
||||
}
|
||||
type AccessControlPolicy struct {
|
||||
AccessControlList AccessControlList `xml:"AccessControlList"`
|
||||
Owner types.Owner
|
||||
}
|
||||
|
||||
func ParseACL(data []byte) (ACL, error) {
|
||||
if len(data) == 0 {
|
||||
return ACL{}, nil
|
||||
}
|
||||
|
||||
var acl ACL
|
||||
if err := json.Unmarshal(data, &acl); err != nil {
|
||||
return acl, fmt.Errorf("parse acl: %w", err)
|
||||
}
|
||||
return acl, nil
|
||||
}
|
||||
|
||||
func ParseACLOutput(data []byte) (GetBucketAclOutput, error) {
|
||||
var acl ACL
|
||||
if err := json.Unmarshal(data, &acl); err != nil {
|
||||
return GetBucketAclOutput{}, fmt.Errorf("parse acl: %w", err)
|
||||
}
|
||||
|
||||
grants := []types.Grant{}
|
||||
|
||||
for _, elem := range acl.Grantees {
|
||||
acs := elem.Access
|
||||
grants = append(grants, types.Grant{Grantee: &types.Grantee{ID: &acs}, Permission: elem.Permission})
|
||||
}
|
||||
|
||||
return GetBucketAclOutput{
|
||||
Owner: &types.Owner{
|
||||
ID: &acl.Owner,
|
||||
},
|
||||
AccessControlList: AccessControlList{
|
||||
Grants: grants,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, error) {
|
||||
if input == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
if acl.Owner != *input.AccessControlPolicy.Owner.ID {
|
||||
return nil, s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
// if the ACL is specified, set the ACL, else replace the grantees
|
||||
if input.ACL != "" {
|
||||
acl.ACL = input.ACL
|
||||
acl.Grantees = []Grantee{}
|
||||
} else {
|
||||
grantees := []Grantee{}
|
||||
accs := []string{}
|
||||
|
||||
if input.GrantRead != nil {
|
||||
fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{}
|
||||
|
||||
if *input.GrantFullControl != "" {
|
||||
fullControlList = splitUnique(*input.GrantFullControl, ",")
|
||||
fmt.Println(fullControlList)
|
||||
for _, str := range fullControlList {
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
|
||||
}
|
||||
}
|
||||
if *input.GrantRead != "" {
|
||||
readList = splitUnique(*input.GrantRead, ",")
|
||||
for _, str := range readList {
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
|
||||
}
|
||||
}
|
||||
if *input.GrantReadACP != "" {
|
||||
readACPList = splitUnique(*input.GrantReadACP, ",")
|
||||
for _, str := range readACPList {
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
|
||||
}
|
||||
}
|
||||
if *input.GrantWrite != "" {
|
||||
writeList = splitUnique(*input.GrantWrite, ",")
|
||||
for _, str := range writeList {
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
|
||||
}
|
||||
}
|
||||
if *input.GrantWriteACP != "" {
|
||||
writeACPList = splitUnique(*input.GrantWriteACP, ",")
|
||||
for _, str := range writeACPList {
|
||||
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
|
||||
}
|
||||
}
|
||||
|
||||
accs = append(append(append(append(fullControlList, readList...), writeACPList...), readACPList...), writeList...)
|
||||
} else {
|
||||
cache := make(map[string]bool)
|
||||
for _, grt := range input.AccessControlPolicy.Grants {
|
||||
grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
|
||||
if _, ok := cache[*grt.Grantee.ID]; !ok {
|
||||
cache[*grt.Grantee.ID] = true
|
||||
accs = append(accs, *grt.Grantee.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if the specified accounts exist
|
||||
accList, err := checkIfAccountsExist(accs, iam)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(accList) > 0 {
|
||||
return nil, fmt.Errorf("accounts does not exist: %s", strings.Join(accList, ", "))
|
||||
}
|
||||
|
||||
acl.Grantees = grantees
|
||||
acl.ACL = ""
|
||||
}
|
||||
|
||||
result, err := json.Marshal(acl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func checkIfAccountsExist(accs []string, iam IAMService) ([]string, error) {
|
||||
result := []string{}
|
||||
|
||||
for _, acc := range accs {
|
||||
_, err := iam.GetUserAccount(acc)
|
||||
if err != nil && err != ErrNoSuchUser {
|
||||
return nil, fmt.Errorf("check user account: %w", err)
|
||||
}
|
||||
if err == ErrNoSuchUser {
|
||||
result = append(result, acc)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func splitUnique(s, divider string) []string {
|
||||
elements := strings.Split(s, divider)
|
||||
uniqueElements := make(map[string]bool)
|
||||
result := make([]string, 0, len(elements))
|
||||
|
||||
for _, element := range elements {
|
||||
if _, ok := uniqueElements[element]; !ok {
|
||||
result = append(result, element)
|
||||
uniqueElements[element] = true
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func VerifyACL(acl ACL, bucket, access string, permission types.Permission, isRoot bool) error {
|
||||
if isRoot {
|
||||
return nil
|
||||
}
|
||||
|
||||
if acl.Owner == access {
|
||||
return nil
|
||||
}
|
||||
|
||||
if acl.ACL != "" {
|
||||
if (permission == "READ" || permission == "READ_ACP") && (acl.ACL != "public-read" && acl.ACL != "public-read-write") {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
if (permission == "WRITE" || permission == "WRITE_ACP") && acl.ACL != "public-read-write" {
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
return nil
|
||||
} else {
|
||||
grantee := Grantee{Access: access, Permission: permission}
|
||||
granteeFullCtrl := Grantee{Access: access, Permission: "FULL_CONTROL"}
|
||||
|
||||
isFound := false
|
||||
|
||||
for _, grt := range acl.Grantees {
|
||||
if grt == grantee || grt == granteeFullCtrl {
|
||||
isFound = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if isFound {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
|
||||
func IsAdmin(access string, isRoot bool) error {
|
||||
var data IAMConfig
|
||||
|
||||
if isRoot {
|
||||
return nil
|
||||
}
|
||||
|
||||
file, err := os.ReadFile("users.json")
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read config file: %w", err)
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(file, &data); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
acc, ok := data.AccessAccounts[access]
|
||||
if !ok {
|
||||
return fmt.Errorf("user does not exist")
|
||||
}
|
||||
|
||||
if acc.Role == "admin" {
|
||||
return nil
|
||||
}
|
||||
return s3err.GetAPIError(s3err.ErrAccessDenied)
|
||||
}
|
||||
36
auth/iam.go
36
auth/iam.go
@@ -1,36 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"errors"
|
||||
)
|
||||
|
||||
// Account is a gateway IAM account
|
||||
type Account struct {
|
||||
Secret string `json:"secret"`
|
||||
Role string `json:"role"`
|
||||
}
|
||||
|
||||
// IAMService is the interface for all IAM service implementations
|
||||
//
|
||||
//go:generate moq -out ../s3api/controllers/iam_moq_test.go -pkg controllers . IAMService
|
||||
type IAMService interface {
|
||||
CreateAccount(access string, account Account) error
|
||||
GetUserAccount(access string) (Account, error)
|
||||
DeleteUserAccount(access string) error
|
||||
}
|
||||
|
||||
var ErrNoSuchUser = errors.New("user not found")
|
||||
@@ -1,181 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// IAMServiceInternal manages the internal IAM service
|
||||
type IAMServiceInternal struct {
|
||||
storer Storer
|
||||
|
||||
mu sync.RWMutex
|
||||
accts IAMConfig
|
||||
serial uint32
|
||||
}
|
||||
|
||||
// UpdateAcctFunc accepts the current data and returns the new data to be stored
|
||||
type UpdateAcctFunc func([]byte) ([]byte, error)
|
||||
|
||||
// Storer is the interface to manage the peristent IAM data for the internal
|
||||
// IAM service
|
||||
type Storer interface {
|
||||
InitIAM() error
|
||||
GetIAM() ([]byte, error)
|
||||
StoreIAM(UpdateAcctFunc) error
|
||||
}
|
||||
|
||||
// IAMConfig stores all internal IAM accounts
|
||||
type IAMConfig struct {
|
||||
AccessAccounts map[string]Account `json:"accessAccounts"`
|
||||
}
|
||||
|
||||
var _ IAMService = &IAMServiceInternal{}
|
||||
|
||||
// NewInternal creates a new instance for the Internal IAM service
|
||||
func NewInternal(s Storer) (*IAMServiceInternal, error) {
|
||||
i := &IAMServiceInternal{
|
||||
storer: s,
|
||||
}
|
||||
|
||||
err := i.updateCache()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("refresh iam cache: %w", err)
|
||||
}
|
||||
|
||||
return i, nil
|
||||
}
|
||||
|
||||
// CreateAccount creates a new IAM account. Returns an error if the account
|
||||
// already exists.
|
||||
func (s *IAMServiceInternal) CreateAccount(access string, account Account) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
return s.storer.StoreIAM(func(data []byte) ([]byte, error) {
|
||||
var conf IAMConfig
|
||||
|
||||
if len(data) > 0 {
|
||||
if err := json.Unmarshal(data, &conf); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse iam: %w", err)
|
||||
}
|
||||
} else {
|
||||
conf = IAMConfig{AccessAccounts: map[string]Account{}}
|
||||
}
|
||||
|
||||
_, ok := conf.AccessAccounts[access]
|
||||
if ok {
|
||||
return nil, fmt.Errorf("account already exists")
|
||||
}
|
||||
conf.AccessAccounts[access] = account
|
||||
|
||||
b, err := json.Marshal(conf)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to serialize iam: %w", err)
|
||||
}
|
||||
s.accts = conf
|
||||
|
||||
return b, nil
|
||||
})
|
||||
}
|
||||
|
||||
// GetUserAccount retrieves account info for the requested user. Returns
|
||||
// ErrNoSuchUser if the account does not exist.
|
||||
func (s *IAMServiceInternal) GetUserAccount(access string) (Account, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
data, err := s.storer.GetIAM()
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("get iam data: %w", err)
|
||||
}
|
||||
|
||||
serial := crc32.ChecksumIEEE(data)
|
||||
if serial != s.serial {
|
||||
s.mu.RUnlock()
|
||||
err := s.updateCache()
|
||||
s.mu.RLock()
|
||||
if err != nil {
|
||||
return Account{}, fmt.Errorf("refresh iam cache: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
acct, ok := s.accts.AccessAccounts[access]
|
||||
if !ok {
|
||||
return Account{}, ErrNoSuchUser
|
||||
}
|
||||
|
||||
return acct, nil
|
||||
}
|
||||
|
||||
// updateCache must be called with no locks held
|
||||
func (s *IAMServiceInternal) updateCache() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
data, err := s.storer.GetIAM()
|
||||
if err != nil {
|
||||
return fmt.Errorf("get iam data: %w", err)
|
||||
}
|
||||
|
||||
serial := crc32.ChecksumIEEE(data)
|
||||
|
||||
if len(data) > 0 {
|
||||
if err := json.Unmarshal(data, &s.accts); err != nil {
|
||||
return fmt.Errorf("failed to parse the config file: %w", err)
|
||||
}
|
||||
} else {
|
||||
s.accts.AccessAccounts = make(map[string]Account)
|
||||
}
|
||||
|
||||
s.serial = serial
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteUserAccount deletes the specified user account. Does not check if
|
||||
// account exists.
|
||||
func (s *IAMServiceInternal) DeleteUserAccount(access string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
return s.storer.StoreIAM(func(data []byte) ([]byte, error) {
|
||||
if len(data) == 0 {
|
||||
// empty config, do nothing
|
||||
return data, nil
|
||||
}
|
||||
|
||||
var conf IAMConfig
|
||||
|
||||
if err := json.Unmarshal(data, &conf); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse iam: %w", err)
|
||||
}
|
||||
|
||||
delete(conf.AccessAccounts, access)
|
||||
|
||||
b, err := json.Marshal(conf)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to serialize iam: %w", err)
|
||||
}
|
||||
|
||||
s.accts = conf
|
||||
|
||||
return b, nil
|
||||
})
|
||||
}
|
||||
@@ -1,66 +1,55 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package backend
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/scoutgw/s3err"
|
||||
)
|
||||
|
||||
//go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
|
||||
//go:generate moq -out backend_moq_test.go . Backend
|
||||
//go:generate moq -out ../s3api/controllers/backend_moq_test.go . Backend
|
||||
type Backend interface {
|
||||
fmt.Stringer
|
||||
GetIAMConfig() ([]byte, error)
|
||||
Shutdown()
|
||||
|
||||
ListBuckets(_ context.Context, owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error)
|
||||
HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
|
||||
GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
|
||||
CreateBucket(context.Context, *s3.CreateBucketInput) error
|
||||
PutBucketAcl(_ context.Context, bucket string, data []byte) error
|
||||
DeleteBucket(context.Context, *s3.DeleteBucketInput) error
|
||||
ListBuckets() (*s3.ListBucketsOutput, error)
|
||||
HeadBucket(bucket string) (*s3.HeadBucketOutput, error)
|
||||
GetBucketAcl(bucket string) (*s3.GetBucketAclOutput, error)
|
||||
PutBucket(bucket string) error
|
||||
DeleteBucket(bucket string) error
|
||||
|
||||
CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
|
||||
CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error)
|
||||
AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error
|
||||
ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
|
||||
ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error)
|
||||
UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error)
|
||||
UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error)
|
||||
SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
|
||||
CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
|
||||
CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error)
|
||||
AbortMultipartUpload(*s3.AbortMultipartUploadInput) error
|
||||
ListMultipartUploads(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error)
|
||||
ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error)
|
||||
CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error)
|
||||
PutObjectPart(bucket, object, uploadID string, part int, r io.Reader) (etag string, err error)
|
||||
|
||||
PutObject(context.Context, *s3.PutObjectInput) (string, error)
|
||||
HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
|
||||
GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error)
|
||||
GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error)
|
||||
GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
|
||||
CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
|
||||
ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
|
||||
ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
|
||||
DeleteObject(context.Context, *s3.DeleteObjectInput) error
|
||||
DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
|
||||
PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
|
||||
RestoreObject(context.Context, *s3.RestoreObjectInput) error
|
||||
PutObject(bucket, object string, r io.Reader) (string, error)
|
||||
HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, error)
|
||||
GetObject(bucket, object string, startOffset, length int64, writer io.Writer, etag string) (*s3.GetObjectOutput, error)
|
||||
GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error)
|
||||
GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error)
|
||||
CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error)
|
||||
ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error)
|
||||
ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error)
|
||||
DeleteObject(bucket, object string) error
|
||||
DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error
|
||||
PutBucketAcl(*s3.PutBucketAclInput) error
|
||||
PutObjectAcl(*s3.PutObjectAclInput) error
|
||||
RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error
|
||||
UploadPart(bucket, object, uploadId string, Body io.ReadSeeker) (*s3.UploadPartOutput, error)
|
||||
UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error)
|
||||
|
||||
GetTags(_ context.Context, bucket, object string) (map[string]string, error)
|
||||
SetTags(_ context.Context, bucket, object string, tags map[string]string) error
|
||||
RemoveTags(_ context.Context, bucket, object string) error
|
||||
IsTaggingSupported() bool
|
||||
GetTags(bucket, object string) (map[string]string, error)
|
||||
SetTags(bucket, object string, tags map[string]string) error
|
||||
RemoveTags(bucket, object string) error
|
||||
}
|
||||
|
||||
type BackendUnsupported struct{}
|
||||
@@ -70,97 +59,105 @@ var _ Backend = &BackendUnsupported{}
|
||||
func New() Backend {
|
||||
return &BackendUnsupported{}
|
||||
}
|
||||
|
||||
func (BackendUnsupported) GetIAMConfig() ([]byte, error) {
|
||||
return nil, fmt.Errorf("not supported")
|
||||
}
|
||||
func (BackendUnsupported) Shutdown() {}
|
||||
func (BackendUnsupported) String() string {
|
||||
return "Unsupported"
|
||||
}
|
||||
func (BackendUnsupported) ListBuckets(context.Context, string, bool) (s3response.ListAllMyBucketsResult, error) {
|
||||
return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) UploadPartCopy(context.Context, *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
|
||||
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
|
||||
func (BackendUnsupported) ListBuckets() (*s3.ListBucketsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
|
||||
func (BackendUnsupported) PutBucketAcl(*s3.PutBucketAclInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutObjectAcl(*s3.PutObjectAclInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput) error {
|
||||
func (BackendUnsupported) UploadPart(bucket, object, uploadId string, Body io.ReadSeeker) (*s3.UploadPartOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetBucketAcl(bucket string) (*s3.GetBucketAclOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutBucket(bucket string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
|
||||
func (BackendUnsupported) DeleteBucket(bucket string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
|
||||
return s3response.SelectObjectContentResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
func (BackendUnsupported) CreateMultipartUpload(input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
func (BackendUnsupported) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput) error {
|
||||
func (BackendUnsupported) AbortMultipartUpload(input *s3.AbortMultipartUploadInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListMultipartUploads(context.Context, *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
|
||||
return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListParts(context.Context, *s3.ListPartsInput) (s3response.ListPartsResult, error) {
|
||||
return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (*s3.ListPartsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) UploadPart(context.Context, *s3.UploadPartInput) (etag string, err error) {
|
||||
func (BackendUnsupported) CopyPart(srcBucket, srcObject, DstBucket, uploadID, rangeHeader string, part int) (*types.CopyPartResult, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) PutObjectPart(bucket, object, uploadID string, part int, r io.Reader) (etag string, err error) {
|
||||
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) PutObject(context.Context, *s3.PutObjectInput) (string, error) {
|
||||
func (BackendUnsupported) PutObject(bucket, object string, r io.Reader) (string, error) {
|
||||
return "", s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteObject(context.Context, *s3.DeleteObjectInput) error {
|
||||
func (BackendUnsupported) DeleteObject(bucket, object string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
|
||||
return s3response.DeleteObjectsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObject(context.Context, *s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) {
|
||||
func (BackendUnsupported) GetObject(bucket, object string, startOffset, length int64, writer io.Writer, etag string) (*s3.GetObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) HeadObject(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
func (BackendUnsupported) HeadObject(bucket, object string, etag string) (*s3.HeadObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectAcl(context.Context, *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) {
|
||||
func (BackendUnsupported) GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) GetObjectAttributes(context.Context, *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
|
||||
func (BackendUnsupported) GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) CopyObject(context.Context, *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
|
||||
func (BackendUnsupported) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListObjects(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
func (BackendUnsupported) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) ListObjectsV2(context.Context, *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
func (BackendUnsupported) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListBucketsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) GetTags(_ context.Context, bucket, object string) (map[string]string, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) IsTaggingSupported() bool { return false }
|
||||
func (BackendUnsupported) GetTags(bucket, object string) (map[string]string, error) {
|
||||
return nil, fmt.Errorf("not supported")
|
||||
}
|
||||
func (BackendUnsupported) SetTags(_ context.Context, bucket, object string, tags map[string]string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) SetTags(bucket, object string, tags map[string]string) error {
|
||||
return fmt.Errorf("not supported")
|
||||
}
|
||||
func (BackendUnsupported) RemoveTags(_ context.Context, bucket, object string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
func (BackendUnsupported) RemoveTags(bucket, object string) error {
|
||||
return fmt.Errorf("not supported")
|
||||
}
|
||||
|
||||
1794
backend/backend_moq_test.go
Normal file
1794
backend/backend_moq_test.go
Normal file
File diff suppressed because it is too large
Load Diff
204
backend/backend_test.go
Normal file
204
backend/backend_test.go
Normal file
@@ -0,0 +1,204 @@
|
||||
package backend
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/scoutgw/s3err"
|
||||
)
|
||||
|
||||
func TestBackend_ListBuckets(t *testing.T) {
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
}
|
||||
type test struct {
|
||||
name string
|
||||
c Backend
|
||||
args args
|
||||
wantErr bool
|
||||
}
|
||||
var tests []test
|
||||
tests = append(tests, test{
|
||||
name: "list-Bucket",
|
||||
c: &BackendMock{
|
||||
ListBucketsFunc: func() (*s3.ListBucketsOutput, error) {
|
||||
return &s3.ListBucketsOutput{
|
||||
Buckets: []types.Bucket{
|
||||
{
|
||||
Name: aws.String("t1"),
|
||||
},
|
||||
},
|
||||
}, s3err.GetAPIError(0)
|
||||
},
|
||||
},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
},
|
||||
wantErr: false,
|
||||
}, test{
|
||||
name: "list-Bucket-error",
|
||||
c: &BackendMock{
|
||||
ListBucketsFunc: func() (*s3.ListBucketsOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
},
|
||||
},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
},
|
||||
wantErr: true,
|
||||
})
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if _, err := tt.c.ListBuckets(); (err.(s3err.APIError).Code != "") != tt.wantErr {
|
||||
t.Errorf("Backend.ListBuckets() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestBackend_HeadBucket(t *testing.T) {
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
BucketName string
|
||||
}
|
||||
type test struct {
|
||||
name string
|
||||
c Backend
|
||||
args args
|
||||
wantErr bool
|
||||
}
|
||||
var tests []test
|
||||
tests = append(tests, test{
|
||||
name: "head-buckets-error",
|
||||
c: &BackendMock{
|
||||
HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
},
|
||||
},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
BucketName: "b1",
|
||||
},
|
||||
wantErr: true,
|
||||
})
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if _, err := tt.c.HeadBucket(tt.args.BucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
|
||||
t.Errorf("Backend.HeadBucket() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestBackend_GetBucketAcl(t *testing.T) {
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
bucketName string
|
||||
}
|
||||
type test struct {
|
||||
name string
|
||||
c Backend
|
||||
args args
|
||||
wantErr bool
|
||||
}
|
||||
var tests []test
|
||||
tests = append(tests, test{
|
||||
name: "get bucket acl error",
|
||||
c: &BackendMock{
|
||||
GetBucketAclFunc: func(bucket string) (*s3.GetBucketAclOutput, error) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
},
|
||||
},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
bucketName: "b1",
|
||||
},
|
||||
wantErr: true,
|
||||
})
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if _, err := tt.c.GetBucketAcl(tt.args.bucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
|
||||
t.Errorf("Backend.GetBucketAcl() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestBackend_PutBucket(t *testing.T) {
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
bucketName string
|
||||
}
|
||||
type test struct {
|
||||
name string
|
||||
c Backend
|
||||
args args
|
||||
wantErr bool
|
||||
}
|
||||
var tests []test
|
||||
tests = append(tests, test{
|
||||
name: "put bucket ",
|
||||
c: &BackendMock{
|
||||
PutBucketFunc: func(bucket string) error {
|
||||
return s3err.GetAPIError(0)
|
||||
},
|
||||
},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
bucketName: "b1",
|
||||
},
|
||||
wantErr: false,
|
||||
}, test{
|
||||
name: "put bucket error",
|
||||
c: &BackendMock{
|
||||
PutBucketFunc: func(bucket string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
},
|
||||
},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
bucketName: "b2",
|
||||
},
|
||||
wantErr: true,
|
||||
})
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if err := tt.c.PutBucket(tt.args.bucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
|
||||
t.Errorf("Backend.PutBucket() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestBackend_DeleteBucket(t *testing.T) {
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
bucketName string
|
||||
}
|
||||
type test struct {
|
||||
name string
|
||||
c Backend
|
||||
args args
|
||||
wantErr bool
|
||||
}
|
||||
var tests []test
|
||||
tests = append(tests, test{
|
||||
name: "Delete Bucket Error",
|
||||
c: &BackendMock{
|
||||
DeleteBucketFunc: func(bucket string) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
},
|
||||
},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
bucketName: "b1",
|
||||
},
|
||||
wantErr: true,
|
||||
})
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if err := tt.c.DeleteBucket(tt.args.bucketName); (err.(s3err.APIError).Code != "") != tt.wantErr {
|
||||
t.Errorf("Backend.DeleteBucket() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,51 +1,18 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package backend
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
var (
|
||||
// RFC3339TimeFormat RFC3339 time format
|
||||
RFC3339TimeFormat = "2006-01-02T15:04:05.999Z"
|
||||
)
|
||||
|
||||
func IsValidBucketName(name string) bool { return true }
|
||||
|
||||
type ByBucketName []s3response.ListAllMyBucketsEntry
|
||||
type ByBucketName []types.Bucket
|
||||
|
||||
func (d ByBucketName) Len() int { return len(d) }
|
||||
func (d ByBucketName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
|
||||
func (d ByBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name }
|
||||
|
||||
type ByObjectName []types.Object
|
||||
|
||||
func (d ByObjectName) Len() int { return len(d) }
|
||||
func (d ByObjectName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
|
||||
func (d ByObjectName) Less(i, j int) bool { return *d[i].Key < *d[j].Key }
|
||||
func (d ByBucketName) Less(i, j int) bool { return *d[i].Name < *d[j].Name }
|
||||
|
||||
func GetStringPtr(s string) *string {
|
||||
return &s
|
||||
@@ -54,69 +21,3 @@ func GetStringPtr(s string) *string {
|
||||
func GetTimePtr(t time.Time) *time.Time {
|
||||
return &t
|
||||
}
|
||||
|
||||
var (
|
||||
errInvalidRange = s3err.GetAPIError(s3err.ErrInvalidRange)
|
||||
)
|
||||
|
||||
// ParseRange parses input range header and returns startoffset, length, and
|
||||
// error. If no endoffset specified, then length is set to -1.
|
||||
func ParseRange(file fs.FileInfo, acceptRange string) (int64, int64, error) {
|
||||
if acceptRange == "" {
|
||||
return 0, file.Size(), nil
|
||||
}
|
||||
|
||||
rangeKv := strings.Split(acceptRange, "=")
|
||||
|
||||
if len(rangeKv) < 2 {
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
bRange := strings.Split(rangeKv[1], "-")
|
||||
if len(bRange) < 1 || len(bRange) > 2 {
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
startOffset, err := strconv.ParseInt(bRange[0], 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
endOffset := int64(-1)
|
||||
if len(bRange) == 1 || bRange[1] == "" {
|
||||
return startOffset, endOffset, nil
|
||||
}
|
||||
|
||||
endOffset, err = strconv.ParseInt(bRange[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
if endOffset < startOffset {
|
||||
return 0, 0, errInvalidRange
|
||||
}
|
||||
|
||||
return startOffset, endOffset - startOffset + 1, nil
|
||||
}
|
||||
|
||||
func GetMultipartMD5(parts []types.CompletedPart) string {
|
||||
var partsEtagBytes []byte
|
||||
for _, part := range parts {
|
||||
partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...)
|
||||
}
|
||||
s3MD5 := fmt.Sprintf("%s-%d", md5String(partsEtagBytes), len(parts))
|
||||
return s3MD5
|
||||
}
|
||||
|
||||
func getEtagBytes(etag string) []byte {
|
||||
decode, err := hex.DecodeString(strings.ReplaceAll(etag, string('"'), ""))
|
||||
if err != nil {
|
||||
return []byte(etag)
|
||||
}
|
||||
return decode
|
||||
}
|
||||
|
||||
func md5String(data []byte) string {
|
||||
sum := md5.Sum(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,89 +1,14 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package posix
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type tmpfile struct {
|
||||
f *os.File
|
||||
bucket string
|
||||
objname string
|
||||
size int64
|
||||
func openTmpFile(dir string) (*os.File, error) {
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
|
||||
// Create a temp file for upload while in progress (see link comments below).
|
||||
err := os.MkdirAll(dir, 0700)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make temp dir: %w", err)
|
||||
}
|
||||
f, err := os.CreateTemp(dir,
|
||||
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
tempname := tmp.f.Name()
|
||||
// cleanup in case anything goes wrong, if rename succeeds then
|
||||
// this will no longer exist
|
||||
defer os.Remove(tempname)
|
||||
|
||||
// We use Rename as the atomic operation for object puts. The upload is
|
||||
// written to a temp file to not conflict with any other simultaneous
|
||||
// uploads. The final operation is to move the temp file into place for
|
||||
// the object. This ensures the object semantics of last upload completed
|
||||
// wins and is not some combination of writes from simultaneous uploads.
|
||||
objPath := filepath.Join(tmp.bucket, tmp.objname)
|
||||
err := os.Remove(objPath)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("remove stale path: %w", err)
|
||||
}
|
||||
|
||||
err = tmp.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("close tmpfile: %w", err)
|
||||
}
|
||||
|
||||
err = os.Rename(tempname, objPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rename tmpfile: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
if int64(len(b)) > tmp.size {
|
||||
return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
|
||||
}
|
||||
|
||||
n, err := tmp.f.Write(b)
|
||||
tmp.size -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
tmp.f.Close()
|
||||
func linkTmpFile(f *os.File, path string) error {
|
||||
return fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
@@ -1,105 +1,31 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package posix
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const procfddir = "/proc/self/fd"
|
||||
|
||||
type tmpfile struct {
|
||||
f *os.File
|
||||
bucket string
|
||||
objname string
|
||||
isOTmp bool
|
||||
size int64
|
||||
}
|
||||
|
||||
func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
|
||||
// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
|
||||
// This can help reduce contention within the namespace (parent directories),
|
||||
// etc. And will auto cleanup the inode on close if we never link this
|
||||
// file descriptor into the namespace.
|
||||
// Not all filesystems support this, so fallback to CreateTemp for when
|
||||
// this is not supported.
|
||||
func openTmpFile(dir string) (*os.File, error) {
|
||||
fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
|
||||
if err != nil {
|
||||
// O_TMPFILE not supported, try fallback
|
||||
err := os.MkdirAll(dir, 0700)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make temp dir: %w", err)
|
||||
}
|
||||
f, err := os.CreateTemp(dir,
|
||||
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
|
||||
// falloc is best effort, its fine if this fails
|
||||
if size > 0 {
|
||||
tmp.falloc()
|
||||
}
|
||||
return tmp, nil
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
|
||||
// later to link file into namespace
|
||||
f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
|
||||
|
||||
tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
|
||||
// falloc is best effort, its fine if this fails
|
||||
if size > 0 {
|
||||
tmp.falloc()
|
||||
}
|
||||
return tmp, nil
|
||||
return os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd))), nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) falloc() error {
|
||||
err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fallocate: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
// We use Linkat/Rename as the atomic operation for object puts. The
|
||||
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
|
||||
// with any other simultaneous uploads. The final operation is to move the
|
||||
// temp file into place for the object. This ensures the object semantics
|
||||
// of last upload completed wins and is not some combination of writes
|
||||
// from simultaneous uploads.
|
||||
objPath := filepath.Join(tmp.bucket, tmp.objname)
|
||||
err := os.Remove(objPath)
|
||||
func linkTmpFile(f *os.File, path string) error {
|
||||
err := os.Remove(path)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("remove stale path: %w", err)
|
||||
}
|
||||
|
||||
if !tmp.isOTmp {
|
||||
// O_TMPFILE not suported, use fallback
|
||||
return tmp.fallbackLink()
|
||||
return fmt.Errorf("remove stale part: %w", err)
|
||||
}
|
||||
|
||||
procdir, err := os.Open(procfddir)
|
||||
@@ -108,57 +34,17 @@ func (tmp *tmpfile) link() error {
|
||||
}
|
||||
defer procdir.Close()
|
||||
|
||||
dir, err := os.Open(filepath.Dir(objPath))
|
||||
dir, err := os.Open(filepath.Dir(path))
|
||||
if err != nil {
|
||||
return fmt.Errorf("open parent dir: %w", err)
|
||||
}
|
||||
defer dir.Close()
|
||||
|
||||
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
|
||||
int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
|
||||
err = unix.Linkat(int(procdir.Fd()), filepath.Base(f.Name()),
|
||||
int(dir.Fd()), filepath.Base(path), unix.AT_SYMLINK_FOLLOW)
|
||||
if err != nil {
|
||||
return fmt.Errorf("link tmpfile (%q in %q): %w",
|
||||
filepath.Dir(objPath), filepath.Base(tmp.f.Name()), err)
|
||||
}
|
||||
|
||||
err = tmp.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("close tmpfile: %w", err)
|
||||
return fmt.Errorf("link tmpfile: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) fallbackLink() error {
|
||||
tempname := tmp.f.Name()
|
||||
// cleanup in case anything goes wrong, if rename succeeds then
|
||||
// this will no longer exist
|
||||
defer os.Remove(tempname)
|
||||
|
||||
err := tmp.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("close tmpfile: %w", err)
|
||||
}
|
||||
|
||||
objPath := filepath.Join(tmp.bucket, tmp.objname)
|
||||
err = os.Rename(tempname, objPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rename tmpfile: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
if int64(len(b)) > tmp.size {
|
||||
return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
|
||||
}
|
||||
|
||||
n, err := tmp.f.Write(b)
|
||||
tmp.size -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
tmp.f.Close()
|
||||
}
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package posix
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type tmpfile struct {
|
||||
f *os.File
|
||||
bucket string
|
||||
objname string
|
||||
size int64
|
||||
}
|
||||
|
||||
func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
|
||||
// Create a temp file for upload while in progress (see link comments below).
|
||||
err := os.MkdirAll(dir, 0700)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make temp dir: %w", err)
|
||||
}
|
||||
f, err := os.CreateTemp(dir,
|
||||
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &tmpfile{f: f, bucket: bucket, objname: obj, size: size}, nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
tempname := tmp.f.Name()
|
||||
// cleanup in case anything goes wrong, if rename succeeds then
|
||||
// this will no longer exist
|
||||
defer os.Remove(tempname)
|
||||
|
||||
// We use Rename as the atomic operation for object puts. The upload is
|
||||
// written to a temp file to not conflict with any other simultaneous
|
||||
// uploads. The final operation is to move the temp file into place for
|
||||
// the object. This ensures the object semantics of last upload completed
|
||||
// wins and is not some combination of writes from simultaneous uploads.
|
||||
objPath := filepath.Join(tmp.bucket, tmp.objname)
|
||||
err := os.Remove(objPath)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("remove stale path: %w", err)
|
||||
}
|
||||
|
||||
err = tmp.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("close tmpfile: %w", err)
|
||||
}
|
||||
|
||||
err = os.Rename(tempname, objPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rename tmpfile: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
if int64(len(b)) > tmp.size {
|
||||
return 0, fmt.Errorf("write exceeds content length")
|
||||
}
|
||||
|
||||
n, err := tmp.f.Write(b)
|
||||
tmp.size -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
tmp.f.Close()
|
||||
}
|
||||
@@ -1,778 +1,12 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package scoutfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/versity/scoutfs-go"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/scoutgw/backend"
|
||||
"github.com/versity/scoutgw/backend/posix"
|
||||
)
|
||||
|
||||
type ScoutFS struct {
|
||||
*posix.Posix
|
||||
rootfd *os.File
|
||||
rootdir string
|
||||
|
||||
// glaciermode enables the following behavior:
|
||||
// GET object: if file offline, return invalid object state
|
||||
// HEAD object: if file offline, set obj storage class to GLACIER
|
||||
// if file offline and staging, x-amz-restore: ongoing-request="true"
|
||||
// if file offline and not staging, x-amz-restore: ongoing-request="false"
|
||||
// if file online, x-amz-restore: ongoing-request="false", expiry-date="Fri, 2 Dec 2050 00:00:00 GMT"
|
||||
// note: this expiry-date is not used but provided for client glacier compatibility
|
||||
// ListObjects: if file offline, set obj storage class to GLACIER
|
||||
// RestoreObject: add batch stage request to file
|
||||
glaciermode bool
|
||||
}
|
||||
|
||||
var _ backend.Backend = &ScoutFS{}
|
||||
|
||||
const (
|
||||
metaTmpDir = ".sgwtmp"
|
||||
metaTmpMultipartDir = metaTmpDir + "/multipart"
|
||||
tagHdr = "X-Amz-Tagging"
|
||||
emptyMD5 = "d41d8cd98f00b204e9800998ecf8427e"
|
||||
etagkey = "user.etag"
|
||||
)
|
||||
|
||||
var (
|
||||
stageComplete = "ongoing-request=\"false\", expiry-date=\"Fri, 2 Dec 2050 00:00:00 GMT\""
|
||||
stageInProgress = "true"
|
||||
stageNotInProgress = "false"
|
||||
)
|
||||
|
||||
const (
|
||||
// ScoutFS special xattr types
|
||||
|
||||
systemPrefix = "scoutfs.hide."
|
||||
onameAttr = systemPrefix + "objname"
|
||||
flagskey = systemPrefix + "sam_flags"
|
||||
stagecopykey = systemPrefix + "sam_stagereq"
|
||||
)
|
||||
|
||||
const (
|
||||
// ScoutAM Flags
|
||||
|
||||
// Staging - file requested stage
|
||||
Staging uint64 = 1 << iota
|
||||
// StageFail - all copies failed to stage
|
||||
StageFail
|
||||
// NoArchive - no archive copies of file should be made
|
||||
NoArchive
|
||||
// ExtCacheRequested means file policy requests Ext Cache
|
||||
ExtCacheRequested
|
||||
// ExtCacheDone means this file ext cache copy has been
|
||||
// created already (and possibly pruned, so may not exist)
|
||||
ExtCacheDone
|
||||
)
|
||||
|
||||
// Option sets various options for scoutfs
|
||||
type Option func(s *ScoutFS)
|
||||
|
||||
// WithGlacierEmulation sets glacier mode emulation
|
||||
func WithGlacierEmulation() Option {
|
||||
return func(s *ScoutFS) { s.glaciermode = true }
|
||||
}
|
||||
|
||||
func (s *ScoutFS) Shutdown() {
|
||||
s.Posix.Shutdown()
|
||||
s.rootfd.Close()
|
||||
_ = s.rootdir
|
||||
}
|
||||
|
||||
func (*ScoutFS) String() string {
|
||||
return "ScoutFS Gateway"
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
|
||||
// ioctl to not have to read and copy the part data to the final object. This
|
||||
// saves a read and write cycle for all mutlipart uploads.
|
||||
func (s *ScoutFS) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
uploadID := *input.UploadId
|
||||
parts := input.MultipartUpload.Parts
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
sum, err := s.checkUploadIDExists(bucket, object, uploadID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
|
||||
|
||||
// check all parts ok
|
||||
last := len(parts) - 1
|
||||
partsize := int64(0)
|
||||
var totalsize int64
|
||||
for i, p := range parts {
|
||||
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))
|
||||
fi, err := os.Lstat(partPath)
|
||||
if err != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
|
||||
if i == 0 {
|
||||
partsize = fi.Size()
|
||||
}
|
||||
totalsize += fi.Size()
|
||||
// all parts except the last need to be the same size
|
||||
if i < last && partsize != fi.Size() {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
// non-last part sizes need to be multiples of 4k for move blocks
|
||||
// TODO: fallback to no move blocks if not 4k aligned?
|
||||
if i == 0 && i < last && fi.Size()%4096 != 0 {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
|
||||
b, err := xattr.Get(partPath, "user.etag")
|
||||
etag := string(b)
|
||||
if err != nil {
|
||||
etag = ""
|
||||
}
|
||||
if etag != *parts[i].ETag {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
}
|
||||
}
|
||||
|
||||
// use totalsize=0 because we wont be writing to the file, only moving
|
||||
// extents around. so we dont want to fallocate this.
|
||||
f, err := openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, 0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open temp file: %w", err)
|
||||
}
|
||||
defer f.cleanup()
|
||||
|
||||
for _, p := range parts {
|
||||
pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err)
|
||||
}
|
||||
|
||||
// scoutfs move data is a metadata only operation that moves the data
|
||||
// extent references from the source, appeding to the destination.
|
||||
// this needs to be 4k aligned.
|
||||
err = scoutfs.MoveData(pf, f.f)
|
||||
pf.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("move blocks part %v: %v", p.PartNumber, err)
|
||||
}
|
||||
}
|
||||
|
||||
userMetaData := make(map[string]string)
|
||||
upiddir := filepath.Join(objdir, uploadID)
|
||||
loadUserMetaData(upiddir, userMetaData)
|
||||
|
||||
objname := filepath.Join(bucket, object)
|
||||
dir := filepath.Dir(objname)
|
||||
if dir != "" {
|
||||
if err = mkdirAll(dir, os.FileMode(0755), bucket, object); err != nil {
|
||||
if err != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)
|
||||
}
|
||||
}
|
||||
}
|
||||
err = f.link()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("link object in namespace: %w", err)
|
||||
}
|
||||
|
||||
for k, v := range userMetaData {
|
||||
err = xattr.Set(objname, "user."+k, []byte(v))
|
||||
if err != nil {
|
||||
// cleanup object if returning error
|
||||
os.Remove(objname)
|
||||
return nil, fmt.Errorf("set user attr %q: %w", k, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate s3 compatible md5sum for complete multipart.
|
||||
s3MD5 := backend.GetMultipartMD5(parts)
|
||||
|
||||
err = xattr.Set(objname, "user.etag", []byte(s3MD5))
|
||||
if err != nil {
|
||||
// cleanup object if returning error
|
||||
os.Remove(objname)
|
||||
return nil, fmt.Errorf("set etag attr: %w", err)
|
||||
}
|
||||
|
||||
// cleanup tmp dirs
|
||||
os.RemoveAll(upiddir)
|
||||
// use Remove for objdir in case there are still other uploads
|
||||
// for same object name outstanding
|
||||
os.Remove(objdir)
|
||||
|
||||
return &s3.CompleteMultipartUploadOutput{
|
||||
Bucket: &bucket,
|
||||
ETag: &s3MD5,
|
||||
Key: &object,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, error) {
|
||||
sum := sha256.Sum256([]byte(object))
|
||||
objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum))
|
||||
|
||||
_, err := os.Stat(filepath.Join(objdir, uploadID))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return [32]byte{}, s3err.GetAPIError(s3err.ErrNoSuchUpload)
|
||||
}
|
||||
if err != nil {
|
||||
return [32]byte{}, fmt.Errorf("stat upload: %w", err)
|
||||
}
|
||||
return sum, nil
|
||||
}
|
||||
|
||||
func loadUserMetaData(path string, m map[string]string) (contentType, contentEncoding string) {
|
||||
ents, err := xattr.List(path)
|
||||
if err != nil || len(ents) == 0 {
|
||||
return
|
||||
}
|
||||
for _, e := range ents {
|
||||
if !isValidMeta(e) {
|
||||
continue
|
||||
}
|
||||
b, err := xattr.Get(path, e)
|
||||
if err == syscall.ENODATA {
|
||||
m[strings.TrimPrefix(e, "user.")] = ""
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
m[strings.TrimPrefix(e, "user.")] = string(b)
|
||||
}
|
||||
|
||||
b, err := xattr.Get(path, "user.content-type")
|
||||
contentType = string(b)
|
||||
if err != nil {
|
||||
contentType = ""
|
||||
}
|
||||
if contentType != "" {
|
||||
m["content-type"] = contentType
|
||||
}
|
||||
|
||||
b, err = xattr.Get(path, "user.content-encoding")
|
||||
contentEncoding = string(b)
|
||||
if err != nil {
|
||||
contentEncoding = ""
|
||||
}
|
||||
if contentEncoding != "" {
|
||||
m["content-encoding"] = contentEncoding
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func isValidMeta(val string) bool {
|
||||
if strings.HasPrefix(val, "user.X-Amz-Meta") {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(val, "user.Expires") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// mkdirAll is similar to os.MkdirAll but it will return ErrObjectParentIsFile
|
||||
// when appropriate
|
||||
func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
|
||||
// Fast path: if we can tell whether path is a directory or file, stop with success or error.
|
||||
dir, err := os.Stat(path)
|
||||
if err == nil {
|
||||
if dir.IsDir() {
|
||||
return nil
|
||||
}
|
||||
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
|
||||
}
|
||||
|
||||
// Slow path: make sure parent exists and then call Mkdir for path.
|
||||
i := len(path)
|
||||
for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
|
||||
i--
|
||||
}
|
||||
|
||||
j := i
|
||||
for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
|
||||
j--
|
||||
}
|
||||
|
||||
if j > 1 {
|
||||
// Create parent.
|
||||
err = mkdirAll(path[:j-1], perm, bucket, object)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Parent now exists; invoke Mkdir and use its result.
|
||||
err = os.Mkdir(path, perm)
|
||||
if err != nil {
|
||||
// Handle arguments like "foo/." by
|
||||
// double-checking that directory doesn't exist.
|
||||
dir, err1 := os.Lstat(path)
|
||||
if err1 == nil && dir.IsDir() {
|
||||
return nil
|
||||
}
|
||||
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
objPath := filepath.Join(bucket, object)
|
||||
fi, err := os.Stat(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat object: %w", err)
|
||||
}
|
||||
|
||||
userMetaData := make(map[string]string)
|
||||
contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
|
||||
|
||||
b, err := xattr.Get(objPath, etagkey)
|
||||
etag := string(b)
|
||||
if err != nil {
|
||||
etag = ""
|
||||
}
|
||||
|
||||
stclass := types.StorageClassStandard
|
||||
requestOngoing := ""
|
||||
if s.glaciermode {
|
||||
requestOngoing = stageComplete
|
||||
|
||||
// Check if there are any offline exents associated with this file.
|
||||
// If so, we will set storage class to glacier.
|
||||
st, err := scoutfs.StatMore(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat more: %w", err)
|
||||
}
|
||||
if st.Offline_blocks != 0 {
|
||||
stclass = types.StorageClassGlacier
|
||||
requestOngoing = stageNotInProgress
|
||||
|
||||
ok, err := isStaging(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("check stage status: %w", err)
|
||||
}
|
||||
if ok {
|
||||
requestOngoing = stageInProgress
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return &s3.HeadObjectOutput{
|
||||
ContentLength: fi.Size(),
|
||||
ContentType: &contentType,
|
||||
ContentEncoding: &contentEncoding,
|
||||
ETag: &etag,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Metadata: userMetaData,
|
||||
StorageClass: stclass,
|
||||
Restore: &requestOngoing,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
acceptRange := *input.Range
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
objPath := filepath.Join(bucket, object)
|
||||
fi, err := os.Stat(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat object: %w", err)
|
||||
}
|
||||
|
||||
startOffset, length, err := backend.ParseRange(fi, acceptRange)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if length == -1 {
|
||||
length = fi.Size() - startOffset + 1
|
||||
}
|
||||
|
||||
if startOffset+length > fi.Size() {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
if s.glaciermode {
|
||||
// Check if there are any offline exents associated with this file.
|
||||
// If so, we will return the InvalidObjectState error.
|
||||
st, err := scoutfs.StatMore(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat more: %w", err)
|
||||
}
|
||||
if st.Offline_blocks != 0 {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidObjectState)
|
||||
}
|
||||
}
|
||||
|
||||
f, err := os.Open(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open object: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
rdr := io.NewSectionReader(f, startOffset, length)
|
||||
_, err = io.Copy(writer, rdr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("copy data: %w", err)
|
||||
}
|
||||
|
||||
userMetaData := make(map[string]string)
|
||||
|
||||
contentType, contentEncoding := loadUserMetaData(objPath, userMetaData)
|
||||
|
||||
b, err := xattr.Get(objPath, etagkey)
|
||||
etag := string(b)
|
||||
if err != nil {
|
||||
etag = ""
|
||||
}
|
||||
|
||||
tags, err := s.getXattrTags(bucket, object)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get object tags: %w", err)
|
||||
}
|
||||
|
||||
return &s3.GetObjectOutput{
|
||||
AcceptRanges: &acceptRange,
|
||||
ContentLength: length,
|
||||
ContentEncoding: &contentEncoding,
|
||||
ContentType: &contentType,
|
||||
ETag: &etag,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Metadata: userMetaData,
|
||||
TagCount: int32(len(tags)),
|
||||
StorageClass: types.StorageClassStandard,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error) {
|
||||
tags := make(map[string]string)
|
||||
b, err := xattr.Get(filepath.Join(bucket, object), "user."+tagHdr)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if isNoAttr(err) {
|
||||
return tags, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get tags: %w", err)
|
||||
}
|
||||
|
||||
err = json.Unmarshal(b, &tags)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unmarshal tags: %w", err)
|
||||
}
|
||||
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
bucket := *input.Bucket
|
||||
prefix := *input.Prefix
|
||||
marker := *input.Marker
|
||||
delim := *input.Delimiter
|
||||
maxkeys := input.MaxKeys
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
fileSystem := os.DirFS(bucket)
|
||||
results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
|
||||
s.fileToObj(bucket), []string{metaTmpDir})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("walk %v: %w", bucket, err)
|
||||
}
|
||||
|
||||
return &s3.ListObjectsOutput{
|
||||
CommonPrefixes: results.CommonPrefixes,
|
||||
Contents: results.Objects,
|
||||
Delimiter: &delim,
|
||||
IsTruncated: results.Truncated,
|
||||
Marker: &marker,
|
||||
MaxKeys: maxkeys,
|
||||
Name: &bucket,
|
||||
NextMarker: &results.NextMarker,
|
||||
Prefix: &prefix,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
bucket := *input.Bucket
|
||||
prefix := *input.Prefix
|
||||
marker := *input.ContinuationToken
|
||||
delim := *input.Delimiter
|
||||
maxkeys := input.MaxKeys
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
fileSystem := os.DirFS(bucket)
|
||||
results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
|
||||
s.fileToObj(bucket), []string{metaTmpDir})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("walk %v: %w", bucket, err)
|
||||
}
|
||||
|
||||
return &s3.ListObjectsV2Output{
|
||||
CommonPrefixes: results.CommonPrefixes,
|
||||
Contents: results.Objects,
|
||||
Delimiter: &delim,
|
||||
IsTruncated: results.Truncated,
|
||||
ContinuationToken: &marker,
|
||||
MaxKeys: int32(maxkeys),
|
||||
Name: &bucket,
|
||||
NextContinuationToken: &results.NextMarker,
|
||||
Prefix: &prefix,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
|
||||
return func(path string, d fs.DirEntry) (types.Object, error) {
|
||||
objPath := filepath.Join(bucket, path)
|
||||
if d.IsDir() {
|
||||
// directory object only happens if directory empty
|
||||
// check to see if this is a directory object by checking etag
|
||||
etagBytes, err := xattr.Get(objPath, etagkey)
|
||||
if isNoAttr(err) || errors.Is(err, fs.ErrNotExist) {
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil {
|
||||
return types.Object{}, fmt.Errorf("get etag: %w", err)
|
||||
}
|
||||
etag := string(etagBytes)
|
||||
|
||||
fi, err := d.Info()
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil {
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
key := path + "/"
|
||||
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &key,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// file object, get object info and fill out object data
|
||||
etagBytes, err := xattr.Get(objPath, etagkey)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil && !isNoAttr(err) {
|
||||
return types.Object{}, fmt.Errorf("get etag: %w", err)
|
||||
}
|
||||
etag := string(etagBytes)
|
||||
|
||||
fi, err := d.Info()
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil {
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
sc := types.ObjectStorageClassStandard
|
||||
if s.glaciermode {
|
||||
// Check if there are any offline exents associated with this file.
|
||||
// If so, we will return the InvalidObjectState error.
|
||||
st, err := scoutfs.StatMore(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return types.Object{}, backend.ErrSkipObj
|
||||
}
|
||||
if err != nil {
|
||||
return types.Object{}, fmt.Errorf("stat more: %w", err)
|
||||
}
|
||||
if st.Offline_blocks != 0 {
|
||||
sc = types.ObjectStorageClassGlacier
|
||||
}
|
||||
}
|
||||
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Size: fi.Size(),
|
||||
StorageClass: sc,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// RestoreObject will set stage request on file if offline and do nothing if
|
||||
// file is online
|
||||
func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) error {
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat bucket: %w", err)
|
||||
}
|
||||
|
||||
err = setStaging(filepath.Join(bucket, object))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("stage object: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func setStaging(objname string) error {
|
||||
b, err := xattr.Get(objname, flagskey)
|
||||
if err != nil && !isNoAttr(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
var oldflags uint64
|
||||
if !isNoAttr(err) {
|
||||
err = json.Unmarshal(b, &oldflags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
newflags := oldflags | Staging
|
||||
|
||||
if newflags == oldflags {
|
||||
// no flags change, just return
|
||||
return nil
|
||||
}
|
||||
|
||||
return fSetNewGlobalFlags(objname, newflags)
|
||||
}
|
||||
|
||||
func isStaging(objname string) (bool, error) {
|
||||
b, err := xattr.Get(objname, flagskey)
|
||||
if err != nil && !isNoAttr(err) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
var flags uint64
|
||||
if !isNoAttr(err) {
|
||||
err = json.Unmarshal(b, &flags)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
return flags&Staging == Staging, nil
|
||||
}
|
||||
|
||||
func fSetNewGlobalFlags(objname string, flags uint64) error {
|
||||
b, err := json.Marshal(&flags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return xattr.Set(objname, flagskey, b)
|
||||
}
|
||||
|
||||
func isNoAttr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
xerr, ok := err.(*xattr.Error)
|
||||
if ok && xerr.Err == xattr.ENOATTR {
|
||||
return true
|
||||
}
|
||||
if err == syscall.ENODATA {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
var _ backend.Backend = ScoutFS{}
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package scoutfs
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func New(rootdir string, opts ...Option) (*ScoutFS, error) {
|
||||
return nil, fmt.Errorf("scoutfs only available on linux")
|
||||
}
|
||||
|
||||
type tmpfile struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
var (
|
||||
errNotSupported = errors.New("not supported")
|
||||
)
|
||||
|
||||
func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
|
||||
return nil, errNotSupported
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
return errNotSupported
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
return 0, errNotSupported
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package scoutfs
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
)
|
||||
|
||||
func New(rootdir string, opts ...Option) (*ScoutFS, error) {
|
||||
p, err := posix.New(rootdir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
f, err := os.Open(rootdir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open %v: %w", rootdir, err)
|
||||
}
|
||||
|
||||
s := &ScoutFS{Posix: p, rootfd: f, rootdir: rootdir}
|
||||
for _, opt := range opts {
|
||||
opt(s)
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
const procfddir = "/proc/self/fd"
|
||||
|
||||
type tmpfile struct {
|
||||
f *os.File
|
||||
bucket string
|
||||
objname string
|
||||
isOTmp bool
|
||||
size int64
|
||||
}
|
||||
|
||||
func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
|
||||
// O_TMPFILE allows for a file handle to an unnamed file in the filesystem.
|
||||
// This can help reduce contention within the namespace (parent directories),
|
||||
// etc. And will auto cleanup the inode on close if we never link this
|
||||
// file descriptor into the namespace.
|
||||
// Not all filesystems support this, so fallback to CreateTemp for when
|
||||
// this is not supported.
|
||||
fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, 0666)
|
||||
if err != nil {
|
||||
// O_TMPFILE not supported, try fallback
|
||||
err := os.MkdirAll(dir, 0700)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make temp dir: %w", err)
|
||||
}
|
||||
f, err := os.CreateTemp(dir,
|
||||
fmt.Sprintf("%x.", sha256.Sum256([]byte(obj))))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tmp := &tmpfile{f: f, bucket: bucket, objname: obj, size: size}
|
||||
// falloc is best effort, its fine if this fails
|
||||
if size > 0 {
|
||||
tmp.falloc()
|
||||
}
|
||||
return tmp, nil
|
||||
}
|
||||
|
||||
// for O_TMPFILE, filename is /proc/self/fd/<fd> to be used
|
||||
// later to link file into namespace
|
||||
f := os.NewFile(uintptr(fd), filepath.Join(procfddir, strconv.Itoa(fd)))
|
||||
|
||||
tmp := &tmpfile{f: f, bucket: bucket, objname: obj, isOTmp: true, size: size}
|
||||
// falloc is best effort, its fine if this fails
|
||||
if size > 0 {
|
||||
tmp.falloc()
|
||||
}
|
||||
return tmp, nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) falloc() error {
|
||||
err := syscall.Fallocate(int(tmp.f.Fd()), 0, 0, tmp.size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fallocate: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
// We use Linkat/Rename as the atomic operation for object puts. The
|
||||
// upload is written to a temp (or unnamed/O_TMPFILE) file to not conflict
|
||||
// with any other simultaneous uploads. The final operation is to move the
|
||||
// temp file into place for the object. This ensures the object semantics
|
||||
// of last upload completed wins and is not some combination of writes
|
||||
// from simultaneous uploads.
|
||||
objPath := filepath.Join(tmp.bucket, tmp.objname)
|
||||
err := os.Remove(objPath)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("remove stale path: %w", err)
|
||||
}
|
||||
|
||||
if !tmp.isOTmp {
|
||||
// O_TMPFILE not suported, use fallback
|
||||
return tmp.fallbackLink()
|
||||
}
|
||||
|
||||
procdir, err := os.Open(procfddir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open proc dir: %w", err)
|
||||
}
|
||||
defer procdir.Close()
|
||||
|
||||
dir, err := os.Open(filepath.Dir(objPath))
|
||||
if err != nil {
|
||||
return fmt.Errorf("open parent dir: %w", err)
|
||||
}
|
||||
defer dir.Close()
|
||||
|
||||
err = unix.Linkat(int(procdir.Fd()), filepath.Base(tmp.f.Name()),
|
||||
int(dir.Fd()), filepath.Base(objPath), unix.AT_SYMLINK_FOLLOW)
|
||||
if err != nil {
|
||||
return fmt.Errorf("link tmpfile: %w", err)
|
||||
}
|
||||
|
||||
err = tmp.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("close tmpfile: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) fallbackLink() error {
|
||||
tempname := tmp.f.Name()
|
||||
// cleanup in case anything goes wrong, if rename succeeds then
|
||||
// this will no longer exist
|
||||
defer os.Remove(tempname)
|
||||
|
||||
err := tmp.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("close tmpfile: %w", err)
|
||||
}
|
||||
|
||||
objPath := filepath.Join(tmp.bucket, tmp.objname)
|
||||
err = os.Rename(tempname, objPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rename tmpfile: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
if int64(len(b)) > tmp.size {
|
||||
return 0, fmt.Errorf("write exceeds content length %v", tmp.size)
|
||||
}
|
||||
|
||||
n, err := tmp.f.Write(b)
|
||||
tmp.size -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
tmp.f.Close()
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package scoutfs
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func New(rootdir string, opts ...Option) (*ScoutFS, error) {
|
||||
return nil, fmt.Errorf("scoutfs only available on linux")
|
||||
}
|
||||
|
||||
type tmpfile struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
var (
|
||||
errNotSupported = errors.New("not supported")
|
||||
)
|
||||
|
||||
func openTmpFile(dir, bucket, obj string, size int64) (*tmpfile, error) {
|
||||
return nil, errNotSupported
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) link() error {
|
||||
return errNotSupported
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) Write(b []byte) (int, error) {
|
||||
return 0, errNotSupported
|
||||
}
|
||||
|
||||
func (tmp *tmpfile) cleanup() {
|
||||
}
|
||||
219
backend/walk.go
219
backend/walk.go
@@ -1,219 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package backend
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
)
|
||||
|
||||
type WalkResults struct {
|
||||
CommonPrefixes []types.CommonPrefix
|
||||
Objects []types.Object
|
||||
Truncated bool
|
||||
NextMarker string
|
||||
}
|
||||
|
||||
type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
|
||||
|
||||
var ErrSkipObj = errors.New("skip this object")
|
||||
|
||||
// Walk walks the supplied fs.FS and returns results compatible with list
|
||||
// objects responses
|
||||
func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
|
||||
cpmap := make(map[string]struct{})
|
||||
var objects []types.Object
|
||||
|
||||
var pastMarker bool
|
||||
if marker == "" {
|
||||
pastMarker = true
|
||||
}
|
||||
|
||||
var pastMax bool
|
||||
var newMarker string
|
||||
var truncated bool
|
||||
|
||||
err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if pastMax {
|
||||
newMarker = path
|
||||
truncated = true
|
||||
return fs.SkipAll
|
||||
}
|
||||
|
||||
if d.IsDir() {
|
||||
// Ignore the root directory
|
||||
if path == "." {
|
||||
return nil
|
||||
}
|
||||
|
||||
if contains(d.Name(), skipdirs) {
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
// If prefix is defined and the directory does not match prefix,
|
||||
// do not descend into the directory because nothing will
|
||||
// match this prefix. Make sure to append the / at the end of
|
||||
// directories since this is implied as a directory path name.
|
||||
// If path is a prefix of prefix, then path could still be
|
||||
// building to match. So only skip if path isnt a prefix of prefix
|
||||
// and prefix isnt a prefix of path.
|
||||
if prefix != "" &&
|
||||
!strings.HasPrefix(path+string(os.PathSeparator), prefix) &&
|
||||
!strings.HasPrefix(prefix, path+string(os.PathSeparator)) {
|
||||
return fs.SkipDir
|
||||
}
|
||||
|
||||
// TODO: can we do better here rather than a second readdir
|
||||
// per directory?
|
||||
ents, err := fs.ReadDir(fileSystem, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("readdir %q: %w", path, err)
|
||||
}
|
||||
if len(ents) == 0 {
|
||||
dirobj, err := getObj(path, d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("directory to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, dirobj)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if !pastMarker {
|
||||
if path != marker {
|
||||
return nil
|
||||
}
|
||||
pastMarker = true
|
||||
}
|
||||
|
||||
// If object doesn't have prefix, don't include in results.
|
||||
if prefix != "" && !strings.HasPrefix(path, prefix) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if delimiter == "" {
|
||||
// If no delimiter specified, then all files with matching
|
||||
// prefix are included in results
|
||||
obj, err := getObj(path, d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("file to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, obj)
|
||||
|
||||
if max > 0 && (len(objects)+len(cpmap)) == int(max) {
|
||||
pastMax = true
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Since delimiter is specified, we only want results that
|
||||
// do not contain the delimiter beyond the prefix. If the
|
||||
// delimiter exists past the prefix, then the substring
|
||||
// between the prefix and delimiter is part of common prefixes.
|
||||
//
|
||||
// For example:
|
||||
// prefix = A/
|
||||
// delimiter = /
|
||||
// and objects:
|
||||
// A/file
|
||||
// A/B/file
|
||||
// B/C
|
||||
// would return:
|
||||
// objects: A/file
|
||||
// common prefix: A/B/
|
||||
//
|
||||
// Note: No objects are included past the common prefix since
|
||||
// these are all rolled up into the common prefix.
|
||||
// Note: The delimiter can be anything, so we have to operate on
|
||||
// the full path without any assumptions on posix directory hierarchy
|
||||
// here. Usually the delimiter will be "/", but thats not required.
|
||||
suffix := strings.TrimPrefix(path, prefix)
|
||||
before, _, found := strings.Cut(suffix, delimiter)
|
||||
if !found {
|
||||
obj, err := getObj(path, d)
|
||||
if err == ErrSkipObj {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("file to object %q: %w", path, err)
|
||||
}
|
||||
objects = append(objects, obj)
|
||||
if (len(objects) + len(cpmap)) == int(max) {
|
||||
pastMax = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Common prefixes are a set, so should not have duplicates.
|
||||
// These are abstractly a "directory", so need to include the
|
||||
// delimiter at the end.
|
||||
cpmap[prefix+before+delimiter] = struct{}{}
|
||||
if (len(objects) + len(cpmap)) == int(max) {
|
||||
pastMax = true
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return WalkResults{}, err
|
||||
}
|
||||
|
||||
var commonPrefixStrings []string
|
||||
for k := range cpmap {
|
||||
commonPrefixStrings = append(commonPrefixStrings, k)
|
||||
}
|
||||
sort.Strings(commonPrefixStrings)
|
||||
commonPrefixes := make([]types.CommonPrefix, 0, len(commonPrefixStrings))
|
||||
for _, cp := range commonPrefixStrings {
|
||||
pfx := cp
|
||||
commonPrefixes = append(commonPrefixes, types.CommonPrefix{
|
||||
Prefix: &pfx,
|
||||
})
|
||||
}
|
||||
|
||||
return WalkResults{
|
||||
CommonPrefixes: commonPrefixes,
|
||||
Objects: objects,
|
||||
Truncated: truncated,
|
||||
NextMarker: newMarker,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func contains(a string, strs []string) bool {
|
||||
for _, s := range strs {
|
||||
if s == a {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,204 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/backend"
|
||||
)
|
||||
|
||||
type walkTest struct {
|
||||
fsys fs.FS
|
||||
expected backend.WalkResults
|
||||
getobj backend.GetObjFunc
|
||||
}
|
||||
|
||||
func getObj(path string, d fs.DirEntry) (types.Object, error) {
|
||||
if d.IsDir() {
|
||||
etag := getMD5(path)
|
||||
|
||||
fi, err := d.Info()
|
||||
if err != nil {
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
etag := getMD5(path)
|
||||
|
||||
fi, err := d.Info()
|
||||
if err != nil {
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Size: fi.Size(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func getMD5(text string) string {
|
||||
hash := md5.Sum([]byte(text))
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
func TestWalk(t *testing.T) {
|
||||
tests := []walkTest{
|
||||
{
|
||||
// test case from
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html
|
||||
fsys: fstest.MapFS{
|
||||
"sample.jpg": {},
|
||||
"photos/2006/January/sample.jpg": {},
|
||||
"photos/2006/February/sample2.jpg": {},
|
||||
"photos/2006/February/sample3.jpg": {},
|
||||
"photos/2006/February/sample4.jpg": {},
|
||||
},
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetStringPtr("photos/"),
|
||||
}},
|
||||
Objects: []types.Object{{
|
||||
Key: backend.GetStringPtr("sample.jpg"),
|
||||
}},
|
||||
},
|
||||
getobj: getObj,
|
||||
},
|
||||
{
|
||||
// test case single dir/single file
|
||||
fsys: fstest.MapFS{
|
||||
"test/file": {},
|
||||
},
|
||||
expected: backend.WalkResults{
|
||||
CommonPrefixes: []types.CommonPrefix{{
|
||||
Prefix: backend.GetStringPtr("test/"),
|
||||
}},
|
||||
Objects: []types.Object{},
|
||||
},
|
||||
getobj: getObj,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
|
||||
if err != nil {
|
||||
t.Fatalf("walk: %v", err)
|
||||
}
|
||||
|
||||
compareResults(res, tt.expected, t)
|
||||
}
|
||||
}
|
||||
|
||||
func compareResults(got, wanted backend.WalkResults, t *testing.T) {
|
||||
if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
|
||||
t.Errorf("unexpected common prefix, got %v wanted %v",
|
||||
printCommonPrefixes(got.CommonPrefixes),
|
||||
printCommonPrefixes(wanted.CommonPrefixes))
|
||||
}
|
||||
|
||||
if !compareObjects(got.Objects, wanted.Objects) {
|
||||
t.Errorf("unexpected object, got %v wanted %v",
|
||||
printObjects(got.Objects),
|
||||
printObjects(wanted.Objects))
|
||||
}
|
||||
}
|
||||
|
||||
func compareCommonPrefix(a, b []types.CommonPrefix) bool {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return true
|
||||
}
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, cp := range a {
|
||||
if containsCommonPrefix(cp, b) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsCommonPrefix(c types.CommonPrefix, list []types.CommonPrefix) bool {
|
||||
for _, cp := range list {
|
||||
if *c.Prefix == *cp.Prefix {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func printCommonPrefixes(list []types.CommonPrefix) string {
|
||||
res := "["
|
||||
for _, cp := range list {
|
||||
if res == "[" {
|
||||
res = res + *cp.Prefix
|
||||
} else {
|
||||
res = res + ", " + *cp.Prefix
|
||||
}
|
||||
}
|
||||
return res + "]"
|
||||
}
|
||||
|
||||
func compareObjects(a, b []types.Object) bool {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return true
|
||||
}
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, cp := range a {
|
||||
if containsObject(cp, b) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsObject(c types.Object, list []types.Object) bool {
|
||||
for _, cp := range list {
|
||||
if *c.Key == *cp.Key {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func printObjects(list []types.Object) string {
|
||||
res := "["
|
||||
for _, cp := range list {
|
||||
if res == "[" {
|
||||
res = res + *cp.Key
|
||||
} else {
|
||||
res = res + ", " + *cp.Key
|
||||
}
|
||||
}
|
||||
return res + "]"
|
||||
}
|
||||
18
cmd/scoutgw/main.go
Normal file
18
cmd/scoutgw/main.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/scoutgw/backend"
|
||||
"github.com/versity/scoutgw/s3api"
|
||||
"log"
|
||||
)
|
||||
|
||||
func main() {
|
||||
app := fiber.New(fiber.Config{})
|
||||
back := backend.New()
|
||||
if api, err := s3api.New(app, back, ":7070"); err != nil {
|
||||
log.Fatalln(err)
|
||||
} else if err = api.Serve(); err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
var (
|
||||
adminAccess string
|
||||
adminSecret string
|
||||
)
|
||||
|
||||
func adminCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "admin",
|
||||
Usage: "admin CLI tool",
|
||||
Description: `admin CLI tool for interacting with admin api.
|
||||
Here is the available api list:
|
||||
create-user
|
||||
`,
|
||||
Subcommands: []*cli.Command{
|
||||
{
|
||||
Name: "create-user",
|
||||
Usage: "Create a new user",
|
||||
Action: createUser,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "access value for the new user",
|
||||
Required: true,
|
||||
Aliases: []string{"a"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "secret",
|
||||
Usage: "secret value for the new user",
|
||||
Required: true,
|
||||
Aliases: []string{"s"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "role",
|
||||
Usage: "role for the new user",
|
||||
Required: true,
|
||||
Aliases: []string{"r"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "delete-user",
|
||||
Usage: "Delete a user",
|
||||
Action: deleteUser,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "access value for the user to be deleted",
|
||||
Required: true,
|
||||
Aliases: []string{"a"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Flags: []cli.Flag{
|
||||
// TODO: create a configuration file for this
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "admin access account",
|
||||
EnvVars: []string{"ADMIN_ACCESS_KEY_ID", "ADMIN_ACCESS_KEY"},
|
||||
Aliases: []string{"a"},
|
||||
Destination: &adminAccess,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "secret",
|
||||
Usage: "admin secret access key",
|
||||
EnvVars: []string{"ADMIN_SECRET_ACCESS_KEY", "ADMIN_SECRET_KEY"},
|
||||
Aliases: []string{"s"},
|
||||
Destination: &adminSecret,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func createUser(ctx *cli.Context) error {
|
||||
access, secret, role := ctx.String("access"), ctx.String("secret"), ctx.String("role")
|
||||
if access == "" || secret == "" {
|
||||
return fmt.Errorf("invalid input parameters for the new user")
|
||||
}
|
||||
if role != "admin" && role != "user" {
|
||||
return fmt.Errorf("invalid input parameter for role")
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("http://localhost:7070/create-user?access=%v&secret=%v&role=%v", access, secret, role), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256([]byte{})
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("%s\n", body)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteUser(ctx *cli.Context) error {
|
||||
access := ctx.String("access")
|
||||
if access == "" {
|
||||
return fmt.Errorf("invalid input parameter for the new user")
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("http://localhost:7070/delete-user?access=%v", access), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256([]byte{})
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: adminAccess, SecretAccessKey: adminSecret}, req, hexPayload, "s3", region, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("%s\n", body)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,291 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
var (
|
||||
port string
|
||||
rootUserAccess string
|
||||
rootUserSecret string
|
||||
region string
|
||||
certFile, keyFile string
|
||||
kafkaURL, kafkaTopic, kafkaKey string
|
||||
natsURL, natsTopic string
|
||||
logWebhookURL string
|
||||
accessLog string
|
||||
debug bool
|
||||
)
|
||||
|
||||
var (
|
||||
// Version is the latest tag (set within Makefile)
|
||||
Version = "git"
|
||||
// Build is the commit hash (set within Makefile)
|
||||
Build = "norev"
|
||||
// BuildTime is the date/time of build (set within Makefile)
|
||||
BuildTime = "none"
|
||||
)
|
||||
|
||||
func main() {
|
||||
setupSignalHandler()
|
||||
|
||||
app := initApp()
|
||||
|
||||
app.Commands = []*cli.Command{
|
||||
posixCommand(),
|
||||
scoutfsCommand(),
|
||||
adminCommand(),
|
||||
testCommand(),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() {
|
||||
<-sigDone
|
||||
fmt.Fprintf(os.Stderr, "terminating signal caught, shutting down\n")
|
||||
cancel()
|
||||
}()
|
||||
|
||||
if err := app.RunContext(ctx, os.Args); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func initApp() *cli.App {
|
||||
return &cli.App{
|
||||
Name: "versitygw",
|
||||
Usage: "Start S3 gateway service with specified backend storage.",
|
||||
Description: `The S3 gateway is an S3 protocol translator that allows an S3 client
|
||||
to access the supported backend storage as if it was a native S3 service.`,
|
||||
Action: func(ctx *cli.Context) error {
|
||||
return ctx.App.Command("help").Run(ctx)
|
||||
},
|
||||
Flags: initFlags(),
|
||||
}
|
||||
}
|
||||
|
||||
func initFlags() []cli.Flag {
|
||||
return []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "version",
|
||||
Usage: "list versitygw version",
|
||||
Aliases: []string{"v"},
|
||||
Action: func(*cli.Context, bool) error {
|
||||
fmt.Println("Version :", Version)
|
||||
fmt.Println("Build :", Build)
|
||||
fmt.Println("BuildTime:", BuildTime)
|
||||
os.Exit(0)
|
||||
return nil
|
||||
},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "port",
|
||||
Usage: "gateway listen address <ip>:<port> or :<port>",
|
||||
Value: ":7070",
|
||||
Destination: &port,
|
||||
Aliases: []string{"p"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "root user access key",
|
||||
EnvVars: []string{"ROOT_ACCESS_KEY_ID", "ROOT_ACCESS_KEY"},
|
||||
Aliases: []string{"a"},
|
||||
Destination: &rootUserAccess,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "secret",
|
||||
Usage: "root user secret access key",
|
||||
EnvVars: []string{"ROOT_SECRET_ACCESS_KEY", "ROOT_SECRET_KEY"},
|
||||
Aliases: []string{"s"},
|
||||
Destination: &rootUserSecret,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "region",
|
||||
Usage: "s3 region string",
|
||||
Value: "us-east-1",
|
||||
Destination: ®ion,
|
||||
Aliases: []string{"r"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "cert",
|
||||
Usage: "TLS cert file",
|
||||
Destination: &certFile,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "key",
|
||||
Usage: "TLS key file",
|
||||
Destination: &keyFile,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "debug",
|
||||
Usage: "enable debug output",
|
||||
Destination: &debug,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "access-log",
|
||||
Usage: "enable server access logging to specified file",
|
||||
EnvVars: []string{"LOGFILE"},
|
||||
Destination: &accessLog,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "log-webhook-url",
|
||||
Usage: "webhook url to send the audit logs",
|
||||
EnvVars: []string{"WEBHOOK"},
|
||||
Destination: &logWebhookURL,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "event-kafka-url",
|
||||
Usage: "kafka server url to send the bucket notifications.",
|
||||
Destination: &kafkaURL,
|
||||
Aliases: []string{"eku"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "event-kafka-topic",
|
||||
Usage: "kafka server pub-sub topic to send the bucket notifications to",
|
||||
Destination: &kafkaTopic,
|
||||
Aliases: []string{"ekt"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "event-kafka-key",
|
||||
Usage: "kafka server put-sub topic key to send the bucket notifications to",
|
||||
Destination: &kafkaKey,
|
||||
Aliases: []string{"ekk"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "event-nats-url",
|
||||
Usage: "nats server url to send the bucket notifications",
|
||||
Destination: &natsURL,
|
||||
Aliases: []string{"enu"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "event-nats-topic",
|
||||
Usage: "nats server pub-sub topic to send the bucket notifications to",
|
||||
Destination: &natsTopic,
|
||||
Aliases: []string{"ent"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func runGateway(ctx *cli.Context, be backend.Backend, s auth.Storer) error {
|
||||
app := fiber.New(fiber.Config{
|
||||
AppName: "versitygw",
|
||||
ServerHeader: "VERSITYGW",
|
||||
BodyLimit: 5 * 1024 * 1024 * 1024,
|
||||
})
|
||||
|
||||
var opts []s3api.Option
|
||||
|
||||
if certFile != "" || keyFile != "" {
|
||||
if certFile == "" {
|
||||
return fmt.Errorf("TLS key specified without cert file")
|
||||
}
|
||||
if keyFile == "" {
|
||||
return fmt.Errorf("TLS cert specified without key file")
|
||||
}
|
||||
|
||||
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("tls: load certs: %v", err)
|
||||
}
|
||||
opts = append(opts, s3api.WithTLS(cert))
|
||||
}
|
||||
|
||||
if debug {
|
||||
opts = append(opts, s3api.WithDebug())
|
||||
}
|
||||
|
||||
err := s.InitIAM()
|
||||
if err != nil {
|
||||
return fmt.Errorf("init iam: %w", err)
|
||||
}
|
||||
|
||||
iam, err := auth.NewInternal(s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("setup internal iam service: %w", err)
|
||||
}
|
||||
|
||||
logger, err := s3log.InitLogger(&s3log.LogConfig{
|
||||
LogFile: accessLog,
|
||||
WebhookURL: logWebhookURL,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("setup logger: %w", err)
|
||||
}
|
||||
|
||||
evSender, err := s3event.InitEventSender(&s3event.EventConfig{
|
||||
KafkaURL: kafkaURL,
|
||||
KafkaTopic: kafkaTopic,
|
||||
KafkaTopicKey: kafkaKey,
|
||||
NatsURL: natsURL,
|
||||
NatsTopic: natsTopic,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to connect to the message broker: %w", err)
|
||||
}
|
||||
|
||||
srv, err := s3api.New(app, be, middlewares.RootUserConfig{
|
||||
Access: rootUserAccess,
|
||||
Secret: rootUserSecret,
|
||||
}, port, region, iam, logger, evSender, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init gateway: %v", err)
|
||||
}
|
||||
|
||||
c := make(chan error, 1)
|
||||
go func() { c <- srv.Serve() }()
|
||||
|
||||
// for/select blocks until shutdown
|
||||
Loop:
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
break Loop
|
||||
case err = <-c:
|
||||
break Loop
|
||||
case <-sigHup:
|
||||
if logger != nil {
|
||||
err = logger.HangUp()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("HUP logger: %w", err)
|
||||
break Loop
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
be.Shutdown()
|
||||
if logger != nil {
|
||||
lerr := logger.Shutdown()
|
||||
if lerr != nil {
|
||||
fmt.Fprintf(os.Stderr, "shutdown logger: %v\n", lerr)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
)
|
||||
|
||||
func posixCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "posix",
|
||||
Usage: "posix filesystem storage backend",
|
||||
Description: `Any posix filesystem that supports extended attributes. The top level
|
||||
directory for the gateway must be provided. All sub directories of the
|
||||
top level directory are treated as buckets, and all files/directories
|
||||
below the "bucket directory" are treated as the objects. The object
|
||||
name is split on "/" separator to translate to posix storage.
|
||||
For example:
|
||||
top level: /mnt/fs/gwroot
|
||||
bucket: mybucket
|
||||
object: a/b/c/myobject
|
||||
will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
|
||||
Action: runPosix,
|
||||
}
|
||||
}
|
||||
|
||||
func runPosix(ctx *cli.Context) error {
|
||||
if ctx.NArg() == 0 {
|
||||
return fmt.Errorf("no directory provided for operation")
|
||||
}
|
||||
|
||||
be, err := posix.New(ctx.Args().Get(0))
|
||||
if err != nil {
|
||||
return fmt.Errorf("init posix: %v", err)
|
||||
}
|
||||
|
||||
return runGateway(ctx, be, be)
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/backend/scoutfs"
|
||||
)
|
||||
|
||||
var (
|
||||
glacier bool
|
||||
)
|
||||
|
||||
func scoutfsCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "scoutfs",
|
||||
Usage: "scoutfs filesystem storage backend",
|
||||
Description: `Support for ScoutFS.
|
||||
The top level directory for the gateway must be provided. All sub directories
|
||||
of the top level directory are treated as buckets, and all files/directories
|
||||
below the "bucket directory" are treated as the objects. The object name is
|
||||
split on "/" separator to translate to posix storage.
|
||||
For example:
|
||||
top level: /mnt/fs/gwroot
|
||||
bucket: mybucket
|
||||
object: a/b/c/myobject
|
||||
will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject
|
||||
|
||||
ScoutFS contains optimizations for multipart uploads using extent
|
||||
move interfaces as well as support for tiered filesystems.`,
|
||||
Action: runScoutfs,
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "glacier",
|
||||
Usage: "enable glacier emulation mode",
|
||||
Aliases: []string{"g"},
|
||||
Destination: &glacier,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func runScoutfs(ctx *cli.Context) error {
|
||||
if ctx.NArg() == 0 {
|
||||
return fmt.Errorf("no directory provided for operation")
|
||||
}
|
||||
|
||||
var opts []scoutfs.Option
|
||||
if glacier {
|
||||
opts = append(opts, scoutfs.WithGlacierEmulation())
|
||||
}
|
||||
|
||||
be, err := scoutfs.New(ctx.Args().Get(0), opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init scoutfs: %v", err)
|
||||
}
|
||||
|
||||
return runGateway(ctx, be, be)
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
var (
|
||||
sigDone = make(chan bool, 1)
|
||||
sigHup = make(chan bool, 1)
|
||||
)
|
||||
|
||||
func setupSignalHandler() {
|
||||
sigs := make(chan os.Signal, 1)
|
||||
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
|
||||
|
||||
go func() {
|
||||
for sig := range sigs {
|
||||
fmt.Fprintf(os.Stderr, "caught signal %v\n", sig)
|
||||
switch sig {
|
||||
case syscall.SIGINT, syscall.SIGTERM:
|
||||
sigDone <- true
|
||||
case syscall.SIGHUP:
|
||||
sigHup <- true
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -1,311 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/integration"
|
||||
)
|
||||
|
||||
var (
|
||||
awsID string
|
||||
awsSecret string
|
||||
endpoint string
|
||||
prefix string
|
||||
dstBucket string
|
||||
partSize int64
|
||||
objSize int64
|
||||
concurrency int
|
||||
files int
|
||||
upload bool
|
||||
download bool
|
||||
pathStyle bool
|
||||
checksumDisable bool
|
||||
)
|
||||
|
||||
func testCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "test",
|
||||
Usage: "Client side testing command for the gateway",
|
||||
Description: `The testing CLI is used to test group of versitygw actions.
|
||||
It also includes some performance and stress testing`,
|
||||
Subcommands: initTestCommands(),
|
||||
Flags: initTestFlags(),
|
||||
}
|
||||
}
|
||||
|
||||
func initTestFlags() []cli.Flag {
|
||||
return []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "aws user access key",
|
||||
EnvVars: []string{"AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY"},
|
||||
Aliases: []string{"a"},
|
||||
Destination: &awsID,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "secret",
|
||||
Usage: "aws user secret access key",
|
||||
EnvVars: []string{"AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY"},
|
||||
Aliases: []string{"s"},
|
||||
Destination: &awsSecret,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "endpoint",
|
||||
Usage: "s3 server endpoint",
|
||||
Destination: &endpoint,
|
||||
Aliases: []string{"e"},
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "debug",
|
||||
Usage: "enable debug mode",
|
||||
Aliases: []string{"d"},
|
||||
Destination: &debug,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func initTestCommands() []*cli.Command {
|
||||
return []*cli.Command{
|
||||
{
|
||||
Name: "bucket-actions",
|
||||
Usage: "Test bucket creation, checking the existence, deletes it.",
|
||||
Description: `Calls s3 gateway create-bucket action to create a new bucket,
|
||||
calls head-bucket action to check the existence, then calls delete-bucket action to delete the bucket.`,
|
||||
Action: getAction(integration.TestMakeBucket),
|
||||
},
|
||||
{
|
||||
Name: "object-actions",
|
||||
Usage: "Test put/get/delete/copy objects.",
|
||||
Description: `Creates a bucket with s3 gateway action, puts an object in it,
|
||||
tries to copy into another bucket, that doesn't exist, creates the destination bucket for copying,
|
||||
copies the object, get's the object to check the length and content,
|
||||
get's the copied object to check the length and content, deletes all the objects inside the source bucket,
|
||||
deletes both the objects and buckets.`,
|
||||
Action: getAction(integration.TestPutGetObject),
|
||||
},
|
||||
{
|
||||
Name: "put-get-mp-object",
|
||||
Usage: "Test put & get multipart object.",
|
||||
Description: `Creates a bucket with s3 gateway action, puts an object in it with multipart upload,
|
||||
gets the object from the bucket, deletes both the object and bucket.`,
|
||||
Action: getAction(integration.TestPutGetMPObject),
|
||||
},
|
||||
{
|
||||
Name: "put-dir-object",
|
||||
Usage: "Test put directory object.",
|
||||
Description: `Creates a bucket with s3 gateway action, puts a directory object in it,
|
||||
lists the bucket's objects, deletes both the objects and bucket.`,
|
||||
Action: getAction(integration.TestPutDirObject),
|
||||
},
|
||||
{
|
||||
Name: "list-objects",
|
||||
Usage: "Test list-objects action.",
|
||||
Description: `Creates a bucket with s3 gateway action, puts 2 directory objects in it,
|
||||
lists the bucket's objects, deletes both the objects and bucket.`,
|
||||
Action: getAction(integration.TestListObject),
|
||||
},
|
||||
{
|
||||
Name: "abort-mp",
|
||||
Usage: "Tests abort-multipart-upload action.",
|
||||
Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
|
||||
lists the multipart upload, aborts the multipart upload, lists the multipart upload again,
|
||||
deletes both the objects and bucket.`,
|
||||
Action: getAction(integration.TestListAbortMultiPartObject),
|
||||
},
|
||||
{
|
||||
Name: "list-parts",
|
||||
Usage: "Tests list-parts action.",
|
||||
Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
|
||||
lists the upload parts, deletes both the objects and bucket.`,
|
||||
Action: getAction(integration.TestListMultiParts),
|
||||
},
|
||||
{
|
||||
Name: "incorrect-mp",
|
||||
Usage: "Tests incorrect multipart case.",
|
||||
Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
|
||||
uploads different parts, completes the multipart upload with incorrect part numbers,
|
||||
calls the head-object action, compares the content length, removes both the object and bucket`,
|
||||
Action: getAction(integration.TestIncorrectMultiParts),
|
||||
},
|
||||
{
|
||||
Name: "incomplete-mp",
|
||||
Usage: "Tests incomplete multi parts.",
|
||||
Description: `Creates a bucket with s3 gateway action, creates a multipart upload,
|
||||
upload a part, lists the parts, checks if the uploaded part is in the list,
|
||||
removes both the object and the bucket`,
|
||||
Action: getAction(integration.TestIncompleteMultiParts),
|
||||
},
|
||||
{
|
||||
Name: "incomplete-put-object",
|
||||
Usage: "Tests incomplete put objects case.",
|
||||
Description: `Creates a bucket with s3 gateway action, puts an object in it,
|
||||
gets the object with head-object action, expects the object to be got,
|
||||
removes both the object and bucket`,
|
||||
Action: getAction(integration.TestIncompletePutObject),
|
||||
},
|
||||
{
|
||||
Name: "get-range",
|
||||
Usage: "Tests get object by range.",
|
||||
Description: `Creates a bucket with s3 gateway action, puts an object in it,
|
||||
gets the object by specifying the object range, compares the range with the original one,
|
||||
removes both the object and the bucket`,
|
||||
Action: getAction(integration.TestRangeGet),
|
||||
},
|
||||
{
|
||||
Name: "invalid-mp",
|
||||
Usage: "Tests invalid multi part case.",
|
||||
Description: `Creates a bucket with s3 gateway action, creates a multi part upload,
|
||||
uploads an invalid part, gets the object with head-object action, expects to get error,
|
||||
removes both the object and bucket`,
|
||||
Action: getAction(integration.TestInvalidMultiParts),
|
||||
},
|
||||
{
|
||||
Name: "object-tag-actions",
|
||||
Usage: "Tests get/put/delete object tag actions.",
|
||||
Description: `Creates a bucket with s3 gateway action, puts an object in it,
|
||||
puts some tags for the object, gets the tags, compares the results, removes the tags,
|
||||
gets the tags again, checks it to be empty, then removes both the object and bucket`,
|
||||
Action: getAction(integration.TestPutGetRemoveTags),
|
||||
},
|
||||
{
|
||||
Name: "bucket-acl-actions",
|
||||
Usage: "Tests put/get bucket actions.",
|
||||
Description: `Creates a bucket with s3 gateway action, puts some bucket acls
|
||||
gets the acl, verifies it, then removes the bucket`,
|
||||
Action: getAction(integration.TestAclActions),
|
||||
},
|
||||
{
|
||||
Name: "full-flow",
|
||||
Usage: "Tests the full flow of gateway.",
|
||||
Description: `Runs all the available tests to test the full flow of the gateway.`,
|
||||
Action: getAction(integration.TestFullFlow),
|
||||
},
|
||||
{
|
||||
Name: "bench",
|
||||
Usage: "Runs download/upload performance test on the gateway",
|
||||
Description: `Uploads/downloads some number(specified by flags) of files with some capacity(bytes).
|
||||
Logs the results to the console`,
|
||||
Flags: []cli.Flag{
|
||||
&cli.IntFlag{
|
||||
Name: "files",
|
||||
Usage: "Number of objects to read/write",
|
||||
Value: 1,
|
||||
Destination: &files,
|
||||
},
|
||||
&cli.Int64Flag{
|
||||
Name: "objsize",
|
||||
Usage: "Uploading object size",
|
||||
Value: 0,
|
||||
Destination: &objSize,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "prefix",
|
||||
Usage: "Object name prefix",
|
||||
Destination: &prefix,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "upload",
|
||||
Usage: "Upload data to the gateway",
|
||||
Value: false,
|
||||
Destination: &upload,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "download",
|
||||
Usage: "Download data to the gateway",
|
||||
Value: false,
|
||||
Destination: &download,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "bucket",
|
||||
Usage: "Destination bucket name to read/write data",
|
||||
Destination: &dstBucket,
|
||||
},
|
||||
&cli.Int64Flag{
|
||||
Name: "partSize",
|
||||
Usage: "Upload/download size per thread",
|
||||
Value: 64 * 1024 * 1024,
|
||||
Destination: &partSize,
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "concurrency",
|
||||
Usage: "Upload/download threads per object",
|
||||
Value: 1,
|
||||
Destination: &concurrency,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "pathStyle",
|
||||
Usage: "Use Pathstyle bucket addressing",
|
||||
Value: false,
|
||||
Destination: &pathStyle,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "checksumDis",
|
||||
Usage: "Disable server checksum",
|
||||
Value: false,
|
||||
Destination: &checksumDisable,
|
||||
},
|
||||
},
|
||||
Action: func(ctx *cli.Context) error {
|
||||
if upload && download {
|
||||
return fmt.Errorf("must only specify one of upload or download")
|
||||
}
|
||||
if !upload && !download {
|
||||
return fmt.Errorf("must specify one of upload or download")
|
||||
}
|
||||
|
||||
if dstBucket == "" {
|
||||
return fmt.Errorf("must specify bucket")
|
||||
}
|
||||
|
||||
opts := []integration.Option{
|
||||
integration.WithAccess(awsID),
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
integration.WithConcurrency(concurrency),
|
||||
integration.WithPartSize(partSize),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
if pathStyle {
|
||||
opts = append(opts, integration.WithPathStyle())
|
||||
}
|
||||
if checksumDisable {
|
||||
opts = append(opts, integration.WithDisableChecksum())
|
||||
}
|
||||
|
||||
s3conf := integration.NewS3Conf(opts...)
|
||||
|
||||
return integration.TestPerformance(s3conf, upload, download, files, objSize, dstBucket, prefix)
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type testFunc func(*integration.S3Conf)
|
||||
|
||||
func getAction(tf testFunc) func(*cli.Context) error {
|
||||
return func(ctx *cli.Context) error {
|
||||
opts := []integration.Option{
|
||||
integration.WithAccess(awsID),
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
|
||||
s := integration.NewS3Conf(opts...)
|
||||
tf(s)
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("RAN:", integration.RunCount, "PASS:", integration.PassCount, "FAIL:", integration.FailCount)
|
||||
if integration.FailCount > 0 {
|
||||
return fmt.Errorf("test failed with %v errors", integration.FailCount)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
67
go.mod
67
go.mod
@@ -1,60 +1,37 @@
|
||||
module github.com/versity/versitygw
|
||||
module github.com/versity/scoutgw
|
||||
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.20.0
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1
|
||||
github.com/aws/smithy-go v1.14.0
|
||||
github.com/gofiber/fiber/v2 v2.48.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.18.0
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1
|
||||
github.com/gofiber/fiber/v2 v2.45.0
|
||||
github.com/valyala/fasthttp v1.47.0
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/nats-io/nats.go v1.28.0
|
||||
github.com/pkg/xattr v0.4.9
|
||||
github.com/segmentio/kafka-go v0.4.42
|
||||
github.com/urfave/cli/v2 v2.25.7
|
||||
github.com/valyala/fasthttp v1.48.0
|
||||
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
|
||||
golang.org/x/sys v0.10.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.21.1 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.9.20 // indirect
|
||||
github.com/nats-io/nkeys v0.4.4 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.18 // indirect
|
||||
github.com/stretchr/testify v1.8.1 // indirect
|
||||
golang.org/x/crypto v0.11.0 // indirect
|
||||
google.golang.org/protobuf v1.31.0 // indirect
|
||||
golang.org/x/sys v0.8.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.32
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.31
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||
github.com/klauspost/compress v1.16.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2 // indirect
|
||||
github.com/aws/smithy-go v1.13.5 // indirect
|
||||
github.com/klauspost/compress v1.16.5 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/mattn/go-isatty v0.0.18 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.14 // indirect
|
||||
github.com/philhofer/fwd v1.1.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.4 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect
|
||||
github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect
|
||||
github.com/tinylib/msgp v1.1.8 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/tcplisten v1.0.0 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
||||
)
|
||||
|
||||
180
go.sum
180
go.sum
@@ -1,167 +1,115 @@
|
||||
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
|
||||
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/aws/aws-sdk-go-v2 v1.20.0 h1:INUDpYLt4oiPOJl0XwZDK2OVAVf0Rzo+MGVTv9f+gy8=
|
||||
github.com/aws/aws-sdk-go-v2 v1.20.0/go.mod h1:uWOr0m0jDsiWw8nnXiqZ+YG6LdvAlGYDLLf2NmHZoy4=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11 h1:/MS8AzqYNAhhRNalOmxUvYs8VEbNGifTnzhPFdcRQkQ=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11/go.mod h1:va22++AdXht4ccO3kH2SHkHHYvZ2G9Utz+CXKmm2CaU=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.32 h1:tqEOvkbTxwEV7hToRcJ1xZRjcATqwDVsWbAscgRKyNI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.32/go.mod h1:U3ZF0fQRRA4gnbn9GGvOWLoT2EzzZfAWeKwnVrm1rDc=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.31 h1:vJyON3lG7R8VOErpJJBclBADiWTwzcwdkQpTKx8D2sk=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.31/go.mod h1:T4sESjBtY2lNxLgkIASmeP57b5j7hTQqCbqG0tWnxC4=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7 h1:X3H6+SU21x+76LRglk21dFRgMTJMa5QcpW+SqUf5BBg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7/go.mod h1:3we0V09SwcJBzNlnyovrR2wWJhWmVdqAsmVs4uronv8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76 h1:DJ1kHj0GI9BbX+XhF0kHxlzOVjcncmDUXmCvXdbfdAE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76/go.mod h1:/AZCdswMSgwpB2yMSFfY5H4pVeBLnCuPehdmO/r3xSM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37 h1:zr/gxAZkMcvP71ZhQOcvdm8ReLjFgIXnIn0fw5AM7mo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37/go.mod h1:Pdn4j43v49Kk6+82spO3Tu5gSeQXRsxo56ePPQAvFiA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31 h1:0HCMIkAkVY9KMgueD8tf4bRTUanzEYvhw7KkPXIMpO0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31/go.mod h1:fTJDMe8LOFYtqiFFFeHA+SVMAwqLhoq0kcInYoLa9Js=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38 h1:+i1DOFrW3YZ3apE45tCal9+aDKK6kNEbW6Ib7e1nFxE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38/go.mod h1:1/jLp0OgOaWIetycOmycW+vYTYgTZFPttJQRgsI1PoU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 h1:U5yySdwt2HPo/pnQec04DImLzWORbeWML1fJiLkKruI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0/go.mod h1:EhC/83j8/hL/UB1WmExo3gkElaja/KlmZM/gl1rTfjM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12 h1:uAiiHnWihGP2rVp64fHwzLDrswGjEjsPszwRYMiYQPU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12/go.mod h1:fUTHpOXqRQpXvEpDPSa3zxCc2fnpW6YnBoba+eQr+Bg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32 h1:kvN1jPHr9UffqqG3bSgZ8tx4+1zKVHz/Ktw/BwW6hX8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32/go.mod h1:QmMEM7es84EUkbYWcpnkx8i5EW2uERPfrTFeOch128Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31 h1:auGDJ0aLZahF5SPvkJ6WcUuX7iQ7kyl2MamV7Tm8QBk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31/go.mod h1:3+lloe3sZuBQw1aBc5MyndvodzQlyqCZ7x1QPDHaWP4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0 h1:Wgjft9X4W5pMeuqgPCHIQtbZ87wsgom7S5F8obreg+c=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0/go.mod h1:FWNzS4+zcWAP05IF7TDYTY1ysZAzIvogxWaDT9p8fsA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1 h1:mTgFVlfQT8gikc5+/HwD8UL9jnUro5MGv8n/VEYF12I=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1/go.mod h1:6SOWLiobcZZshbmECRTADIRYliPL0etqFSigauQEeT0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 h1:DSNpSbfEgFXRV+IfEcKE5kTbqxm+MeF5WgyeRlsLnHY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.13.1/go.mod h1:TC9BubuFMVScIU+TLKamO6VZiYTkYoEHqlSQwAe2omw=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 h1:hd0SKLMdOL/Sl6Z0np1PX9LeH2gqNtBe0MhTedA8MGI=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1/go.mod h1:XO/VcyoQ8nKyKfFW/3DMsRQXsfh/052tHTWmg3xBXRg=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.21.1 h1:pAOJj+80tC8sPVgSDHzMYD6KLWsaLQ1kZw31PTeORbs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.21.1/go.mod h1:G8SbvL0rFk4WOJroU8tKBczhsbhj2p/YY7qeJezJ3CI=
|
||||
github.com/aws/smithy-go v1.14.0 h1:+X90sB94fizKjDmwb4vyl2cTTPXTE5E2G/1mjByb0io=
|
||||
github.com/aws/smithy-go v1.14.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25 h1:AzwRi5OKKwo4QNqPf7TjeO+tK8AyOK3GVSwmRPo7/Cs=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25/go.mod h1:SUbB4wcbSEyCvqBxv/O/IBf93RbEze7U7OnoTlpPB+g=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28 h1:vGWm5vTpMr39tEZfQeDiDAMgk+5qsnvRny3FjLpnH5w=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28/go.mod h1:spfrICMD6wCAhjhzHuy6DOZZ+LAIY10UxhUmLzpJTTs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 h1:0iKliEXAcCa2qVtRs7Ot5hItA2MsufrphbRFlz1Owxo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2 h1:NbWkRxEEIRSCqxhsHQuMiTH7yo+JZW1gp8v3elSVMTQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2/go.mod h1:4tfW5l4IAB32VWCDEBxCRtR9T4BWy4I4kr1spr8NgZM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 h1:O+9nAy9Bb6bJFTpeNFtd9UfHbgxO1o4ZDAM9rQp5NsY=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1/go.mod h1:J9kLNzEiHSeGMyN7238EjJmBpCniVzFda75Gxl/NqB8=
|
||||
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
|
||||
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/gofiber/fiber/v2 v2.48.0 h1:cRVMCb9aUJDsyHxGFLwz/sGzDggdailZZyptU9F9cU0=
|
||||
github.com/gofiber/fiber/v2 v2.48.0/go.mod h1:xqJgfqrc23FJuqGOW6DVgi3HyZEm2Mn9pRqUb2kHSX8=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/gofiber/fiber/v2 v2.45.0 h1:p4RpkJT9GAW6parBSbcNFH2ApnAuW3OzaQzbOCoDu+s=
|
||||
github.com/gofiber/fiber/v2 v2.45.0/go.mod h1:DNl0/c37WLe0g92U6lx1VMQuxGUQY5V7EIaVoEsUffc=
|
||||
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
|
||||
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
|
||||
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
|
||||
github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
|
||||
github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4=
|
||||
github.com/nats-io/nats-server/v2 v2.9.20 h1:bt1dW6xsL1hWWwv7Hovm+EJt5L6iplyqlgEFkoEUk0k=
|
||||
github.com/nats-io/nats-server/v2 v2.9.20/go.mod h1:aTb/xtLCGKhfTFLxP591CMWfkdgBmcUUSkiSOe5A3gw=
|
||||
github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c=
|
||||
github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc=
|
||||
github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA=
|
||||
github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
|
||||
github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
|
||||
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
|
||||
github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
|
||||
github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
|
||||
github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
|
||||
github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
|
||||
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU=
|
||||
github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg=
|
||||
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4=
|
||||
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8=
|
||||
github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4=
|
||||
github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk=
|
||||
github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
|
||||
github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
|
||||
github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
|
||||
github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
|
||||
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc=
|
||||
github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
|
||||
github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c=
|
||||
github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
|
||||
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9/go.mod h1:gJsq73k+4685y+rbDIpPY8i/5GbsiwP6JFoFyUDB1fQ=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
|
||||
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
|
||||
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
|
||||
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
|
||||
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
|
||||
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"hash"
|
||||
"io"
|
||||
)
|
||||
|
||||
type RReader struct {
|
||||
buf []byte
|
||||
dataleft int
|
||||
hash hash.Hash
|
||||
}
|
||||
|
||||
func NewDataReader(totalsize, bufsize int) *RReader {
|
||||
b := make([]byte, bufsize)
|
||||
rand.Read(b)
|
||||
return &RReader{
|
||||
buf: b,
|
||||
dataleft: totalsize,
|
||||
hash: sha256.New(),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *RReader) Read(p []byte) (int, error) {
|
||||
n := min(len(p), len(r.buf), r.dataleft)
|
||||
r.dataleft -= n
|
||||
err := error(nil)
|
||||
if n == 0 {
|
||||
err = io.EOF
|
||||
}
|
||||
r.hash.Write(r.buf[:n])
|
||||
return copy(p, r.buf[:n]), err
|
||||
}
|
||||
|
||||
func (r *RReader) Sum() []byte {
|
||||
return r.hash.Sum(nil)
|
||||
}
|
||||
|
||||
type ZReader struct {
|
||||
buf []byte
|
||||
dataleft int
|
||||
}
|
||||
|
||||
func NewZeroReader(totalsize, bufsize int) *ZReader {
|
||||
b := make([]byte, bufsize)
|
||||
return &ZReader{buf: b, dataleft: totalsize}
|
||||
}
|
||||
|
||||
func (r *ZReader) Read(p []byte) (int, error) {
|
||||
n := min(len(p), len(r.buf), r.dataleft)
|
||||
r.dataleft -= n
|
||||
err := error(nil)
|
||||
if n == 0 {
|
||||
err = io.EOF
|
||||
}
|
||||
return copy(p, r.buf[:n]), err
|
||||
}
|
||||
|
||||
func min(values ...int) int {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
min := values[0]
|
||||
for _, v := range values {
|
||||
if v < min {
|
||||
min = v
|
||||
}
|
||||
}
|
||||
|
||||
return min
|
||||
}
|
||||
|
||||
type NW struct{}
|
||||
|
||||
func NewNullWriter() NW {
|
||||
return NW{}
|
||||
}
|
||||
|
||||
func (NW) WriteAt(p []byte, off int64) (n int, err error) {
|
||||
return len(p), nil
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package integration
|
||||
|
||||
import "fmt"
|
||||
|
||||
var (
|
||||
colorReset = "\033[0m"
|
||||
colorRed = "\033[31m"
|
||||
colorGreen = "\033[32m"
|
||||
colorCyan = "\033[36m"
|
||||
)
|
||||
|
||||
var (
|
||||
RunCount = 0
|
||||
PassCount = 0
|
||||
FailCount = 0
|
||||
)
|
||||
|
||||
func runF(format string, a ...interface{}) {
|
||||
RunCount++
|
||||
fmt.Printf(colorCyan+"RUN "+colorReset+format+"\n", a...)
|
||||
}
|
||||
|
||||
func failF(format string, a ...interface{}) {
|
||||
FailCount++
|
||||
fmt.Printf(colorRed+"FAIL "+colorReset+format+"\n", a...)
|
||||
}
|
||||
|
||||
func passF(format string, a ...interface{}) {
|
||||
PassCount++
|
||||
fmt.Printf(colorGreen+"PASS "+colorReset+format+"\n", a...)
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/smithy-go/middleware"
|
||||
)
|
||||
|
||||
type S3Conf struct {
|
||||
awsID string
|
||||
awsSecret string
|
||||
awsRegion string
|
||||
endpoint string
|
||||
checksumDisable bool
|
||||
pathStyle bool
|
||||
PartSize int64
|
||||
Concurrency int
|
||||
debug bool
|
||||
}
|
||||
|
||||
func NewS3Conf(opts ...Option) *S3Conf {
|
||||
s := &S3Conf{}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(s)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
type Option func(*S3Conf)
|
||||
|
||||
func WithAccess(ak string) Option {
|
||||
return func(s *S3Conf) { s.awsID = ak }
|
||||
}
|
||||
func WithSecret(sk string) Option {
|
||||
return func(s *S3Conf) { s.awsSecret = sk }
|
||||
}
|
||||
func WithRegion(r string) Option {
|
||||
return func(s *S3Conf) { s.awsRegion = r }
|
||||
}
|
||||
func WithEndpoint(e string) Option {
|
||||
return func(s *S3Conf) { s.endpoint = e }
|
||||
}
|
||||
func WithDisableChecksum() Option {
|
||||
return func(s *S3Conf) { s.checksumDisable = true }
|
||||
}
|
||||
func WithPathStyle() Option {
|
||||
return func(s *S3Conf) { s.pathStyle = true }
|
||||
}
|
||||
func WithPartSize(p int64) Option {
|
||||
return func(s *S3Conf) { s.PartSize = p }
|
||||
}
|
||||
func WithConcurrency(c int) Option {
|
||||
return func(s *S3Conf) { s.Concurrency = c }
|
||||
}
|
||||
func WithDebug() Option {
|
||||
return func(s *S3Conf) { s.debug = true }
|
||||
}
|
||||
|
||||
func (c *S3Conf) getCreds() credentials.StaticCredentialsProvider {
|
||||
// TODO support token/IAM
|
||||
if c.awsSecret == "" {
|
||||
c.awsSecret = os.Getenv("AWS_SECRET_ACCESS_KEY")
|
||||
}
|
||||
if c.awsSecret == "" {
|
||||
log.Fatal("no AWS_SECRET_ACCESS_KEY found")
|
||||
}
|
||||
|
||||
return credentials.NewStaticCredentialsProvider(c.awsID, c.awsSecret, "")
|
||||
}
|
||||
|
||||
func (c *S3Conf) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
PartitionID: "aws",
|
||||
URL: c.endpoint,
|
||||
SigningRegion: c.awsRegion,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *S3Conf) Config() aws.Config {
|
||||
creds := c.getCreds()
|
||||
|
||||
tr := &http.Transport{}
|
||||
client := &http.Client{Transport: tr}
|
||||
|
||||
opts := []func(*config.LoadOptions) error{
|
||||
config.WithRegion(c.awsRegion),
|
||||
config.WithCredentialsProvider(creds),
|
||||
config.WithHTTPClient(client),
|
||||
}
|
||||
|
||||
if c.endpoint != "" && c.endpoint != "aws" {
|
||||
opts = append(opts,
|
||||
config.WithEndpointResolverWithOptions(c))
|
||||
}
|
||||
|
||||
if c.checksumDisable {
|
||||
opts = append(opts,
|
||||
config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
|
||||
}
|
||||
|
||||
if c.debug {
|
||||
opts = append(opts,
|
||||
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
|
||||
}
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(
|
||||
context.TODO(), opts...)
|
||||
if err != nil {
|
||||
log.Fatalln("error:", err)
|
||||
}
|
||||
|
||||
return cfg
|
||||
}
|
||||
|
||||
func (c *S3Conf) UploadData(r io.Reader, bucket, object string) error {
|
||||
uploader := manager.NewUploader(s3.NewFromConfig(c.Config()))
|
||||
uploader.PartSize = c.PartSize
|
||||
uploader.Concurrency = c.Concurrency
|
||||
|
||||
upinfo := &s3.PutObjectInput{
|
||||
Body: r,
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
}
|
||||
|
||||
_, err := uploader.Upload(context.Background(), upinfo)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *S3Conf) DownloadData(w io.WriterAt, bucket, object string) (int64, error) {
|
||||
downloader := manager.NewDownloader(s3.NewFromConfig(c.Config()))
|
||||
downloader.PartSize = c.PartSize
|
||||
downloader.Concurrency = c.Concurrency
|
||||
|
||||
downinfo := &s3.GetObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
}
|
||||
|
||||
return downloader.Download(context.Background(), w, downinfo)
|
||||
}
|
||||
1540
integration/tests.go
1540
integration/tests.go
File diff suppressed because it is too large
Load Diff
@@ -1,163 +0,0 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
)
|
||||
|
||||
func setup(s *S3Conf, bucket string) error {
|
||||
s3client := s3.NewFromConfig(s.Config())
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
cancel()
|
||||
return err
|
||||
}
|
||||
|
||||
func teardown(s *S3Conf, bucket string) error {
|
||||
s3client := s3.NewFromConfig(s.Config())
|
||||
|
||||
deleteObject := func(bucket, key, versionId *string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
VersionId: versionId,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete object %v: %v", *key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
in := &s3.ListObjectsV2Input{Bucket: &bucket}
|
||||
for {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.ListObjectsV2(ctx, in)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list objects: %v", err)
|
||||
}
|
||||
|
||||
for _, item := range out.Contents {
|
||||
err = deleteObject(&bucket, item.Key, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if out.IsTruncated {
|
||||
in.ContinuationToken = out.ContinuationToken
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
cancel()
|
||||
return err
|
||||
}
|
||||
|
||||
func isEqual(a, b []byte) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i, d := range a {
|
||||
if d != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func contains(name string, list []types.Object) bool {
|
||||
for _, item := range list {
|
||||
fmt.Println(*item.Key)
|
||||
if strings.EqualFold(name, *item.Key) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsUID(name, id string, list []types.MultipartUpload) bool {
|
||||
for _, item := range list {
|
||||
if strings.EqualFold(name, *item.Key) && strings.EqualFold(id, *item.UploadId) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsPart(part int32, list []types.Part) bool {
|
||||
for _, item := range list {
|
||||
if item.PartNumber == part {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func areTagsSame(tags1, tags2 []types.Tag) bool {
|
||||
if len(tags1) != len(tags2) {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, tag := range tags1 {
|
||||
if !containsTag(tag, tags2) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func containsTag(tag types.Tag, list []types.Tag) bool {
|
||||
for _, item := range list {
|
||||
if *item.Key == *tag.Key && *item.Value == *tag.Value {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func checkGrants(grts1, grts2 []types.Grant) bool {
|
||||
if len(grts1) != len(grts2) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i, grt := range grts1 {
|
||||
if grt.Permission != grts2[i].Permission {
|
||||
return false
|
||||
}
|
||||
if *grt.Grantee.ID != *grts2[i].Grantee.ID {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func execCommand(args ...string) ([]byte, error) {
|
||||
cmd := exec.Command("./versitygw", args...)
|
||||
|
||||
return cmd.CombinedOutput()
|
||||
}
|
||||
|
||||
func getString(str *string) string {
|
||||
if str == nil {
|
||||
return ""
|
||||
}
|
||||
return *str
|
||||
}
|
||||
37
runtests.sh
37
runtests.sh
@@ -1,37 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# make temp dirs
|
||||
mkdir /tmp/gw
|
||||
rm -rf /tmp/covdata
|
||||
mkdir /tmp/covdata
|
||||
|
||||
# run server in background
|
||||
GOCOVERDIR=/tmp/covdata ./versitygw -a user -s pass posix /tmp/gw &
|
||||
GW_PID=$!
|
||||
|
||||
# wait a second for server to start up
|
||||
sleep 1
|
||||
|
||||
# check if server is still running
|
||||
if ! kill -0 $GW_PID; then
|
||||
echo "server no longer running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# run tests
|
||||
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow; then
|
||||
echo "tests failed"
|
||||
kill $GW_PID
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# kill off server
|
||||
kill $GW_PID
|
||||
exit 0
|
||||
|
||||
# if the above binary was built with -cover enabled (make testbin),
|
||||
# then the following can be used for code coverage reports:
|
||||
# go tool covdata percent -i=/tmp/covdata
|
||||
# go tool covdata textfmt -i=/tmp/covdata -o profile.txt
|
||||
# go tool cover -html=profile.txt
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
)
|
||||
|
||||
type AdminController struct {
|
||||
IAMService auth.IAMService
|
||||
}
|
||||
|
||||
func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
|
||||
access, secret, role := ctx.Query("access"), ctx.Query("secret"), ctx.Query("role")
|
||||
requesterRole := ctx.Locals("role").(string)
|
||||
|
||||
if requesterRole != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
if role != "user" && role != "admin" {
|
||||
return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin'")
|
||||
}
|
||||
|
||||
user := auth.Account{Secret: secret, Role: role}
|
||||
|
||||
err := c.IAMService.CreateAccount(access, user)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create a user: %w", err)
|
||||
}
|
||||
|
||||
return ctx.SendString("The user has been created successfully")
|
||||
}
|
||||
|
||||
func (c AdminController) DeleteUser(ctx *fiber.Ctx) error {
|
||||
access := ctx.Query("access")
|
||||
requesterRole := ctx.Locals("role").(string)
|
||||
if requesterRole != "admin" {
|
||||
return fmt.Errorf("access denied: only admin users have access to this resource")
|
||||
}
|
||||
|
||||
err := c.IAMService.DeleteUserAccount(access)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ctx.SendString("The user has been deleted successfully")
|
||||
}
|
||||
@@ -1,173 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
)
|
||||
|
||||
func TestAdminController_CreateUser(t *testing.T) {
|
||||
type args struct {
|
||||
req *http.Request
|
||||
}
|
||||
|
||||
adminController := AdminController{
|
||||
IAMService: &IAMServiceMock{
|
||||
CreateAccountFunc: func(access string, account auth.Account) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("role", "admin")
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
app.Patch("/create-user", adminController.CreateUser)
|
||||
|
||||
appErr := fiber.New()
|
||||
|
||||
appErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("role", "user")
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appErr.Patch("/create-user", adminController.CreateUser)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
args args
|
||||
wantErr bool
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "Admin-create-user-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/create-user?access=test&secret=test&role=user", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Admin-create-user-invalid-user-role",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/create-user?access=test&secret=test&role=invalid", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
{
|
||||
name: "Admin-create-user-invalid-requester-role",
|
||||
app: appErr,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/create-user?access=test&secret=test&role=admin", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
resp, err := tt.app.Test(tt.args.req)
|
||||
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("AdminController.CreateUser() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
|
||||
if resp.StatusCode != tt.statusCode {
|
||||
t.Errorf("AdminController.CreateUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminController_DeleteUser(t *testing.T) {
|
||||
type args struct {
|
||||
req *http.Request
|
||||
}
|
||||
|
||||
adminController := AdminController{
|
||||
IAMService: &IAMServiceMock{
|
||||
DeleteUserAccountFunc: func(access string) error {
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
app.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("role", "admin")
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
app.Patch("/delete-user", adminController.DeleteUser)
|
||||
|
||||
appErr := fiber.New()
|
||||
|
||||
appErr.Use(func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("role", "user")
|
||||
return ctx.Next()
|
||||
})
|
||||
|
||||
appErr.Patch("/delete-user", adminController.DeleteUser)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
app *fiber.App
|
||||
args args
|
||||
wantErr bool
|
||||
statusCode int
|
||||
}{
|
||||
{
|
||||
name: "Admin-delete-user-success",
|
||||
app: app,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 200,
|
||||
},
|
||||
{
|
||||
name: "Admin-delete-user-invalid-requester-role",
|
||||
app: appErr,
|
||||
args: args{
|
||||
req: httptest.NewRequest(http.MethodPatch, "/delete-user?access=test", nil),
|
||||
},
|
||||
wantErr: false,
|
||||
statusCode: 500,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
resp, err := tt.app.Test(tt.args.req)
|
||||
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("AdminController.DeleteUser() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
|
||||
if resp.StatusCode != tt.statusCode {
|
||||
t.Errorf("AdminController.DeleteUser() statusCode = %v, wantStatusCode = %v", resp.StatusCode, tt.statusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,169 +0,0 @@
|
||||
// Code generated by moq; DO NOT EDIT.
|
||||
// github.com/matryer/moq
|
||||
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/versity/versitygw/auth"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Ensure, that IAMServiceMock does implement auth.IAMService.
|
||||
// If this is not the case, regenerate this file with moq.
|
||||
var _ auth.IAMService = &IAMServiceMock{}
|
||||
|
||||
// IAMServiceMock is a mock implementation of auth.IAMService.
|
||||
//
|
||||
// func TestSomethingThatUsesIAMService(t *testing.T) {
|
||||
//
|
||||
// // make and configure a mocked auth.IAMService
|
||||
// mockedIAMService := &IAMServiceMock{
|
||||
// CreateAccountFunc: func(access string, account auth.Account) error {
|
||||
// panic("mock out the CreateAccount method")
|
||||
// },
|
||||
// DeleteUserAccountFunc: func(access string) error {
|
||||
// panic("mock out the DeleteUserAccount method")
|
||||
// },
|
||||
// GetUserAccountFunc: func(access string) (auth.Account, error) {
|
||||
// panic("mock out the GetUserAccount method")
|
||||
// },
|
||||
// }
|
||||
//
|
||||
// // use mockedIAMService in code that requires auth.IAMService
|
||||
// // and then make assertions.
|
||||
//
|
||||
// }
|
||||
type IAMServiceMock struct {
|
||||
// CreateAccountFunc mocks the CreateAccount method.
|
||||
CreateAccountFunc func(access string, account auth.Account) error
|
||||
|
||||
// DeleteUserAccountFunc mocks the DeleteUserAccount method.
|
||||
DeleteUserAccountFunc func(access string) error
|
||||
|
||||
// GetUserAccountFunc mocks the GetUserAccount method.
|
||||
GetUserAccountFunc func(access string) (auth.Account, error)
|
||||
|
||||
// calls tracks calls to the methods.
|
||||
calls struct {
|
||||
// CreateAccount holds details about calls to the CreateAccount method.
|
||||
CreateAccount []struct {
|
||||
// Access is the access argument value.
|
||||
Access string
|
||||
// Account is the account argument value.
|
||||
Account auth.Account
|
||||
}
|
||||
// DeleteUserAccount holds details about calls to the DeleteUserAccount method.
|
||||
DeleteUserAccount []struct {
|
||||
// Access is the access argument value.
|
||||
Access string
|
||||
}
|
||||
// GetUserAccount holds details about calls to the GetUserAccount method.
|
||||
GetUserAccount []struct {
|
||||
// Access is the access argument value.
|
||||
Access string
|
||||
}
|
||||
}
|
||||
lockCreateAccount sync.RWMutex
|
||||
lockDeleteUserAccount sync.RWMutex
|
||||
lockGetUserAccount sync.RWMutex
|
||||
}
|
||||
|
||||
// CreateAccount calls CreateAccountFunc.
|
||||
func (mock *IAMServiceMock) CreateAccount(access string, account auth.Account) error {
|
||||
if mock.CreateAccountFunc == nil {
|
||||
panic("IAMServiceMock.CreateAccountFunc: method is nil but IAMService.CreateAccount was just called")
|
||||
}
|
||||
callInfo := struct {
|
||||
Access string
|
||||
Account auth.Account
|
||||
}{
|
||||
Access: access,
|
||||
Account: account,
|
||||
}
|
||||
mock.lockCreateAccount.Lock()
|
||||
mock.calls.CreateAccount = append(mock.calls.CreateAccount, callInfo)
|
||||
mock.lockCreateAccount.Unlock()
|
||||
return mock.CreateAccountFunc(access, account)
|
||||
}
|
||||
|
||||
// CreateAccountCalls gets all the calls that were made to CreateAccount.
|
||||
// Check the length with:
|
||||
//
|
||||
// len(mockedIAMService.CreateAccountCalls())
|
||||
func (mock *IAMServiceMock) CreateAccountCalls() []struct {
|
||||
Access string
|
||||
Account auth.Account
|
||||
} {
|
||||
var calls []struct {
|
||||
Access string
|
||||
Account auth.Account
|
||||
}
|
||||
mock.lockCreateAccount.RLock()
|
||||
calls = mock.calls.CreateAccount
|
||||
mock.lockCreateAccount.RUnlock()
|
||||
return calls
|
||||
}
|
||||
|
||||
// DeleteUserAccount calls DeleteUserAccountFunc.
|
||||
func (mock *IAMServiceMock) DeleteUserAccount(access string) error {
|
||||
if mock.DeleteUserAccountFunc == nil {
|
||||
panic("IAMServiceMock.DeleteUserAccountFunc: method is nil but IAMService.DeleteUserAccount was just called")
|
||||
}
|
||||
callInfo := struct {
|
||||
Access string
|
||||
}{
|
||||
Access: access,
|
||||
}
|
||||
mock.lockDeleteUserAccount.Lock()
|
||||
mock.calls.DeleteUserAccount = append(mock.calls.DeleteUserAccount, callInfo)
|
||||
mock.lockDeleteUserAccount.Unlock()
|
||||
return mock.DeleteUserAccountFunc(access)
|
||||
}
|
||||
|
||||
// DeleteUserAccountCalls gets all the calls that were made to DeleteUserAccount.
|
||||
// Check the length with:
|
||||
//
|
||||
// len(mockedIAMService.DeleteUserAccountCalls())
|
||||
func (mock *IAMServiceMock) DeleteUserAccountCalls() []struct {
|
||||
Access string
|
||||
} {
|
||||
var calls []struct {
|
||||
Access string
|
||||
}
|
||||
mock.lockDeleteUserAccount.RLock()
|
||||
calls = mock.calls.DeleteUserAccount
|
||||
mock.lockDeleteUserAccount.RUnlock()
|
||||
return calls
|
||||
}
|
||||
|
||||
// GetUserAccount calls GetUserAccountFunc.
|
||||
func (mock *IAMServiceMock) GetUserAccount(access string) (auth.Account, error) {
|
||||
if mock.GetUserAccountFunc == nil {
|
||||
panic("IAMServiceMock.GetUserAccountFunc: method is nil but IAMService.GetUserAccount was just called")
|
||||
}
|
||||
callInfo := struct {
|
||||
Access string
|
||||
}{
|
||||
Access: access,
|
||||
}
|
||||
mock.lockGetUserAccount.Lock()
|
||||
mock.calls.GetUserAccount = append(mock.calls.GetUserAccount, callInfo)
|
||||
mock.lockGetUserAccount.Unlock()
|
||||
return mock.GetUserAccountFunc(access)
|
||||
}
|
||||
|
||||
// GetUserAccountCalls gets all the calls that were made to GetUserAccount.
|
||||
// Check the length with:
|
||||
//
|
||||
// len(mockedIAMService.GetUserAccountCalls())
|
||||
func (mock *IAMServiceMock) GetUserAccountCalls() []struct {
|
||||
Access string
|
||||
} {
|
||||
var calls []struct {
|
||||
Access string
|
||||
}
|
||||
mock.lockGetUserAccount.RLock()
|
||||
calls = mock.calls.GetUserAccount
|
||||
mock.lockGetUserAccount.RUnlock()
|
||||
return calls
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package middlewares
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
isRoot, access := ctx.Locals("isRoot").(bool), ctx.Locals("access").(string)
|
||||
path := ctx.Path()
|
||||
pathParts := strings.Split(path, "/")
|
||||
bucket := pathParts[1]
|
||||
if path == "/" && ctx.Method() == http.MethodGet {
|
||||
return ctx.Next()
|
||||
}
|
||||
if ctx.Method() == http.MethodPatch {
|
||||
return ctx.Next()
|
||||
}
|
||||
if len(pathParts) == 2 && pathParts[1] != "" && ctx.Method() == http.MethodPut && !ctx.Request().URI().QueryArgs().Has("acl") {
|
||||
if err := auth.IsAdmin(access, isRoot); err != nil {
|
||||
return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
|
||||
}
|
||||
return ctx.Next()
|
||||
}
|
||||
//TODO: provide correct action names for the logger, after implementing DetectAction middleware
|
||||
data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
|
||||
if err != nil {
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
parsedAcl, err := auth.ParseACL(data)
|
||||
if err != nil {
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
ctx.Locals("parsedAcl", parsedAcl)
|
||||
return ctx.Next()
|
||||
}
|
||||
}
|
||||
@@ -1,187 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package middlewares
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/smithy-go/logging"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
const (
|
||||
iso8601Format = "20060102T150405Z"
|
||||
)
|
||||
|
||||
type RootUserConfig struct {
|
||||
Access string
|
||||
Secret string
|
||||
}
|
||||
|
||||
func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
|
||||
acct := accounts{root: root, iam: iam}
|
||||
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("region", region)
|
||||
ctx.Locals("startTime", time.Now())
|
||||
authorization := ctx.Get("Authorization")
|
||||
if authorization == "" {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
// Check the signature version
|
||||
authParts := strings.Split(authorization, ",")
|
||||
for i, el := range authParts {
|
||||
authParts[i] = strings.TrimSpace(el)
|
||||
}
|
||||
|
||||
if len(authParts) != 3 {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
startParts := strings.Split(authParts[0], " ")
|
||||
|
||||
if startParts[0] != "AWS4-HMAC-SHA256" {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
credKv := strings.Split(startParts[1], "=")
|
||||
if len(credKv) != 2 {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
creds := strings.Split(credKv[1], "/")
|
||||
if len(creds) < 4 {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
ctx.Locals("access", creds[0])
|
||||
ctx.Locals("isRoot", creds[0] == root.Access)
|
||||
|
||||
signHdrKv := strings.Split(authParts[1], "=")
|
||||
if len(signHdrKv) != 2 {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
signedHdrs := strings.Split(signHdrKv[1], ";")
|
||||
|
||||
account, err := acct.getAccount(creds[0])
|
||||
if err == auth.ErrNoSuchUser {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
if err != nil {
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
ctx.Locals("role", account.Role)
|
||||
|
||||
// Check X-Amz-Date header
|
||||
date := ctx.Get("X-Amz-Date")
|
||||
if date == "" {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
// Parse the date and check the date validity
|
||||
tdate, err := time.Parse(iso8601Format, date)
|
||||
if err != nil {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
hashPayloadHeader := ctx.Get("X-Amz-Content-Sha256")
|
||||
ok := isSpecialPayload(hashPayloadHeader)
|
||||
|
||||
if !ok {
|
||||
// Calculate the hash of the request payload
|
||||
hashedPayload := sha256.Sum256(ctx.Body())
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
// Compare the calculated hash with the hash provided
|
||||
if hashPayloadHeader != hexPayload {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new http request instance from fasthttp request
|
||||
req, err := utils.CreateHttpRequestFromCtx(ctx, signedHdrs)
|
||||
if err != nil {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{
|
||||
AccessKeyID: creds[0],
|
||||
SecretAccessKey: account.Secret,
|
||||
}, req, hashPayloadHeader, creds[3], region, tdate, func(options *v4.SignerOptions) {
|
||||
if debug {
|
||||
options.LogSigning = true
|
||||
options.Logger = logging.NewStandardLogger(os.Stderr)
|
||||
}
|
||||
})
|
||||
if signErr != nil {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
parts := strings.Split(req.Header.Get("Authorization"), " ")
|
||||
if len(parts) < 4 {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
calculatedSign := strings.Split(parts[3], "=")[1]
|
||||
expectedSign := strings.Split(authParts[2], "=")[1]
|
||||
|
||||
if expectedSign != calculatedSign {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
}
|
||||
}
|
||||
|
||||
type accounts struct {
|
||||
root RootUserConfig
|
||||
iam auth.IAMService
|
||||
}
|
||||
|
||||
func (a accounts) getAccount(access string) (auth.Account, error) {
|
||||
if access == a.root.Access {
|
||||
return auth.Account{
|
||||
Secret: a.root.Secret,
|
||||
Role: "admin",
|
||||
}, nil
|
||||
}
|
||||
|
||||
return a.iam.GetUserAccount(access)
|
||||
}
|
||||
|
||||
func isSpecialPayload(str string) bool {
|
||||
specialValues := map[string]bool{
|
||||
"UNSIGNED-PAYLOAD": true,
|
||||
"STREAMING-UNSIGNED-PAYLOAD-TRAILER": true,
|
||||
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD": true,
|
||||
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER": true,
|
||||
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD": true,
|
||||
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
|
||||
}
|
||||
|
||||
return specialValues[str]
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package middlewares
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
)
|
||||
|
||||
func RequestLogger(isDebug bool) fiber.Handler {
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
ctx.Locals("isDebug", isDebug)
|
||||
if isDebug {
|
||||
log.Println("Request headers: ")
|
||||
ctx.Request().Header.VisitAll(func(key, val []byte) {
|
||||
log.Printf("%s: %s", key, val)
|
||||
})
|
||||
|
||||
if ctx.Request().URI().QueryArgs().Len() != 0 {
|
||||
fmt.Println()
|
||||
log.Println("Request query arguments: ")
|
||||
ctx.Request().URI().QueryArgs().VisitAll(func(key, val []byte) {
|
||||
log.Printf("%s: %s", key, val)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package middlewares
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
|
||||
func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
|
||||
return func(ctx *fiber.Ctx) error {
|
||||
incomingSum := ctx.Get("Content-Md5")
|
||||
if incomingSum == "" {
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
sum := md5.Sum(ctx.Body())
|
||||
calculatedSum := base64.StdEncoding.EncodeToString(sum[:])
|
||||
|
||||
if incomingSum != calculatedSum {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
}
|
||||
}
|
||||
@@ -1,38 +1,15 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
"github.com/versity/scoutgw/backend"
|
||||
"github.com/versity/scoutgw/s3api/controllers"
|
||||
)
|
||||
|
||||
type S3ApiRouter struct{}
|
||||
|
||||
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender) {
|
||||
s3ApiController := controllers.New(be, iam, logger, evs)
|
||||
adminController := controllers.AdminController{IAMService: iam}
|
||||
|
||||
app.Patch("/create-user", adminController.CreateUser)
|
||||
|
||||
// Admin Delete api
|
||||
app.Patch("/delete-user", adminController.DeleteUser)
|
||||
func (sa *S3ApiRouter) Init(app *fiber.App, be backend.Backend) {
|
||||
s3ApiController := controllers.New(be)
|
||||
// ListBuckets action
|
||||
app.Get("/", s3ApiController.ListBuckets)
|
||||
|
||||
|
||||
@@ -1,32 +1,16 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/scoutgw/backend"
|
||||
)
|
||||
|
||||
func TestS3ApiRouter_Init(t *testing.T) {
|
||||
type args struct {
|
||||
app *fiber.App
|
||||
be backend.Backend
|
||||
iam auth.IAMService
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -39,13 +23,12 @@ func TestS3ApiRouter_Init(t *testing.T) {
|
||||
args: args{
|
||||
app: fiber.New(),
|
||||
be: backend.BackendUnsupported{},
|
||||
iam: &auth.IAMServiceInternal{},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tt.sa.Init(tt.args.app, tt.args.be, tt.args.iam, nil, nil)
|
||||
tt.sa.Init(tt.args.app, tt.args.be)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,29 +1,9 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/versitygw/s3event"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
"github.com/versity/scoutgw/backend"
|
||||
)
|
||||
|
||||
type S3ApiServer struct {
|
||||
@@ -31,52 +11,16 @@ type S3ApiServer struct {
|
||||
backend backend.Backend
|
||||
router *S3ApiRouter
|
||||
port string
|
||||
cert *tls.Certificate
|
||||
debug bool
|
||||
}
|
||||
|
||||
func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, port, region string, iam auth.IAMService, l s3log.AuditLogger, evs s3event.S3EventSender, opts ...Option) (*S3ApiServer, error) {
|
||||
server := &S3ApiServer{
|
||||
app: app,
|
||||
backend: be,
|
||||
router: new(S3ApiRouter),
|
||||
port: port,
|
||||
}
|
||||
func New(app *fiber.App, be backend.Backend, port string) (s3ApiServer *S3ApiServer, err error) {
|
||||
s3ApiServer = &S3ApiServer{app, be, new(S3ApiRouter), port}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(server)
|
||||
}
|
||||
|
||||
// Logging middlewares
|
||||
app.Use(logger.New())
|
||||
app.Use(middlewares.RequestLogger(server.debug))
|
||||
|
||||
// Authentication middlewares
|
||||
app.Use(middlewares.VerifyV4Signature(root, iam, l, region, server.debug))
|
||||
app.Use(middlewares.VerifyMD5Body(l))
|
||||
app.Use(middlewares.AclParser(be, l))
|
||||
|
||||
server.router.Init(app, be, iam, l, evs)
|
||||
|
||||
return server, nil
|
||||
}
|
||||
|
||||
// Option sets various options for New()
|
||||
type Option func(*S3ApiServer)
|
||||
|
||||
// WithTLS sets TLS Credentials
|
||||
func WithTLS(cert tls.Certificate) Option {
|
||||
return func(s *S3ApiServer) { s.cert = &cert }
|
||||
}
|
||||
|
||||
// WithDebug sets debug output
|
||||
func WithDebug() Option {
|
||||
return func(s *S3ApiServer) { s.debug = true }
|
||||
s3ApiServer.router.Init(app, be)
|
||||
return
|
||||
}
|
||||
|
||||
func (sa *S3ApiServer) Serve() (err error) {
|
||||
if sa.cert != nil {
|
||||
return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
|
||||
}
|
||||
return sa.app.Listen(sa.port)
|
||||
}
|
||||
|
||||
@@ -1,28 +1,11 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3api/middlewares"
|
||||
"github.com/versity/scoutgw/backend"
|
||||
)
|
||||
|
||||
func TestNew(t *testing.T) {
|
||||
@@ -30,7 +13,6 @@ func TestNew(t *testing.T) {
|
||||
app *fiber.App
|
||||
be backend.Backend
|
||||
port string
|
||||
root middlewares.RootUserConfig
|
||||
}
|
||||
|
||||
app := fiber.New()
|
||||
@@ -50,7 +32,6 @@ func TestNew(t *testing.T) {
|
||||
app: app,
|
||||
be: be,
|
||||
port: port,
|
||||
root: middlewares.RootUserConfig{},
|
||||
},
|
||||
wantS3ApiServer: &S3ApiServer{
|
||||
app: app,
|
||||
@@ -63,8 +44,7 @@ func TestNew(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.root,
|
||||
tt.args.port, "us-east-1", &auth.IAMServiceInternal{}, nil, nil)
|
||||
gotS3ApiServer, err := New(tt.args.app, tt.args.be, tt.args.port)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -83,26 +63,15 @@ func TestS3ApiServer_Serve(t *testing.T) {
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Serve-invalid-address",
|
||||
name: "Return error when serving S3 api server with invalid address",
|
||||
wantErr: true,
|
||||
sa: &S3ApiServer{
|
||||
app: fiber.New(),
|
||||
backend: backend.BackendUnsupported{},
|
||||
port: "Invalid address",
|
||||
port: "Wrong address",
|
||||
router: &S3ApiRouter{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Serve-invalid-address-with-certificate",
|
||||
wantErr: true,
|
||||
sa: &S3ApiServer{
|
||||
app: fiber.New(),
|
||||
backend: backend.BackendUnsupported{},
|
||||
port: "Invalid address",
|
||||
router: &S3ApiRouter{},
|
||||
cert: &tls.Certificate{},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
)
|
||||
|
||||
func LogCtxDetails(ctx *fiber.Ctx, respBody []byte) {
|
||||
isDebug, ok := ctx.Locals("isDebug").(bool)
|
||||
_, notLogReqBody := ctx.Locals("logReqBody").(bool)
|
||||
_, notLogResBody := ctx.Locals("logResBody").(bool)
|
||||
if isDebug && ok {
|
||||
// Log request body
|
||||
if !notLogReqBody {
|
||||
fmt.Println()
|
||||
log.Printf("Request Body: %s", ctx.Request().Body())
|
||||
}
|
||||
|
||||
// Log path parameters
|
||||
fmt.Println()
|
||||
log.Println("Path parameters: ")
|
||||
for key, val := range ctx.AllParams() {
|
||||
log.Printf("%s: %s", key, val)
|
||||
}
|
||||
|
||||
// Log response headers
|
||||
fmt.Println()
|
||||
log.Println("Response Headers: ")
|
||||
ctx.Response().Header.VisitAll(func(key, val []byte) {
|
||||
log.Printf("%s: %s", key, val)
|
||||
})
|
||||
|
||||
// Log response body
|
||||
if !notLogResBody && len(respBody) > 0 {
|
||||
fmt.Println()
|
||||
log.Printf("Response body %s", ctx.Response().Body())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
|
||||
var (
|
||||
bucketNameRegexp = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]+[a-z0-9]$`)
|
||||
bucketNameIpRegexp = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
|
||||
)
|
||||
|
||||
func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]string) {
|
||||
metadata = make(map[string]string)
|
||||
headers.VisitAll(func(key, value []byte) {
|
||||
if strings.HasPrefix(string(key), "X-Amz-Meta-") {
|
||||
trimmedKey := strings.TrimPrefix(string(key), "X-Amz-Meta-")
|
||||
headerValue := string(value)
|
||||
metadata[trimmedKey] = headerValue
|
||||
}
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Request, error) {
|
||||
req := ctx.Request()
|
||||
|
||||
httpReq, err := http.NewRequest(string(req.Header.Method()), req.URI().String(), bytes.NewReader(req.Body()))
|
||||
if err != nil {
|
||||
return nil, errors.New("error in creating an http request")
|
||||
}
|
||||
|
||||
// Set the request headers
|
||||
req.Header.VisitAll(func(key, value []byte) {
|
||||
keyStr := string(key)
|
||||
if includeHeader(keyStr, signedHdrs) {
|
||||
httpReq.Header.Add(keyStr, string(value))
|
||||
}
|
||||
})
|
||||
|
||||
// Check if Content-Length in signed headers
|
||||
// If content length is non 0, then the header will be included
|
||||
if !includeHeader("Content-Length", signedHdrs) {
|
||||
httpReq.ContentLength = 0
|
||||
}
|
||||
|
||||
// Set the Host header
|
||||
httpReq.Host = string(req.Header.Host())
|
||||
|
||||
return httpReq, nil
|
||||
}
|
||||
|
||||
func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) {
|
||||
for key, val := range meta {
|
||||
ctx.Set(fmt.Sprintf("X-Amz-Meta-%s", key), val)
|
||||
}
|
||||
}
|
||||
|
||||
type CustomHeader struct {
|
||||
Key string
|
||||
Value string
|
||||
}
|
||||
|
||||
func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) {
|
||||
for _, header := range headers {
|
||||
ctx.Set(header.Key, header.Value)
|
||||
}
|
||||
}
|
||||
|
||||
func IsValidBucketName(bucket string) bool {
|
||||
if len(bucket) < 3 || len(bucket) > 63 {
|
||||
return false
|
||||
}
|
||||
// Checks to contain only digits, lowercase letters, dot, hyphen.
|
||||
// Checks to start and end with only digits and lowercase letters.
|
||||
if !bucketNameRegexp.MatchString(bucket) {
|
||||
return false
|
||||
}
|
||||
// Checks not to be a valid IP address
|
||||
if bucketNameIpRegexp.MatchString(bucket) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func includeHeader(hdr string, signedHdrs []string) bool {
|
||||
for _, shdr := range signedHdrs {
|
||||
if strings.EqualFold(hdr, shdr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
|
||||
func TestCreateHttpRequestFromCtx(t *testing.T) {
|
||||
type args struct {
|
||||
ctx *fiber.Ctx
|
||||
}
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
// Expected output, Case 1
|
||||
ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
|
||||
req := ctx.Request()
|
||||
request, _ := http.NewRequest(string(req.Header.Method()), req.URI().String(), bytes.NewReader(req.Body()))
|
||||
|
||||
// Case 2
|
||||
ctx2 := app.AcquireCtx(&fasthttp.RequestCtx{})
|
||||
req2 := ctx2.Request()
|
||||
req2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
|
||||
|
||||
request2, _ := http.NewRequest(string(req2.Header.Method()), req2.URI().String(), bytes.NewReader(req2.Body()))
|
||||
request2.Header.Add("X-Amz-Mfa", "Some valid Mfa")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want *http.Request
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Success-response",
|
||||
args: args{
|
||||
ctx: ctx,
|
||||
},
|
||||
want: request,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "Success-response-With-Headers",
|
||||
args: args{
|
||||
ctx: ctx2,
|
||||
},
|
||||
want: request2,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := CreateHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"})
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(got.Header, tt.want.Header) {
|
||||
t.Errorf("CreateHttpRequestFromCtx() got = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetUserMetaData(t *testing.T) {
|
||||
type args struct {
|
||||
headers *fasthttp.RequestHeader
|
||||
}
|
||||
|
||||
app := fiber.New()
|
||||
|
||||
// Case 1
|
||||
ctx := app.AcquireCtx(&fasthttp.RequestCtx{})
|
||||
req := ctx.Request()
|
||||
|
||||
// Case 2
|
||||
ctx2 := app.AcquireCtx(&fasthttp.RequestCtx{})
|
||||
req2 := ctx2.Request()
|
||||
|
||||
req2.Header.Add("X-Amz-Meta-Name", "Nick")
|
||||
req2.Header.Add("X-Amz-Meta-Age", "27")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
wantMetadata map[string]string
|
||||
}{
|
||||
{
|
||||
name: "Success-empty-response",
|
||||
args: args{
|
||||
headers: &req.Header,
|
||||
},
|
||||
wantMetadata: map[string]string{},
|
||||
},
|
||||
{
|
||||
name: "Success-non-empty-response",
|
||||
args: args{
|
||||
headers: &req2.Header,
|
||||
},
|
||||
wantMetadata: map[string]string{
|
||||
"Age": "27",
|
||||
"Name": "Nick",
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if gotMetadata := GetUserMetaData(tt.args.headers); !reflect.DeepEqual(gotMetadata, tt.wantMetadata) {
|
||||
t.Errorf("GetUserMetaData() = %v, want %v", gotMetadata, tt.wantMetadata)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_includeHeader(t *testing.T) {
|
||||
type args struct {
|
||||
hdr string
|
||||
signedHdrs []string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "include-header-falsy-case",
|
||||
args: args{
|
||||
hdr: "Content-Type",
|
||||
signedHdrs: []string{"X-Amz-Acl", "Content-Encoding"},
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "include-header-falsy-case",
|
||||
args: args{
|
||||
hdr: "Content-Type",
|
||||
signedHdrs: []string{"X-Amz-Acl", "Content-Type"},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := includeHeader(tt.args.hdr, tt.args.signedHdrs); got != tt.want {
|
||||
t.Errorf("includeHeader() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsValidBucketName(t *testing.T) {
|
||||
type args struct {
|
||||
bucket string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "IsValidBucketName-short-name",
|
||||
args: args{
|
||||
bucket: "a",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "IsValidBucketName-start-with-hyphen",
|
||||
args: args{
|
||||
bucket: "-bucket",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "IsValidBucketName-start-with-dot",
|
||||
args: args{
|
||||
bucket: ".bucket",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "IsValidBucketName-contain-invalid-character",
|
||||
args: args{
|
||||
bucket: "my@bucket",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "IsValidBucketName-end-with-hyphen",
|
||||
args: args{
|
||||
bucket: "bucket-",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "IsValidBucketName-end-with-dot",
|
||||
args: args{
|
||||
bucket: "bucket.",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "IsValidBucketName-valid-bucket-name",
|
||||
args: args{
|
||||
bucket: "my-bucket",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := IsValidBucketName(tt.args.bucket); got != tt.want {
|
||||
t.Errorf("IsValidBucketName() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,17 +1,3 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3err
|
||||
|
||||
import (
|
||||
@@ -105,10 +91,7 @@ const (
|
||||
ErrAuthNotSetup
|
||||
ErrNotImplemented
|
||||
ErrPreconditionFailed
|
||||
ErrInvalidObjectState
|
||||
ErrInvalidRange
|
||||
|
||||
// Non-AWS errors
|
||||
ErrExistingObjectIsDirectory
|
||||
ErrObjectParentIsFile
|
||||
)
|
||||
@@ -371,16 +354,6 @@ var errorCodeResponse = map[ErrorCode]APIError{
|
||||
Description: "At least one of the pre-conditions you specified did not hold",
|
||||
HTTPStatusCode: http.StatusPreconditionFailed,
|
||||
},
|
||||
ErrInvalidObjectState: {
|
||||
Code: "InvalidObjectState",
|
||||
Description: "The operation is not valid for the current state of the object",
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
},
|
||||
ErrInvalidRange: {
|
||||
Code: "InvalidRange",
|
||||
Description: "The requested range is not valid for the request. Try another range.",
|
||||
HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
|
||||
},
|
||||
ErrExistingObjectIsDirectory: {
|
||||
Code: "ExistingObjectIsDirectory",
|
||||
Description: "Existing Object is a directory.",
|
||||
|
||||
130
s3event/event.go
130
s3event/event.go
@@ -1,130 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3event
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
)
|
||||
|
||||
type S3EventSender interface {
|
||||
SendEvent(ctx *fiber.Ctx, meta EventMeta)
|
||||
}
|
||||
|
||||
type EventMeta struct {
|
||||
BucketOwner string
|
||||
EventName EventType
|
||||
ObjectSize int64
|
||||
ObjectETag *string
|
||||
VersionId *string
|
||||
}
|
||||
|
||||
type EventFields struct {
|
||||
Records []EventSchema
|
||||
}
|
||||
|
||||
type EventType string
|
||||
|
||||
const (
|
||||
EventObjectPut EventType = "s3:ObjectCreated:Put"
|
||||
EventObjectCopy EventType = "s3:ObjectCreated:Copy"
|
||||
EventCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
|
||||
EventObjectDelete EventType = "s3:ObjectRemoved:Delete"
|
||||
EventObjectRestoreCompleted EventType = "s3:ObjectRestore:Completed"
|
||||
EventObjectTaggingPut EventType = "s3:ObjectTagging:Put"
|
||||
EventObjectTaggingDelete EventType = "s3:ObjectTagging:Delete"
|
||||
EventObjectAclPut EventType = "s3:ObjectAcl:Put"
|
||||
// Not supported
|
||||
// EventObjectRestorePost EventType = "s3:ObjectRestore:Post"
|
||||
// EventObjectRestoreDelete EventType = "s3:ObjectRestore:Delete"
|
||||
)
|
||||
|
||||
type EventSchema struct {
|
||||
EventVersion string `json:"eventVersion"`
|
||||
EventSource string `json:"eventSource"`
|
||||
AwsRegion string `json:"awsRegion"`
|
||||
EventTime string `json:"eventTime"`
|
||||
EventName EventType `json:"eventName"`
|
||||
UserIdentity EventUserIdentity `json:"userIdentity"`
|
||||
RequestParameters EventRequestParams `json:"requestParameters"`
|
||||
ResponseElements EventResponseElements `json:"responseElements"`
|
||||
S3 EventS3Data `json:"s3"`
|
||||
GlacierEventData EventGlacierData `json:"glacierEventData"`
|
||||
}
|
||||
|
||||
type EventUserIdentity struct {
|
||||
PrincipalId string `json:"PrincipalId"`
|
||||
}
|
||||
|
||||
type EventRequestParams struct {
|
||||
SourceIPAddress string `json:"sourceIPAddress"`
|
||||
}
|
||||
|
||||
type EventResponseElements struct {
|
||||
RequestId string `json:"x-amz-request-id"`
|
||||
HostId string `json:"x-amz-id-2"`
|
||||
}
|
||||
|
||||
type EventS3Data struct {
|
||||
S3SchemaVersion string `json:"s3SchemaVersion"`
|
||||
ConfigurationId string `json:"configurationId"`
|
||||
Bucket EventS3BucketData `json:"bucket"`
|
||||
Object EventObjectData `json:"object"`
|
||||
}
|
||||
|
||||
type EventGlacierData struct {
|
||||
RestoreEventData EventRestoreData `json:"restoreEventData"`
|
||||
}
|
||||
|
||||
type EventRestoreData struct {
|
||||
LifecycleRestorationExpiryTime string `json:"lifecycleRestorationExpiryTime"`
|
||||
LifecycleRestoreStorageClass string `json:"lifecycleRestoreStorageClass"`
|
||||
}
|
||||
|
||||
type EventS3BucketData struct {
|
||||
Name string `json:"name"`
|
||||
OwnerIdentity EventUserIdentity `json:"ownerIdentity"`
|
||||
Arn string `json:"arn"`
|
||||
}
|
||||
|
||||
type EventObjectData struct {
|
||||
Key string `json:"key"`
|
||||
Size int64 `json:"size"`
|
||||
ETag *string `json:"eTag"`
|
||||
VersionId *string `json:"versionId"`
|
||||
Sequencer string `json:"sequencer"`
|
||||
}
|
||||
|
||||
type EventConfig struct {
|
||||
KafkaURL string
|
||||
KafkaTopic string
|
||||
KafkaTopicKey string
|
||||
NatsURL string
|
||||
NatsTopic string
|
||||
}
|
||||
|
||||
func InitEventSender(cfg *EventConfig) (S3EventSender, error) {
|
||||
if cfg.KafkaURL != "" && cfg.NatsURL != "" {
|
||||
return nil, fmt.Errorf("there should be specified one of the following: kafka, nats")
|
||||
}
|
||||
if cfg.NatsURL != "" {
|
||||
return InitNatsEventService(cfg.NatsURL, cfg.NatsTopic)
|
||||
}
|
||||
if cfg.KafkaURL != "" {
|
||||
return InitKafkaEventService(cfg.KafkaURL, cfg.KafkaTopic, cfg.KafkaTopicKey)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
153
s3event/kafka.go
153
s3event/kafka.go
@@ -1,153 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3event
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/segmentio/kafka-go"
|
||||
)
|
||||
|
||||
var sequencer = 0
|
||||
|
||||
type Kafka struct {
|
||||
key string
|
||||
writer *kafka.Writer
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func InitKafkaEventService(url, topic, key string) (S3EventSender, error) {
|
||||
if topic == "" {
|
||||
return nil, fmt.Errorf("kafka message topic should be specified")
|
||||
}
|
||||
|
||||
w := kafka.NewWriter(kafka.WriterConfig{
|
||||
Brokers: []string{url},
|
||||
Topic: topic,
|
||||
Balancer: &kafka.LeastBytes{},
|
||||
BatchTimeout: 5 * time.Millisecond,
|
||||
})
|
||||
|
||||
msg := map[string]string{
|
||||
"Service": "S3",
|
||||
"Event": "s3:TestEvent",
|
||||
"Time": time.Now().Format(time.RFC3339),
|
||||
"Bucket": "Test-Bucket",
|
||||
}
|
||||
|
||||
msgJSON, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
message := kafka.Message{
|
||||
Key: []byte(key),
|
||||
Value: msgJSON,
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
err = w.WriteMessages(ctx, message)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Kafka{
|
||||
key: key,
|
||||
writer: w,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ks *Kafka) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
|
||||
ks.mu.Lock()
|
||||
defer ks.mu.Unlock()
|
||||
|
||||
path := strings.Split(ctx.Path(), "/")
|
||||
bucket, object := path[1], strings.Join(path[2:], "/")
|
||||
|
||||
schema := EventSchema{
|
||||
EventVersion: "2.2",
|
||||
EventSource: "aws:s3",
|
||||
AwsRegion: ctx.Locals("region").(string),
|
||||
EventTime: time.Now().Format(time.RFC3339),
|
||||
EventName: meta.EventName,
|
||||
UserIdentity: EventUserIdentity{
|
||||
PrincipalId: ctx.Locals("access").(string),
|
||||
},
|
||||
RequestParameters: EventRequestParams{
|
||||
SourceIPAddress: ctx.IP(),
|
||||
},
|
||||
ResponseElements: EventResponseElements{
|
||||
RequestId: ctx.Get("X-Amz-Request-Id"),
|
||||
HostId: ctx.Get("X-Amx-Id-2"),
|
||||
},
|
||||
S3: EventS3Data{
|
||||
S3SchemaVersion: "1.0",
|
||||
// This field will come up after implementing per bucket notifications
|
||||
ConfigurationId: "kafka-global",
|
||||
Bucket: EventS3BucketData{
|
||||
Name: bucket,
|
||||
OwnerIdentity: EventUserIdentity{
|
||||
PrincipalId: ctx.Locals("access").(string),
|
||||
},
|
||||
Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
|
||||
},
|
||||
Object: EventObjectData{
|
||||
Key: object,
|
||||
Size: meta.ObjectSize,
|
||||
ETag: meta.ObjectETag,
|
||||
VersionId: meta.VersionId,
|
||||
Sequencer: genSequencer(),
|
||||
},
|
||||
},
|
||||
GlacierEventData: EventGlacierData{
|
||||
// Not supported
|
||||
RestoreEventData: EventRestoreData{},
|
||||
},
|
||||
}
|
||||
|
||||
ks.send([]EventSchema{schema})
|
||||
}
|
||||
|
||||
func (ks *Kafka) send(evnt []EventSchema) {
|
||||
msg, err := json.Marshal(evnt)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
message := kafka.Message{
|
||||
Key: []byte(ks.key),
|
||||
Value: msg,
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
err = ks.writer.WriteMessages(ctx, message)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to send kafka event: %v\n", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func genSequencer() string {
|
||||
sequencer = sequencer + 1
|
||||
return fmt.Sprintf("%X", sequencer)
|
||||
}
|
||||
112
s3event/nats.go
112
s3event/nats.go
@@ -1,112 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3event
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
type NatsEventSender struct {
|
||||
topic string
|
||||
client *nats.Conn
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func InitNatsEventService(url, topic string) (S3EventSender, error) {
|
||||
if topic == "" {
|
||||
return nil, fmt.Errorf("nats message topic should be specified")
|
||||
}
|
||||
|
||||
client, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &NatsEventSender{
|
||||
topic: topic,
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ns *NatsEventSender) SendEvent(ctx *fiber.Ctx, meta EventMeta) {
|
||||
ns.mu.Lock()
|
||||
defer ns.mu.Unlock()
|
||||
|
||||
path := strings.Split(ctx.Path(), "/")
|
||||
bucket, object := path[1], strings.Join(path[2:], "/")
|
||||
|
||||
schema := EventSchema{
|
||||
EventVersion: "2.2",
|
||||
EventSource: "aws:s3",
|
||||
AwsRegion: ctx.Locals("region").(string),
|
||||
EventTime: time.Now().Format(time.RFC3339),
|
||||
EventName: meta.EventName,
|
||||
UserIdentity: EventUserIdentity{
|
||||
PrincipalId: ctx.Locals("access").(string),
|
||||
},
|
||||
RequestParameters: EventRequestParams{
|
||||
SourceIPAddress: ctx.IP(),
|
||||
},
|
||||
ResponseElements: EventResponseElements{
|
||||
RequestId: ctx.Get("X-Amz-Request-Id"),
|
||||
HostId: ctx.Get("X-Amx-Id-2"),
|
||||
},
|
||||
S3: EventS3Data{
|
||||
S3SchemaVersion: "1.0",
|
||||
// This field will come up after implementing per bucket notifications
|
||||
ConfigurationId: "nats-global",
|
||||
Bucket: EventS3BucketData{
|
||||
Name: bucket,
|
||||
OwnerIdentity: EventUserIdentity{
|
||||
PrincipalId: ctx.Locals("access").(string),
|
||||
},
|
||||
Arn: fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/")),
|
||||
},
|
||||
Object: EventObjectData{
|
||||
Key: object,
|
||||
Size: meta.ObjectSize,
|
||||
ETag: meta.ObjectETag,
|
||||
VersionId: meta.VersionId,
|
||||
Sequencer: genSequencer(),
|
||||
},
|
||||
},
|
||||
GlacierEventData: EventGlacierData{
|
||||
// Not supported
|
||||
RestoreEventData: EventRestoreData{},
|
||||
},
|
||||
}
|
||||
|
||||
ns.send([]EventSchema{schema})
|
||||
}
|
||||
|
||||
func (ns *NatsEventSender) send(evnt []EventSchema) {
|
||||
msg, err := json.Marshal(evnt)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to parse the event data: %v\n", err.Error())
|
||||
}
|
||||
|
||||
err = ns.client.Publish(ns.topic, msg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to send nats event: %v\n", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3log
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
)
|
||||
|
||||
type AuditLogger interface {
|
||||
Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta)
|
||||
HangUp() error
|
||||
Shutdown() error
|
||||
}
|
||||
|
||||
type LogMeta struct {
|
||||
BucketOwner string
|
||||
ObjectSize int64
|
||||
Action string
|
||||
}
|
||||
|
||||
type LogConfig struct {
|
||||
LogFile string
|
||||
WebhookURL string
|
||||
}
|
||||
|
||||
type LogFields struct {
|
||||
BucketOwner string
|
||||
Bucket string
|
||||
Time time.Time
|
||||
RemoteIP string
|
||||
Requester string
|
||||
RequestID string
|
||||
Operation string
|
||||
Key string
|
||||
RequestURI string
|
||||
HttpStatus int
|
||||
ErrorCode string
|
||||
BytesSent int
|
||||
ObjectSize int64
|
||||
TotalTime int64
|
||||
TurnAroundTime int64
|
||||
Referer string
|
||||
UserAgent string
|
||||
VersionID string
|
||||
HostID string
|
||||
SignatureVersion string
|
||||
CipherSuite string
|
||||
AuthenticationType string
|
||||
HostHeader string
|
||||
TLSVersion string
|
||||
AccessPointARN string
|
||||
AclRequired string
|
||||
}
|
||||
|
||||
func InitLogger(cfg *LogConfig) (AuditLogger, error) {
|
||||
if cfg.WebhookURL != "" && cfg.LogFile != "" {
|
||||
return nil, fmt.Errorf("there should be specified one of the following: file, webhook")
|
||||
}
|
||||
if cfg.WebhookURL != "" {
|
||||
return InitWebhookLogger(cfg.WebhookURL)
|
||||
}
|
||||
if cfg.LogFile != "" {
|
||||
return InitFileLogger(cfg.LogFile)
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func genID() string {
|
||||
src := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
b := make([]byte, 8)
|
||||
|
||||
if _, err := src.Read(b); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return strings.ToUpper(hex.EncodeToString(b))
|
||||
}
|
||||
|
||||
func getTLSVersionName(version uint16) string {
|
||||
switch version {
|
||||
case tls.VersionTLS10:
|
||||
return "TLSv1.0"
|
||||
case tls.VersionTLS11:
|
||||
return "TLSv1.1"
|
||||
case tls.VersionTLS12:
|
||||
return "TLSv1.2"
|
||||
case tls.VersionTLS13:
|
||||
return "TLSv1.3"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
230
s3log/file.go
230
s3log/file.go
@@ -1,230 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3log
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
const (
|
||||
logFileMode = 0600
|
||||
timeFormat = "02/January/2006:15:04:05 -0700"
|
||||
)
|
||||
|
||||
// FileLogger is a local file audit log
|
||||
type FileLogger struct {
|
||||
logfile string
|
||||
f *os.File
|
||||
gotErr bool
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
var _ AuditLogger = &FileLogger{}
|
||||
|
||||
// InitFileLogger initializes audit logs to local file
|
||||
func InitFileLogger(logname string) (AuditLogger, error) {
|
||||
f, err := os.OpenFile(logname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open log: %w", err)
|
||||
}
|
||||
|
||||
f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
|
||||
|
||||
return &FileLogger{logfile: logname, f: f}, nil
|
||||
}
|
||||
|
||||
// Log sends log message to file logger
|
||||
func (f *FileLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
if f.gotErr {
|
||||
return
|
||||
}
|
||||
|
||||
lf := LogFields{}
|
||||
|
||||
access := "-"
|
||||
reqURI := ctx.Request().URI().String()
|
||||
path := strings.Split(ctx.Path(), "/")
|
||||
bucket, object := path[1], strings.Join(path[2:], "/")
|
||||
errorCode := ""
|
||||
httpStatus := 200
|
||||
startTime := ctx.Locals("startTime").(time.Time)
|
||||
tlsConnState := ctx.Context().TLSConnectionState()
|
||||
if tlsConnState != nil {
|
||||
lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
|
||||
lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
serr, ok := err.(s3err.APIError)
|
||||
if ok {
|
||||
errorCode = serr.Code
|
||||
httpStatus = serr.HTTPStatusCode
|
||||
} else {
|
||||
errorCode = err.Error()
|
||||
httpStatus = 500
|
||||
}
|
||||
}
|
||||
|
||||
switch ctx.Locals("access").(type) {
|
||||
case string:
|
||||
access = ctx.Locals("access").(string)
|
||||
}
|
||||
|
||||
lf.BucketOwner = meta.BucketOwner
|
||||
lf.Bucket = bucket
|
||||
lf.Time = time.Now()
|
||||
lf.RemoteIP = ctx.IP()
|
||||
lf.Requester = access
|
||||
lf.RequestID = genID()
|
||||
lf.Operation = meta.Action
|
||||
lf.Key = object
|
||||
lf.RequestURI = reqURI
|
||||
lf.HttpStatus = httpStatus
|
||||
lf.ErrorCode = errorCode
|
||||
lf.BytesSent = len(body)
|
||||
lf.ObjectSize = meta.ObjectSize
|
||||
lf.TotalTime = time.Since(startTime).Milliseconds()
|
||||
lf.TurnAroundTime = time.Since(startTime).Milliseconds()
|
||||
lf.Referer = ctx.Get("Referer")
|
||||
lf.UserAgent = ctx.Get("User-Agent")
|
||||
lf.VersionID = ctx.Query("versionId")
|
||||
lf.HostID = ctx.Get("X-Amz-Id-2")
|
||||
lf.SignatureVersion = "SigV4"
|
||||
lf.AuthenticationType = "AuthHeader"
|
||||
lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
|
||||
lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
|
||||
lf.AclRequired = "Yes"
|
||||
|
||||
f.writeLog(lf)
|
||||
}
|
||||
|
||||
func (f *FileLogger) writeLog(lf LogFields) {
|
||||
if lf.BucketOwner == "" {
|
||||
lf.BucketOwner = "-"
|
||||
}
|
||||
if lf.Bucket == "" {
|
||||
lf.Bucket = "-"
|
||||
}
|
||||
if lf.RemoteIP == "" {
|
||||
lf.RemoteIP = "-"
|
||||
}
|
||||
if lf.Requester == "" {
|
||||
lf.Requester = "-"
|
||||
}
|
||||
if lf.Operation == "" {
|
||||
lf.Operation = "-"
|
||||
}
|
||||
if lf.Key == "" {
|
||||
lf.Key = "-"
|
||||
}
|
||||
if lf.RequestURI == "" {
|
||||
lf.RequestURI = "-"
|
||||
}
|
||||
if lf.ErrorCode == "" {
|
||||
lf.ErrorCode = "-"
|
||||
}
|
||||
if lf.Referer == "" {
|
||||
lf.Referer = "-"
|
||||
}
|
||||
if lf.UserAgent == "" {
|
||||
lf.UserAgent = "-"
|
||||
}
|
||||
if lf.VersionID == "" {
|
||||
lf.VersionID = "-"
|
||||
}
|
||||
if lf.HostID == "" {
|
||||
lf.HostID = "-"
|
||||
}
|
||||
if lf.CipherSuite == "" {
|
||||
lf.CipherSuite = "-"
|
||||
}
|
||||
if lf.HostHeader == "" {
|
||||
lf.HostHeader = "-"
|
||||
}
|
||||
if lf.TLSVersion == "" {
|
||||
lf.TLSVersion = "-"
|
||||
}
|
||||
|
||||
log := fmt.Sprintf("%v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v %v\n",
|
||||
lf.BucketOwner,
|
||||
lf.Bucket,
|
||||
fmt.Sprintf("[%v]", lf.Time.Format(timeFormat)),
|
||||
lf.RemoteIP,
|
||||
lf.Requester,
|
||||
lf.RequestID,
|
||||
lf.Operation,
|
||||
lf.Key,
|
||||
lf.RequestURI,
|
||||
lf.HttpStatus,
|
||||
lf.ErrorCode,
|
||||
lf.BytesSent,
|
||||
lf.ObjectSize,
|
||||
lf.TotalTime,
|
||||
lf.TurnAroundTime,
|
||||
lf.Referer,
|
||||
lf.UserAgent,
|
||||
lf.VersionID,
|
||||
lf.HostID,
|
||||
lf.SignatureVersion,
|
||||
lf.CipherSuite,
|
||||
lf.AuthenticationType,
|
||||
lf.HostHeader,
|
||||
lf.TLSVersion,
|
||||
lf.AccessPointARN,
|
||||
lf.AclRequired,
|
||||
)
|
||||
|
||||
_, err := f.f.WriteString(log)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error writing to log file: %v\n", err)
|
||||
// TODO: do we need to terminate on log error?
|
||||
// set err for now so that we don't spew errors
|
||||
f.gotErr = true
|
||||
}
|
||||
}
|
||||
|
||||
// HangUp closes current logfile handle and opens a new one
|
||||
// typically needed for log rotations
|
||||
func (f *FileLogger) HangUp() error {
|
||||
err := f.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("close log: %w", err)
|
||||
}
|
||||
|
||||
f.f, err = os.OpenFile(f.logfile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open log: %w", err)
|
||||
}
|
||||
|
||||
f.f.WriteString(fmt.Sprintf("log starts %v\n", time.Now()))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown closes logfile handle
|
||||
func (f *FileLogger) Shutdown() error {
|
||||
return f.f.Close()
|
||||
}
|
||||
156
s3log/webhook.go
156
s3log/webhook.go
@@ -1,156 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3log
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
// WebhookLogger is a webhook URL audit log
|
||||
type WebhookLogger struct {
|
||||
mu sync.Mutex
|
||||
url string
|
||||
}
|
||||
|
||||
var _ AuditLogger = &WebhookLogger{}
|
||||
|
||||
// InitWebhookLogger initializes audit logs to webhook URL
|
||||
func InitWebhookLogger(url string) (AuditLogger, error) {
|
||||
client := &http.Client{
|
||||
Timeout: 3 * time.Second,
|
||||
}
|
||||
_, err := client.Post(url, "application/json", nil)
|
||||
if err != nil {
|
||||
if err, ok := err.(net.Error); ok && !err.Timeout() {
|
||||
return nil, fmt.Errorf("unreachable webhook url: %w", err)
|
||||
}
|
||||
}
|
||||
return &WebhookLogger{
|
||||
url: url,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Log sends log message to webhook
|
||||
func (wl *WebhookLogger) Log(ctx *fiber.Ctx, err error, body []byte, meta LogMeta) {
|
||||
wl.mu.Lock()
|
||||
defer wl.mu.Unlock()
|
||||
|
||||
lf := LogFields{}
|
||||
|
||||
access := "-"
|
||||
reqURI := ctx.Request().URI().String()
|
||||
path := strings.Split(ctx.Path(), "/")
|
||||
bucket, object := path[1], strings.Join(path[2:], "/")
|
||||
errorCode := ""
|
||||
httpStatus := 200
|
||||
startTime := ctx.Locals("startTime").(time.Time)
|
||||
tlsConnState := ctx.Context().TLSConnectionState()
|
||||
if tlsConnState != nil {
|
||||
lf.CipherSuite = tls.CipherSuiteName(tlsConnState.CipherSuite)
|
||||
lf.TLSVersion = getTLSVersionName(tlsConnState.Version)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
serr, ok := err.(s3err.APIError)
|
||||
if ok {
|
||||
errorCode = serr.Code
|
||||
httpStatus = serr.HTTPStatusCode
|
||||
} else {
|
||||
errorCode = err.Error()
|
||||
httpStatus = 500
|
||||
}
|
||||
}
|
||||
|
||||
switch ctx.Locals("access").(type) {
|
||||
case string:
|
||||
access = ctx.Locals("access").(string)
|
||||
}
|
||||
|
||||
lf.BucketOwner = meta.BucketOwner
|
||||
lf.Bucket = bucket
|
||||
lf.Time = time.Now()
|
||||
lf.RemoteIP = ctx.IP()
|
||||
lf.Requester = access
|
||||
lf.RequestID = genID()
|
||||
lf.Operation = meta.Action
|
||||
lf.Key = object
|
||||
lf.RequestURI = reqURI
|
||||
lf.HttpStatus = httpStatus
|
||||
lf.ErrorCode = errorCode
|
||||
lf.BytesSent = len(body)
|
||||
lf.ObjectSize = meta.ObjectSize
|
||||
lf.TotalTime = time.Since(startTime).Milliseconds()
|
||||
lf.TurnAroundTime = time.Since(startTime).Milliseconds()
|
||||
lf.Referer = ctx.Get("Referer")
|
||||
lf.UserAgent = ctx.Get("User-Agent")
|
||||
lf.VersionID = ctx.Query("versionId")
|
||||
lf.HostID = ctx.Get("X-Amz-Id-2")
|
||||
lf.SignatureVersion = "SigV4"
|
||||
lf.AuthenticationType = "AuthHeader"
|
||||
lf.HostHeader = fmt.Sprintf("s3.%v.amazonaws.com", ctx.Locals("region").(string))
|
||||
lf.AccessPointARN = fmt.Sprintf("arn:aws:s3:::%v", strings.Join(path, "/"))
|
||||
lf.AclRequired = "Yes"
|
||||
|
||||
wl.sendLog(lf)
|
||||
}
|
||||
|
||||
func (wl *WebhookLogger) sendLog(lf LogFields) {
|
||||
jsonLog, err := json.Marshal(lf)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to parse the log data: %v\n", err.Error())
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, wl.url, bytes.NewReader(jsonLog))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
|
||||
go makeRequest(req)
|
||||
}
|
||||
|
||||
func makeRequest(req *http.Request) {
|
||||
client := &http.Client{
|
||||
Timeout: 1 * time.Second,
|
||||
}
|
||||
_, err := client.Do(req)
|
||||
if err != nil {
|
||||
if err, ok := err.(net.Error); ok && !err.Timeout() {
|
||||
fmt.Fprintf(os.Stderr, "error sending webhook log: %v\n", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// HangUp does nothing for webhooks
|
||||
func (wl *WebhookLogger) HangUp() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown does nothing for webhooks
|
||||
func (wl *WebhookLogger) Shutdown() error {
|
||||
return nil
|
||||
}
|
||||
@@ -1,692 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<xsd:schema
|
||||
xmlns:tns="http://s3.amazonaws.com/doc/2006-03-01/"
|
||||
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
|
||||
elementFormDefault="qualified"
|
||||
targetNamespace="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
|
||||
<xsd:element name="CreateBucket">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:complexType name="MetadataEntry">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Name" type="xsd:string"/>
|
||||
<xsd:element name="Value" type="xsd:string"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:element name="CreateBucketResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="CreateBucketReturn" type="tns:CreateBucketResult"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:complexType name="Status">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Code" type="xsd:int"/>
|
||||
<xsd:element name="Description" type="xsd:string"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="Result">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Status" type="tns:Status"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="CreateBucketResult">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="BucketName" type="xsd:string"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:element name="DeleteBucket">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
<xsd:element name="DeleteBucketResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="DeleteBucketResponse" type="tns:Status"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:complexType name="BucketLoggingStatus">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="LoggingEnabled" type="tns:LoggingSettings" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="LoggingSettings">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="TargetBucket" type="xsd:string"/>
|
||||
<xsd:element name="TargetPrefix" type="xsd:string"/>
|
||||
<xsd:element name="TargetGrants" type="tns:AccessControlList" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:element name="GetBucketLoggingStatus">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="GetBucketLoggingStatusResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="GetBucketLoggingStatusResponse" type="tns:BucketLoggingStatus"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="SetBucketLoggingStatus">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="BucketLoggingStatus" type="tns:BucketLoggingStatus"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="SetBucketLoggingStatusResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence/>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="GetObjectAccessControlPolicy">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="Key" type="xsd:string"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="GetObjectAccessControlPolicyResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="GetObjectAccessControlPolicyResponse" type="tns:AccessControlPolicy"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="GetBucketAccessControlPolicy">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="GetBucketAccessControlPolicyResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="GetBucketAccessControlPolicyResponse" type="tns:AccessControlPolicy"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:complexType abstract="true" name="Grantee"/>
|
||||
|
||||
<xsd:complexType name="User" abstract="true">
|
||||
<xsd:complexContent>
|
||||
<xsd:extension base="tns:Grantee"/>
|
||||
</xsd:complexContent>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="AmazonCustomerByEmail">
|
||||
<xsd:complexContent>
|
||||
<xsd:extension base="tns:User">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="EmailAddress" type="xsd:string"/>
|
||||
</xsd:sequence>
|
||||
</xsd:extension>
|
||||
</xsd:complexContent>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="CanonicalUser">
|
||||
<xsd:complexContent>
|
||||
<xsd:extension base="tns:User">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="ID" type="xsd:string"/>
|
||||
<xsd:element name="DisplayName" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:extension>
|
||||
</xsd:complexContent>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="Group">
|
||||
<xsd:complexContent>
|
||||
<xsd:extension base="tns:Grantee">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="URI" type="xsd:string"/>
|
||||
</xsd:sequence>
|
||||
</xsd:extension>
|
||||
</xsd:complexContent>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:simpleType name="Permission">
|
||||
<xsd:restriction base="xsd:string">
|
||||
<xsd:enumeration value="READ"/>
|
||||
<xsd:enumeration value="WRITE"/>
|
||||
<xsd:enumeration value="READ_ACP"/>
|
||||
<xsd:enumeration value="WRITE_ACP"/>
|
||||
<xsd:enumeration value="FULL_CONTROL"/>
|
||||
</xsd:restriction>
|
||||
</xsd:simpleType>
|
||||
|
||||
<xsd:simpleType name="StorageClass">
|
||||
<xsd:restriction base="xsd:string">
|
||||
<xsd:enumeration value="STANDARD"/>
|
||||
<xsd:enumeration value="REDUCED_REDUNDANCY"/>
|
||||
<xsd:enumeration value="GLACIER"/>
|
||||
<xsd:enumeration value="UNKNOWN"/>
|
||||
</xsd:restriction>
|
||||
</xsd:simpleType>
|
||||
|
||||
<xsd:complexType name="Grant">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Grantee" type="tns:Grantee"/>
|
||||
<xsd:element name="Permission" type="tns:Permission"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="AccessControlList">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Grant" type="tns:Grant" minOccurs="0" maxOccurs="100"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="CreateBucketConfiguration">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="LocationConstraint" type="tns:LocationConstraint"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="LocationConstraint">
|
||||
<xsd:simpleContent>
|
||||
<xsd:extension base="xsd:string"/>
|
||||
</xsd:simpleContent>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="AccessControlPolicy">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Owner" type="tns:CanonicalUser"/>
|
||||
<xsd:element name="AccessControlList" type="tns:AccessControlList"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:element name="SetObjectAccessControlPolicy">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="Key" type="xsd:string"/>
|
||||
<xsd:element name="AccessControlList" type="tns:AccessControlList"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="SetObjectAccessControlPolicyResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence/>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="SetBucketAccessControlPolicy">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="SetBucketAccessControlPolicyResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence/>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="GetObject">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="Key" type="xsd:string"/>
|
||||
<xsd:element name="GetMetadata" type="xsd:boolean"/>
|
||||
<xsd:element name="GetData" type="xsd:boolean"/>
|
||||
<xsd:element name="InlineData" type="xsd:boolean"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="GetObjectResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="GetObjectResponse" type="tns:GetObjectResult"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:complexType name="GetObjectResult">
|
||||
<xsd:complexContent>
|
||||
<xsd:extension base="tns:Result">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="unbounded"/>
|
||||
<xsd:element name="Data" type="xsd:base64Binary" nillable="true"/>
|
||||
<xsd:element name="LastModified" type="xsd:dateTime"/>
|
||||
<xsd:element name="ETag" type="xsd:string"/>
|
||||
</xsd:sequence>
|
||||
</xsd:extension>
|
||||
</xsd:complexContent>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:element name="GetObjectExtended">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="Key" type="xsd:string"/>
|
||||
<xsd:element name="GetMetadata" type="xsd:boolean"/>
|
||||
<xsd:element name="GetData" type="xsd:boolean"/>
|
||||
<xsd:element name="InlineData" type="xsd:boolean"/>
|
||||
<xsd:element name="ByteRangeStart" type="xsd:long" minOccurs="0"/>
|
||||
<xsd:element name="ByteRangeEnd" type="xsd:long" minOccurs="0"/>
|
||||
<xsd:element name="IfModifiedSince" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="IfUnmodifiedSince" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="IfMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
|
||||
<xsd:element name="IfNoneMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
|
||||
<xsd:element name="ReturnCompleteObjectOnConditionFailure" type="xsd:boolean" minOccurs="0"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="GetObjectExtendedResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="GetObjectResponse" type="tns:GetObjectResult"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="PutObject">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="Key" type="xsd:string"/>
|
||||
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="100"/>
|
||||
<xsd:element name="ContentLength" type="xsd:long"/>
|
||||
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
|
||||
<xsd:element name="StorageClass" type="tns:StorageClass" minOccurs="0"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="PutObjectResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="PutObjectResponse" type="tns:PutObjectResult"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:complexType name="PutObjectResult">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="ETag" type="xsd:string"/>
|
||||
<xsd:element name="LastModified" type="xsd:dateTime"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:element name="PutObjectInline">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="Key" type="xsd:string"/>
|
||||
<xsd:element minOccurs="0" maxOccurs="100" name="Metadata" type="tns:MetadataEntry"/>
|
||||
<xsd:element name="Data" type="xsd:base64Binary"/>
|
||||
<xsd:element name="ContentLength" type="xsd:long"/>
|
||||
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
|
||||
<xsd:element name="StorageClass" type="tns:StorageClass" minOccurs="0"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="PutObjectInlineResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="PutObjectInlineResponse" type="tns:PutObjectResult"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="DeleteObject">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="Key" type="xsd:string"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="DeleteObjectResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="DeleteObjectResponse" type="tns:Status"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="ListBucket">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="Prefix" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Marker" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="MaxKeys" type="xsd:int" minOccurs="0"/>
|
||||
<xsd:element name="Delimiter" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="ListBucketResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="ListBucketResponse" type="tns:ListBucketResult"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="ListVersionsResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="ListVersionsResponse" type="tns:ListVersionsResult"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:complexType name="ListEntry">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Key" type="xsd:string"/>
|
||||
<xsd:element name="LastModified" type="xsd:dateTime"/>
|
||||
<xsd:element name="ETag" type="xsd:string"/>
|
||||
<xsd:element name="Size" type="xsd:long"/>
|
||||
<xsd:element name="Owner" type="tns:CanonicalUser" minOccurs="0"/>
|
||||
<xsd:element name="StorageClass" type="tns:StorageClass"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="VersionEntry">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Key" type="xsd:string"/>
|
||||
<xsd:element name="VersionId" type="xsd:string"/>
|
||||
<xsd:element name="IsLatest" type="xsd:boolean"/>
|
||||
<xsd:element name="LastModified" type="xsd:dateTime"/>
|
||||
<xsd:element name="ETag" type="xsd:string"/>
|
||||
<xsd:element name="Size" type="xsd:long"/>
|
||||
<xsd:element name="Owner" type="tns:CanonicalUser" minOccurs="0"/>
|
||||
<xsd:element name="StorageClass" type="tns:StorageClass"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="DeleteMarkerEntry">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Key" type="xsd:string"/>
|
||||
<xsd:element name="VersionId" type="xsd:string"/>
|
||||
<xsd:element name="IsLatest" type="xsd:boolean"/>
|
||||
<xsd:element name="LastModified" type="xsd:dateTime"/>
|
||||
<xsd:element name="Owner" type="tns:CanonicalUser" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="PrefixEntry">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Prefix" type="xsd:string"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="ListBucketResult">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="unbounded"/>
|
||||
<xsd:element name="Name" type="xsd:string"/>
|
||||
<xsd:element name="Prefix" type="xsd:string"/>
|
||||
<xsd:element name="Marker" type="xsd:string"/>
|
||||
<xsd:element name="NextMarker" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="MaxKeys" type="xsd:int"/>
|
||||
<xsd:element name="Delimiter" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="IsTruncated" type="xsd:boolean"/>
|
||||
<xsd:element name="Contents" type="tns:ListEntry" minOccurs="0" maxOccurs="unbounded"/>
|
||||
<xsd:element name="CommonPrefixes" type="tns:PrefixEntry" minOccurs="0" maxOccurs="unbounded"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="ListVersionsResult">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="unbounded"/>
|
||||
<xsd:element name="Name" type="xsd:string"/>
|
||||
<xsd:element name="Prefix" type="xsd:string"/>
|
||||
<xsd:element name="KeyMarker" type="xsd:string"/>
|
||||
<xsd:element name="VersionIdMarker" type="xsd:string"/>
|
||||
<xsd:element name="NextKeyMarker" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="NextVersionIdMarker" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="MaxKeys" type="xsd:int"/>
|
||||
<xsd:element name="Delimiter" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="IsTruncated" type="xsd:boolean"/>
|
||||
<xsd:choice minOccurs="0" maxOccurs="unbounded">
|
||||
<xsd:element name="Version" type="tns:VersionEntry"/>
|
||||
<xsd:element name="DeleteMarker" type="tns:DeleteMarkerEntry"/>
|
||||
</xsd:choice>
|
||||
<xsd:element name="CommonPrefixes" type="tns:PrefixEntry" minOccurs="0" maxOccurs="unbounded"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:element name="ListAllMyBuckets">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="ListAllMyBucketsResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="ListAllMyBucketsResponse" type="tns:ListAllMyBucketsResult"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:complexType name="ListAllMyBucketsEntry">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Name" type="xsd:string"/>
|
||||
<xsd:element name="CreationDate" type="xsd:dateTime"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="ListAllMyBucketsResult">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Owner" type="tns:CanonicalUser"/>
|
||||
<xsd:element name="Buckets" type="tns:ListAllMyBucketsList"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="ListAllMyBucketsList">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Bucket" type="tns:ListAllMyBucketsEntry" minOccurs="0" maxOccurs="unbounded"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:element name="PostResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Location" type="xsd:anyURI"/>
|
||||
<xsd:element name="Bucket" type="xsd:string"/>
|
||||
<xsd:element name="Key" type="xsd:string"/>
|
||||
<xsd:element name="ETag" type="xsd:string"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:simpleType name="MetadataDirective">
|
||||
<xsd:restriction base="xsd:string">
|
||||
<xsd:enumeration value="COPY"/>
|
||||
<xsd:enumeration value="REPLACE"/>
|
||||
</xsd:restriction>
|
||||
</xsd:simpleType>
|
||||
|
||||
<xsd:element name="CopyObject">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="SourceBucket" type="xsd:string"/>
|
||||
<xsd:element name="SourceKey" type="xsd:string"/>
|
||||
<xsd:element name="DestinationBucket" type="xsd:string"/>
|
||||
<xsd:element name="DestinationKey" type="xsd:string"/>
|
||||
<xsd:element name="MetadataDirective" type="tns:MetadataDirective" minOccurs="0"/>
|
||||
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="100"/>
|
||||
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
|
||||
<xsd:element name="CopySourceIfModifiedSince" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="CopySourceIfUnmodifiedSince" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="CopySourceIfMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
|
||||
<xsd:element name="CopySourceIfNoneMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
|
||||
<xsd:element name="StorageClass" type="tns:StorageClass" minOccurs="0"/>
|
||||
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
|
||||
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
|
||||
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:element name="CopyObjectResponse">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="CopyObjectResult" type="tns:CopyObjectResult" />
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
|
||||
<xsd:complexType name="CopyObjectResult">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="LastModified" type="xsd:dateTime"/>
|
||||
<xsd:element name="ETag" type="xsd:string"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="RequestPaymentConfiguration">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Payer" type="tns:Payer" minOccurs="1" maxOccurs="1"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:simpleType name="Payer">
|
||||
<xsd:restriction base="xsd:string">
|
||||
<xsd:enumeration value="BucketOwner"/>
|
||||
<xsd:enumeration value="Requester"/>
|
||||
</xsd:restriction>
|
||||
</xsd:simpleType>
|
||||
|
||||
<xsd:complexType name="VersioningConfiguration">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Status" type="tns:VersioningStatus" minOccurs="0"/>
|
||||
<xsd:element name="MfaDelete" type="tns:MfaDeleteStatus" minOccurs="0"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:simpleType name="MfaDeleteStatus">
|
||||
<xsd:restriction base="xsd:string">
|
||||
<xsd:enumeration value="Enabled"/>
|
||||
<xsd:enumeration value="Disabled"/>
|
||||
</xsd:restriction>
|
||||
</xsd:simpleType>
|
||||
|
||||
<xsd:simpleType name="VersioningStatus">
|
||||
<xsd:restriction base="xsd:string">
|
||||
<xsd:enumeration value="Enabled"/>
|
||||
<xsd:enumeration value="Suspended"/>
|
||||
</xsd:restriction>
|
||||
</xsd:simpleType>
|
||||
|
||||
<xsd:complexType name="NotificationConfiguration">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="TopicConfiguration" minOccurs="0" maxOccurs="unbounded" type="tns:TopicConfiguration"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
<xsd:complexType name="TopicConfiguration">
|
||||
<xsd:sequence>
|
||||
<xsd:element name="Topic" minOccurs="1" maxOccurs="1" type="xsd:string"/>
|
||||
<xsd:element name="Event" minOccurs="1" maxOccurs="unbounded" type="xsd:string"/>
|
||||
</xsd:sequence>
|
||||
</xsd:complexType>
|
||||
|
||||
</xsd:schema>
|
||||
@@ -1,6 +0,0 @@
|
||||
https://doc.s3.amazonaws.com/2006-03-01/AmazonS3.xsd
|
||||
|
||||
see https://blog.aqwari.net/xml-schema-go/
|
||||
|
||||
go install aqwari.net/xml/cmd/xsdgen@latest
|
||||
xsdgen -o s3api_xsd_generated.go -pkg s3response AmazonS3.xsd
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,136 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3response
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
)
|
||||
|
||||
// Part describes part metadata.
|
||||
type Part struct {
|
||||
PartNumber int
|
||||
LastModified string
|
||||
ETag string
|
||||
Size int64
|
||||
}
|
||||
|
||||
// ListPartsResponse - s3 api list parts response.
|
||||
type ListPartsResult struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
|
||||
|
||||
Bucket string
|
||||
Key string
|
||||
UploadID string `xml:"UploadId"`
|
||||
|
||||
Initiator Initiator
|
||||
Owner Owner
|
||||
|
||||
// The class of storage used to store the object.
|
||||
StorageClass string
|
||||
|
||||
PartNumberMarker int
|
||||
NextPartNumberMarker int
|
||||
MaxParts int
|
||||
IsTruncated bool
|
||||
|
||||
// List of parts.
|
||||
Parts []Part `xml:"Part"`
|
||||
}
|
||||
|
||||
// ListMultipartUploadsResponse - s3 api list multipart uploads response.
|
||||
type ListMultipartUploadsResult struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
|
||||
|
||||
Bucket string
|
||||
KeyMarker string
|
||||
UploadIDMarker string `xml:"UploadIdMarker"`
|
||||
NextKeyMarker string
|
||||
NextUploadIDMarker string `xml:"NextUploadIdMarker"`
|
||||
Delimiter string
|
||||
Prefix string
|
||||
EncodingType string `xml:"EncodingType,omitempty"`
|
||||
MaxUploads int
|
||||
IsTruncated bool
|
||||
|
||||
// List of pending uploads.
|
||||
Uploads []Upload `xml:"Upload"`
|
||||
|
||||
// Delimed common prefixes.
|
||||
CommonPrefixes []CommonPrefix
|
||||
}
|
||||
|
||||
// Upload desribes in progress multipart upload
|
||||
type Upload struct {
|
||||
Key string
|
||||
UploadID string `xml:"UploadId"`
|
||||
Initiator Initiator
|
||||
Owner Owner
|
||||
StorageClass string
|
||||
Initiated string
|
||||
}
|
||||
|
||||
// CommonPrefix ListObjectsResponse common prefixes (directory abstraction)
|
||||
type CommonPrefix struct {
|
||||
Prefix string
|
||||
}
|
||||
|
||||
// Initiator same fields as Owner
|
||||
type Initiator Owner
|
||||
|
||||
// Owner bucket ownership
|
||||
type Owner struct {
|
||||
ID string
|
||||
DisplayName string
|
||||
}
|
||||
|
||||
type Tag struct {
|
||||
Key string `xml:"Key"`
|
||||
Value string `xml:"Value"`
|
||||
}
|
||||
|
||||
type TagSet struct {
|
||||
Tags []Tag `xml:"Tag"`
|
||||
}
|
||||
|
||||
type Tagging struct {
|
||||
TagSet TagSet `xml:"TagSet"`
|
||||
}
|
||||
|
||||
type DeleteObjects struct {
|
||||
Objects []types.ObjectIdentifier `xml:"Object"`
|
||||
}
|
||||
|
||||
type DeleteObjectsResult struct {
|
||||
Deleted []types.DeletedObject
|
||||
Errors []types.Error
|
||||
}
|
||||
type SelectObjectContentPayload struct {
|
||||
Expression *string
|
||||
ExpressionType types.ExpressionType
|
||||
RequestProgress *types.RequestProgress
|
||||
InputSerialization *types.InputSerialization
|
||||
OutputSerialization *types.OutputSerialization
|
||||
ScanRange *types.ScanRange
|
||||
}
|
||||
|
||||
type SelectObjectContentResult struct {
|
||||
Records *types.RecordsEvent
|
||||
Stats *types.StatsEvent
|
||||
Progress *types.ProgressEvent
|
||||
Cont *string
|
||||
End *string
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
%global debug_package %{nil}
|
||||
%define pkg_version @@VERSION@@
|
||||
|
||||
Name: versitygw
|
||||
Version: %{pkg_version}
|
||||
Release: 1%{?dist}
|
||||
Summary: Versity S3 Gateway
|
||||
|
||||
License: Apache-2.0
|
||||
URL: https://github.com/versity/versitygw
|
||||
Source0: %{name}-%{version}.tar.gz
|
||||
%description
|
||||
The S3 gateway is an S3 protocol translator that allows an S3 client
|
||||
to access the supported backend storage as if it was a native S3 service.
|
||||
|
||||
BuildRequires: golang >= 1.20
|
||||
|
||||
%prep
|
||||
%setup
|
||||
|
||||
%build
|
||||
make
|
||||
|
||||
%install
|
||||
mkdir -p %{buildroot}%{_bindir}
|
||||
install -m 0755 %{name} %{buildroot}%{_bindir}/
|
||||
|
||||
%post
|
||||
|
||||
%files
|
||||
%{_bindir}/%{name}
|
||||
Reference in New Issue
Block a user