Compare commits

..

1 Commits

Author SHA1 Message Date
jonaustin09
27a8aa66d9 feat: Added s3 proxy comparison to upload, download and throughput benchmark tests 2023-11-20 14:28:02 -05:00
132 changed files with 2444 additions and 15761 deletions

View File

@@ -1,46 +0,0 @@
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
cmd/versitygw/versitygw
/versitygw
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Go workspace file
go.work
# ignore IntelliJ directories
.idea
# auto generated VERSION file
VERSION
# build output
/versitygw.spec
/versitygw.spec.in
*.tar
*.tar.gz
**/rand.data
/profile.txt
dist/
# Release config files
/.github
# Docker configuration files
*Dockerfile
/docker-compose.yml
# read files
/LICENSE
/NOTICE
/CODE_OF_CONDUCT.md
/README.md

View File

@@ -1,8 +0,0 @@
POSIX_PORT=7071
PROXY_PORT=7070
ACCESS_KEY_ID=user
SECRET_ACCESS_KEY=pass
IAM_DIR=.
SETUP_DIR=.
AZ_ACCOUNT_NAME=devstoreaccount1
AZ_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==

View File

@@ -1,45 +0,0 @@
name: Publish Docker image
on:
release:
types: [published]
jobs:
push_to_registries:
name: Push Docker image to multiple registries
runs-on: ubuntu-latest
permissions:
packages: write
contents: read
steps:
- name: Check out the repo
uses: actions/checkout@v4
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Log in to the Container registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v5
with:
images: |
versity/versitygw
ghcr.io/${{ github.repository }}
- name: Build and push Docker images
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

View File

@@ -1,49 +0,0 @@
name: system tests
on: pull_request
jobs:
build:
name: RunTests
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
- name: Install ShellCheck
run: sudo apt-get install shellcheck
- name: Run ShellCheck
run: shellcheck -S warning ./tests/*.sh
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 'stable'
id: go
- name: Get Dependencies
run: |
go get -v -t -d ./...
- name: Install BATS
run: |
git clone https://github.com/bats-core/bats-core.git
cd bats-core && ./install.sh $HOME
- name: Install s3cmd
run: |
sudo apt-get install s3cmd
- name: Build and run
run: |
make testbin
export AWS_ACCESS_KEY_ID=user
export AWS_SECRET_ACCESS_KEY=pass
export AWS_REGION=us-east-1
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile versity
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile versity
aws configure set aws_region $AWS_REGION --profile versity
mkdir /tmp/gw
export WORKSPACE=$GITHUB_WORKSPACE
openssl genpkey -algorithm RSA -out versitygw.pem -pkeyopt rsa_keygen_bits:2048
openssl req -new -x509 -key versitygw.pem -out cert.pem -days 365 -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
VERSITYGW_TEST_ENV=./tests/.env.default ./tests/run_all.sh

13
.gitignore vendored
View File

@@ -25,9 +25,6 @@ go.work
# ignore IntelliJ directories
.idea
# ignore VS code directories
.vscode
# auto generated VERSION file
VERSION
@@ -39,13 +36,3 @@ VERSION
/profile.txt
dist/
# secrets file for local github-actions testing
tests/.secrets
# IAM users files often created in testing
users.json
# env files for testing
.env*
!.env.default

View File

@@ -9,12 +9,9 @@ builds:
# windows is untested, we can start doing windows releases
# if someone is interested in taking on testing
# - windows
env:
# disable cgo to fix glibc issues: https://github.com/golang/go/issues/58550
# once we need to enable this, we will need to do per distro releases
- CGO_ENABLED=0
main: ./cmd/versitygw
binary: versitygw
binary: ./cmd/versitygw
id: versitygw
goarch:
- amd64
- arm64
@@ -25,7 +22,7 @@ archives:
- format: tar.gz
# this name template makes the OS and Arch compatible with the results of uname.
name_template: >-
{{ .ProjectName }}_v{{ .Version }}_
{{ .ProjectName }}_
{{- title .Os }}_
{{- if eq .Arch "amd64" }}x86_64
{{- else if eq .Arch "386" }}i386
@@ -50,38 +47,5 @@ changelog:
- '^test:'
- '^Merge '
nfpms:
- id: packages
package_name: versitygw
vendor: Versity Software
homepage: https://github.com/versity/versitygw
maintainer: Ben McClelland <ben.mcclelland@versity.com>
description: |-
The Versity S3 Gateway.
A high-performance tool facilitating translation between AWS S3 API
requests and various backend storage systems, including POSIX file
backend storage. Its stateless architecture enables deployment in
clusters for increased throughput, distributing requests across gateways
for optimal performance. With a focus on modularity, it supports future
extensions for additional backend systems.
license: Apache 2.0
builds:
- versitygw
formats:
- deb
- rpm
umask: 0o002
bindir: /usr/bin
epoch: "1"
release: "1"
rpm:
group: "System Environment/Daemons"
# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
# vim: set ts=2 sw=2 tw=0 fo=cnqoj

View File

@@ -1,25 +0,0 @@
FROM golang:latest
WORKDIR /app
COPY go.mod ./
RUN go mod download
COPY ./ ./
WORKDIR /app/cmd/versitygw
ENV CGO_ENABLED=0
RUN go build -o versitygw
FROM alpine:latest
# These arguments can be overriden when building the image
ARG IAM_DIR=/tmp/vgw
ARG SETUP_DIR=/tmp/vgw
RUN mkdir -p $IAM_DIR
RUN mkdir -p $SETUP_DIR
COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
ENTRYPOINT [ "/app/versitygw" ]

View File

@@ -1,18 +0,0 @@
FROM golang:latest
WORKDIR /app
COPY go.mod ./
RUN go mod download
COPY ./ ./
COPY certs/* /etc/pki/tls/certs/
ARG IAM_DIR=/tmp/vgw
ARG SETUP_DIR=/tmp/vgw
RUN mkdir -p $IAM_DIR
RUN mkdir -p $SETUP_DIR
RUN go get github.com/githubnemo/CompileDaemon
RUN go install github.com/githubnemo/CompileDaemon

View File

@@ -1,70 +0,0 @@
FROM --platform=linux/arm64 ubuntu:latest
ARG DEBIAN_FRONTEND=noninteractive
ENV TZ=Etc/UTC
RUN apt-get update && \
apt-get install -y --no-install-recommends \
git \
make \
wget \
curl \
unzip \
tzdata \
s3cmd \
jq \
ca-certificates && \
update-ca-certificates && \
rm -rf /var/lib/apt/lists/*
# Set working directory
WORKDIR /tmp
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install
# Download Go 1.21 (adjust the version and platform as needed)
RUN wget https://golang.org/dl/go1.21.7.linux-arm64.tar.gz
# Extract the downloaded archive
RUN tar -xvf go1.21.7.linux-arm64.tar.gz -C /usr/local
# Set Go environment variables
ENV PATH="/usr/local/go/bin:${PATH}"
ENV GOPATH="/go"
ENV GOBIN="$GOPATH/bin"
# Make the directory for Go packages
RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
# Create tester user
RUN groupadd -r tester && useradd -r -g tester tester
RUN mkdir /home/tester && chown tester:tester /home/tester
ENV HOME=/home/tester
RUN git clone https://github.com/bats-core/bats-core.git && \
cd bats-core && \
./install.sh /home/tester
USER tester
COPY --chown=tester:tester . /home/tester
WORKDIR /home/tester
RUN cp tests/.env.docker.default tests/.env.docker
RUN cp tests/s3cfg.local.default tests/s3cfg.local
RUN make
RUN . tests/.secrets && \
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE && \
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile $AWS_PROFILE && \
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile $AWS_PROFILE && \
aws configure set aws_region $AWS_REGION --profile $AWS_PROFILE
RUN mkdir /tmp/gw
RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen_bits:2048 && \
openssl req -new -x509 -key versitygw-docker.pem -out cert-docker.pem -days 365 \
-subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
ENV WORKSPACE=.
ENV VERSITYGW_TEST_ENV=tests/.env.docker
CMD ["tests/run_all.sh"]

View File

@@ -74,23 +74,3 @@ dist: $(BIN).spec
rm -f VERSION
rm -f $(BIN).spec
gzip -f $(TARFILE)
# Creates and runs S3 gateway instance in a docker container
.PHONY: up-posix
up-posix:
docker compose --env-file .env.dev up posix
# Creates and runs S3 gateway proxy instance in a docker container
.PHONY: up-proxy
up-proxy:
docker compose --env-file .env.dev up proxy
# Creates and runs S3 gateway to azurite instance in a docker container
.PHONY: up-azurite
up-azurite:
docker compose --env-file .env.dev up azurite azuritegw
# Creates and runs both S3 gateway and proxy server instances in docker containers
.PHONY: up-app
up-app:
docker compose --env-file .env.dev up

View File

@@ -8,18 +8,13 @@
[![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/versity/versitygw/blob/main/LICENSE)
**Current status:** Ready for general testing, Issue reports welcome.
**News:**<br>
* New performance analysis article [https://github.com/versity/versitygw/wiki/Performance](https://github.com/versity/versitygw/wiki/Performance)
**Current status:** Beta: Most clients functional, work in progress for more test coverage. Issue reports welcome.
See project [documentation](https://github.com/versity/versitygw/wiki) on the wiki.
* Share filesystem directory via S3 protocol
* Proxy S3 requests to S3 storage
* Simple to deploy S3 server with a single command
* Protocol compatibility in `posix` allows common access to files via posix or S3
* Protocol compatibility allows common access to files via posix or S3
Versity Gateway, a simple to use tool for seamless inline translation between AWS S3 object commands and storage systems. The Versity Gateway bridges the gap between S3-reliant applications and other storage systems, enabling enhanced compatibility and integration while offering exceptional scalability.

View File

@@ -99,34 +99,34 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
grantees := []Grantee{}
accs := []string{}
if input.GrantRead != nil || input.GrantReadACP != nil || input.GrantFullControl != nil || input.GrantWrite != nil || input.GrantWriteACP != nil {
if input.GrantRead != nil {
fullControlList, readList, readACPList, writeList, writeACPList := []string{}, []string{}, []string{}, []string{}, []string{}
if input.GrantFullControl != nil && *input.GrantFullControl != "" {
if *input.GrantFullControl != "" {
fullControlList = splitUnique(*input.GrantFullControl, ",")
for _, str := range fullControlList {
grantees = append(grantees, Grantee{Access: str, Permission: "FULL_CONTROL"})
}
}
if input.GrantRead != nil && *input.GrantRead != "" {
if *input.GrantRead != "" {
readList = splitUnique(*input.GrantRead, ",")
for _, str := range readList {
grantees = append(grantees, Grantee{Access: str, Permission: "READ"})
}
}
if input.GrantReadACP != nil && *input.GrantReadACP != "" {
if *input.GrantReadACP != "" {
readACPList = splitUnique(*input.GrantReadACP, ",")
for _, str := range readACPList {
grantees = append(grantees, Grantee{Access: str, Permission: "READ_ACP"})
}
}
if input.GrantWrite != nil && *input.GrantWrite != "" {
if *input.GrantWrite != "" {
writeList = splitUnique(*input.GrantWrite, ",")
for _, str := range writeList {
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE"})
}
}
if input.GrantWriteACP != nil && *input.GrantWriteACP != "" {
if *input.GrantWriteACP != "" {
writeACPList = splitUnique(*input.GrantWriteACP, ",")
for _, str := range writeACPList {
grantees = append(grantees, Grantee{Access: str, Permission: "WRITE_ACP"})
@@ -137,9 +137,6 @@ func UpdateACL(input *s3.PutBucketAclInput, acl ACL, iam IAMService) ([]byte, er
} else {
cache := make(map[string]bool)
for _, grt := range input.AccessControlPolicy.Grants {
if grt.Grantee == nil || grt.Grantee.ID == nil || grt.Permission == "" {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
grantees = append(grantees, Grantee{Access: *grt.Grantee.ID, Permission: grt.Permission})
if _, ok := cache[*grt.Grantee.ID]; !ok {
cache[*grt.Grantee.ID] = true
@@ -242,34 +239,14 @@ func VerifyACL(acl ACL, access string, permission types.Permission, isRoot bool)
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
func MayCreateBucket(acct Account, isRoot bool) error {
func IsAdmin(acct Account, isRoot bool) error {
if isRoot {
return nil
}
if acct.Role == RoleUser {
return s3err.GetAPIError(s3err.ErrAccessDenied)
}
return nil
}
func IsAdminOrOwner(acct Account, isRoot bool, acl ACL) error {
// Owner check
if acct.Access == acl.Owner {
if acct.Role == "admin" {
return nil
}
// Root user has access over almost everything
if isRoot {
return nil
}
// Admin user case
if acct.Role == RoleAdmin {
return nil
}
// Return access denied in all other cases
return s3err.GetAPIError(s3err.ErrAccessDenied)
}

View File

@@ -16,23 +16,14 @@ package auth
import (
"errors"
"fmt"
"time"
)
type Role string
const (
RoleUser Role = "user"
RoleAdmin Role = "admin"
RoleUserPlus Role = "userplus"
)
// Account is a gateway IAM account
type Account struct {
Access string `json:"access"`
Secret string `json:"secret"`
Role Role `json:"role"`
Role string `json:"role"`
UserID int `json:"userID"`
GroupID int `json:"groupID"`
ProjectID int `json:"projectID"`
@@ -52,25 +43,18 @@ type IAMService interface {
var ErrNoSuchUser = errors.New("user not found")
type Opts struct {
Dir string
LDAPServerURL string
LDAPBindDN string
LDAPPassword string
LDAPQueryBase string
LDAPObjClasses string
LDAPAccessAtr string
LDAPSecretAtr string
LDAPRoleAtr string
S3Access string
S3Secret string
S3Region string
S3Bucket string
S3Endpoint string
S3DisableSSlVerfiy bool
S3Debug bool
CacheDisable bool
CacheTTL int
CachePrune int
Dir string
LDAPServerURL string
LDAPBindDN string
LDAPPassword string
LDAPQueryBase string
LDAPObjClasses string
LDAPAccessAtr string
LDAPSecretAtr string
LDAPRoleAtr string
CacheDisable bool
CacheTTL int
CachePrune int
}
func New(o *Opts) (IAMService, error) {
@@ -80,20 +64,12 @@ func New(o *Opts) (IAMService, error) {
switch {
case o.Dir != "":
svc, err = NewInternal(o.Dir)
fmt.Printf("initializing internal IAM with %q\n", o.Dir)
case o.LDAPServerURL != "":
svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
o.LDAPObjClasses)
fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
case o.S3Endpoint != "":
svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
fmt.Printf("initializing S3 IAM with '%v/%v'\n",
o.S3Endpoint, o.S3Bucket)
default:
// if no iam options selected, default to the single user mode
fmt.Println("No IAM service configured, enabling single account mode")
return IAMServiceSingle{}, nil
}

View File

@@ -130,7 +130,7 @@ func (c *IAMCache) CreateAccount(account Account) error {
acct := Account{
Access: strings.Clone(account.Access),
Secret: strings.Clone(account.Secret),
Role: Role(strings.Clone(string(account.Role))),
Role: strings.Clone(account.Role),
}
c.iamcache.set(acct.Access, acct)

View File

@@ -270,7 +270,7 @@ func (s *IAMServiceInternal) storeIAM(update UpdateAcctFunc) error {
// reset retries on successful read
retries = 0
err = os.Remove(fname)
err = os.Remove(iamFile)
if errors.Is(err, fs.ErrNotExist) {
// racing with someone else updating
// keep retrying after backoff

View File

@@ -46,7 +46,7 @@ func (ld *LdapIAMService) CreateAccount(account Account) error {
userEntry.Attribute("objectClass", ld.objClasses)
userEntry.Attribute(ld.accessAtr, []string{account.Access})
userEntry.Attribute(ld.secretAtr, []string{account.Secret})
userEntry.Attribute(ld.roleAtr, []string{string(account.Role)})
userEntry.Attribute(ld.roleAtr, []string{account.Role})
err := ld.conn.Add(userEntry)
if err != nil {
@@ -78,7 +78,7 @@ func (ld *LdapIAMService) GetUserAccount(access string) (Account, error) {
return Account{
Access: entry.GetAttributeValue(ld.accessAtr),
Secret: entry.GetAttributeValue(ld.secretAtr),
Role: Role(entry.GetAttributeValue(ld.roleAtr)),
Role: entry.GetAttributeValue(ld.roleAtr),
}, nil
}
@@ -120,7 +120,7 @@ func (ld *LdapIAMService) ListUserAccounts() ([]Account, error) {
result = append(result, Account{
Access: el.GetAttributeValue(ld.accessAtr),
Secret: el.GetAttributeValue(ld.secretAtr),
Role: Role(el.GetAttributeValue(ld.roleAtr)),
Role: el.GetAttributeValue(ld.roleAtr),
})
}

View File

@@ -1,263 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sort"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
)
// IAMServiceS3 stores user accounts in an S3 object
// The endpoint, credentials, bucket, and region are provided
// from cli configuration.
// The object format and name is the same as the internal IAM service:
// coming from iAMConfig and iamFile in iam_internal.
type IAMServiceS3 struct {
access string
secret string
region string
bucket string
endpoint string
sslSkipVerify bool
debug bool
client *s3.Client
}
var _ IAMService = &IAMServiceS3{}
func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
if access == "" {
return nil, fmt.Errorf("must provide s3 IAM service access key")
}
if secret == "" {
return nil, fmt.Errorf("must provide s3 IAM service secret key")
}
if region == "" {
return nil, fmt.Errorf("must provide s3 IAM service region")
}
if bucket == "" {
return nil, fmt.Errorf("must provide s3 IAM service bucket")
}
if endpoint == "" {
return nil, fmt.Errorf("must provide s3 IAM service endpoint")
}
i := &IAMServiceS3{
access: access,
secret: secret,
region: region,
bucket: bucket,
endpoint: endpoint,
sslSkipVerify: sslSkipVerify,
debug: debug,
}
cfg, err := i.getConfig()
if err != nil {
return nil, fmt.Errorf("init s3 IAM: %v", err)
}
i.client = s3.NewFromConfig(cfg)
return i, nil
}
func (s *IAMServiceS3) CreateAccount(account Account) error {
conf, err := s.getAccounts()
if err != nil {
return err
}
_, ok := conf.AccessAccounts[account.Access]
if ok {
return fmt.Errorf("account already exists")
}
conf.AccessAccounts[account.Access] = account
return s.storeAccts(conf)
}
func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
conf, err := s.getAccounts()
if err != nil {
return Account{}, err
}
acct, ok := conf.AccessAccounts[access]
if !ok {
return Account{}, ErrNoSuchUser
}
return acct, nil
}
func (s *IAMServiceS3) DeleteUserAccount(access string) error {
conf, err := s.getAccounts()
if err != nil {
return err
}
_, ok := conf.AccessAccounts[access]
if !ok {
return fmt.Errorf("account does not exist")
}
delete(conf.AccessAccounts, access)
return s.storeAccts(conf)
}
func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
conf, err := s.getAccounts()
if err != nil {
return nil, err
}
keys := make([]string, 0, len(conf.AccessAccounts))
for k := range conf.AccessAccounts {
keys = append(keys, k)
}
sort.Strings(keys)
var accs []Account
for _, k := range keys {
accs = append(accs, Account{
Access: k,
Secret: conf.AccessAccounts[k].Secret,
Role: conf.AccessAccounts[k].Role,
UserID: conf.AccessAccounts[k].UserID,
GroupID: conf.AccessAccounts[k].GroupID,
ProjectID: conf.AccessAccounts[k].ProjectID,
})
}
return accs, nil
}
// ResolveEndpoint is used for on prem or non-aws endpoints
func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
URL: s.endpoint,
SigningRegion: s.region,
HostnameImmutable: true,
}, nil
}
func (s *IAMServiceS3) Shutdown() error {
return nil
}
func (s *IAMServiceS3) getConfig() (aws.Config, error) {
creds := credentials.NewStaticCredentialsProvider(s.access, s.secret, "")
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
}
client := &http.Client{Transport: tr}
opts := []func(*config.LoadOptions) error{
config.WithRegion(s.region),
config.WithCredentialsProvider(creds),
config.WithHTTPClient(client),
}
if s.endpoint != "" {
opts = append(opts,
config.WithEndpointResolverWithOptions(s))
}
if s.debug {
opts = append(opts,
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
}
return config.LoadDefaultConfig(context.Background(), opts...)
}
func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
obj := iamFile
out, err := s.client.GetObject(context.Background(), &s3.GetObjectInput{
Bucket: &s.bucket,
Key: &obj,
})
if err != nil {
// if the error is object not exists,
// init empty accounts stuct and return that
var nsk *types.NoSuchKey
if errors.As(err, &nsk) {
return iAMConfig{}, nil
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
if apiErr.ErrorCode() == "NotFound" {
return iAMConfig{}, nil
}
}
// all other errors, return the error
return iAMConfig{}, fmt.Errorf("get %v: %w", obj, err)
}
defer out.Body.Close()
b, err := io.ReadAll(out.Body)
if err != nil {
return iAMConfig{}, fmt.Errorf("read %v: %w", obj, err)
}
conf, err := parseIAM(b)
if err != nil {
return iAMConfig{}, fmt.Errorf("parse iam data: %w", err)
}
return conf, nil
}
func (s *IAMServiceS3) storeAccts(conf iAMConfig) error {
b, err := json.Marshal(conf)
if err != nil {
return fmt.Errorf("failed to serialize iam: %w", err)
}
obj := iamFile
uploader := manager.NewUploader(s.client)
upinfo := &s3.PutObjectInput{
Body: bytes.NewReader(b),
Bucket: &s.bucket,
Key: &obj,
}
_, err = uploader.Upload(context.Background(), upinfo)
if err != nil {
return fmt.Errorf("store accounts in %v: %w", iamFile, err)
}
return nil
}

View File

@@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@@ -1,3 +0,0 @@
AWS SDK for Go
Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
Copyright 2014-2015 Stripe, Inc.

View File

@@ -1,11 +0,0 @@
# AWS SDK Go v2
This directory contains code from the [AWS SDK Go v2](https://github.com/aws/aws-sdk-go-v2) repository, modified in accordance with the Apache 2.0 License.
## Description
The AWS SDK Go v2 is a collection of libraries and tools that enable developers to build applications that integrate with various AWS services. This directory and below contains modified code from the original repository, tailored to suit versitygw specific requirements.
## License
The code in this directory is licensed under the Apache 2.0 License. Please refer to the [LICENSE](./LICENSE) file for more information.

View File

@@ -1,45 +0,0 @@
package auth
import (
"github.com/aws/smithy-go/auth"
smithyhttp "github.com/aws/smithy-go/transport/http"
)
// HTTPAuthScheme is the SDK's internal implementation of smithyhttp.AuthScheme
// for pre-existing implementations where the signer was added to client
// config. SDK clients will key off of this type and ensure per-operation
// updates to those signers persist on the scheme itself.
type HTTPAuthScheme struct {
schemeID string
signer smithyhttp.Signer
}
var _ smithyhttp.AuthScheme = (*HTTPAuthScheme)(nil)
// NewHTTPAuthScheme returns an auth scheme instance with the given config.
func NewHTTPAuthScheme(schemeID string, signer smithyhttp.Signer) *HTTPAuthScheme {
return &HTTPAuthScheme{
schemeID: schemeID,
signer: signer,
}
}
// SchemeID identifies the auth scheme.
func (s *HTTPAuthScheme) SchemeID() string {
return s.schemeID
}
// IdentityResolver gets the identity resolver for the auth scheme.
func (s *HTTPAuthScheme) IdentityResolver(o auth.IdentityResolverOptions) auth.IdentityResolver {
return o.GetIdentityResolver(s.schemeID)
}
// Signer gets the signer for the auth scheme.
func (s *HTTPAuthScheme) Signer() smithyhttp.Signer {
return s.signer
}
// WithSigner returns a new instance of the auth scheme with the updated signer.
func (s *HTTPAuthScheme) WithSigner(signer smithyhttp.Signer) *HTTPAuthScheme {
return NewHTTPAuthScheme(s.schemeID, signer)
}

View File

@@ -1,191 +0,0 @@
package auth
import (
"context"
"fmt"
smithy "github.com/aws/smithy-go"
"github.com/aws/smithy-go/middleware"
)
// SigV4 is a constant representing
// Authentication Scheme Signature Version 4
const SigV4 = "sigv4"
// SigV4A is a constant representing
// Authentication Scheme Signature Version 4A
const SigV4A = "sigv4a"
// SigV4S3Express identifies the S3 S3Express auth scheme.
const SigV4S3Express = "sigv4-s3express"
// None is a constant representing the
// None Authentication Scheme
const None = "none"
// SupportedSchemes is a data structure
// that indicates the list of supported AWS
// authentication schemes
var SupportedSchemes = map[string]bool{
SigV4: true,
SigV4A: true,
SigV4S3Express: true,
None: true,
}
// AuthenticationScheme is a representation of
// AWS authentication schemes
type AuthenticationScheme interface {
isAuthenticationScheme()
}
// AuthenticationSchemeV4 is a AWS SigV4 representation
type AuthenticationSchemeV4 struct {
Name string
SigningName *string
SigningRegion *string
DisableDoubleEncoding *bool
}
func (a *AuthenticationSchemeV4) isAuthenticationScheme() {}
// AuthenticationSchemeV4A is a AWS SigV4A representation
type AuthenticationSchemeV4A struct {
Name string
SigningName *string
SigningRegionSet []string
DisableDoubleEncoding *bool
}
func (a *AuthenticationSchemeV4A) isAuthenticationScheme() {}
// AuthenticationSchemeNone is a representation for the none auth scheme
type AuthenticationSchemeNone struct{}
func (a *AuthenticationSchemeNone) isAuthenticationScheme() {}
// NoAuthenticationSchemesFoundError is used in signaling
// that no authentication schemes have been specified.
type NoAuthenticationSchemesFoundError struct{}
func (e *NoAuthenticationSchemesFoundError) Error() string {
return fmt.Sprint("No authentication schemes specified.")
}
// UnSupportedAuthenticationSchemeSpecifiedError is used in
// signaling that only unsupported authentication schemes
// were specified.
type UnSupportedAuthenticationSchemeSpecifiedError struct {
UnsupportedSchemes []string
}
func (e *UnSupportedAuthenticationSchemeSpecifiedError) Error() string {
return fmt.Sprint("Unsupported authentication scheme specified.")
}
// GetAuthenticationSchemes extracts the relevant authentication scheme data
// into a custom strongly typed Go data structure.
func GetAuthenticationSchemes(p *smithy.Properties) ([]AuthenticationScheme, error) {
var result []AuthenticationScheme
if !p.Has("authSchemes") {
return nil, &NoAuthenticationSchemesFoundError{}
}
authSchemes, _ := p.Get("authSchemes").([]interface{})
var unsupportedSchemes []string
for _, scheme := range authSchemes {
authScheme, _ := scheme.(map[string]interface{})
version := authScheme["name"].(string)
switch version {
case SigV4, SigV4S3Express:
v4Scheme := AuthenticationSchemeV4{
Name: version,
SigningName: getSigningName(authScheme),
SigningRegion: getSigningRegion(authScheme),
DisableDoubleEncoding: getDisableDoubleEncoding(authScheme),
}
result = append(result, AuthenticationScheme(&v4Scheme))
case SigV4A:
v4aScheme := AuthenticationSchemeV4A{
Name: SigV4A,
SigningName: getSigningName(authScheme),
SigningRegionSet: getSigningRegionSet(authScheme),
DisableDoubleEncoding: getDisableDoubleEncoding(authScheme),
}
result = append(result, AuthenticationScheme(&v4aScheme))
case None:
noneScheme := AuthenticationSchemeNone{}
result = append(result, AuthenticationScheme(&noneScheme))
default:
unsupportedSchemes = append(unsupportedSchemes, authScheme["name"].(string))
continue
}
}
if len(result) == 0 {
return nil, &UnSupportedAuthenticationSchemeSpecifiedError{
UnsupportedSchemes: unsupportedSchemes,
}
}
return result, nil
}
type disableDoubleEncoding struct{}
// SetDisableDoubleEncoding sets or modifies the disable double encoding option
// on the context.
//
// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
// to clear all stack values.
func SetDisableDoubleEncoding(ctx context.Context, value bool) context.Context {
return middleware.WithStackValue(ctx, disableDoubleEncoding{}, value)
}
// GetDisableDoubleEncoding retrieves the disable double encoding option
// from the context.
//
// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
// to clear all stack values.
func GetDisableDoubleEncoding(ctx context.Context) (value bool, ok bool) {
value, ok = middleware.GetStackValue(ctx, disableDoubleEncoding{}).(bool)
return value, ok
}
func getSigningName(authScheme map[string]interface{}) *string {
signingName, ok := authScheme["signingName"].(string)
if !ok || signingName == "" {
return nil
}
return &signingName
}
func getSigningRegionSet(authScheme map[string]interface{}) []string {
untypedSigningRegionSet, ok := authScheme["signingRegionSet"].([]interface{})
if !ok {
return nil
}
signingRegionSet := []string{}
for _, item := range untypedSigningRegionSet {
signingRegionSet = append(signingRegionSet, item.(string))
}
return signingRegionSet
}
func getSigningRegion(authScheme map[string]interface{}) *string {
signingRegion, ok := authScheme["signingRegion"].(string)
if !ok || signingRegion == "" {
return nil
}
return &signingRegion
}
func getDisableDoubleEncoding(authScheme map[string]interface{}) *bool {
disableDoubleEncoding, ok := authScheme["disableDoubleEncoding"].(bool)
if !ok {
return nil
}
return &disableDoubleEncoding
}

View File

@@ -1,101 +0,0 @@
package auth
import (
"testing"
smithy "github.com/aws/smithy-go"
)
func TestV4(t *testing.T) {
propsV4 := smithy.Properties{}
propsV4.Set("authSchemes", interface{}([]interface{}{
map[string]interface{}{
"disableDoubleEncoding": true,
"name": "sigv4",
"signingName": "s3",
"signingRegion": "us-west-2",
},
}))
result, err := GetAuthenticationSchemes(&propsV4)
if err != nil {
t.Fatalf("Did not expect error, got %v", err)
}
_, ok := result[0].(AuthenticationScheme)
if !ok {
t.Fatalf("Did not get expected AuthenticationScheme. %v", result[0])
}
v4Scheme, ok := result[0].(*AuthenticationSchemeV4)
if !ok {
t.Fatalf("Did not get expected AuthenticationSchemeV4. %v", result[0])
}
if v4Scheme.Name != "sigv4" {
t.Fatalf("Did not get expected AuthenticationSchemeV4 signer version name")
}
}
func TestV4A(t *testing.T) {
propsV4A := smithy.Properties{}
propsV4A.Set("authSchemes", []interface{}{
map[string]interface{}{
"disableDoubleEncoding": true,
"name": "sigv4a",
"signingName": "s3",
"signingRegionSet": []string{"*"},
},
})
result, err := GetAuthenticationSchemes(&propsV4A)
if err != nil {
t.Fatalf("Did not expect error, got %v", err)
}
_, ok := result[0].(AuthenticationScheme)
if !ok {
t.Fatalf("Did not get expected AuthenticationScheme. %v", result[0])
}
v4AScheme, ok := result[0].(*AuthenticationSchemeV4A)
if !ok {
t.Fatalf("Did not get expected AuthenticationSchemeV4A. %v", result[0])
}
if v4AScheme.Name != "sigv4a" {
t.Fatalf("Did not get expected AuthenticationSchemeV4A signer version name")
}
}
func TestV4S3Express(t *testing.T) {
props := smithy.Properties{}
props.Set("authSchemes", []interface{}{
map[string]interface{}{
"name": SigV4S3Express,
"signingName": "s3",
"signingRegion": "us-east-1",
"disableDoubleEncoding": true,
},
})
result, err := GetAuthenticationSchemes(&props)
if err != nil {
t.Fatalf("Did not expect error, got %v", err)
}
scheme, ok := result[0].(*AuthenticationSchemeV4)
if !ok {
t.Fatalf("Did not get expected AuthenticationSchemeV4. %v", result[0])
}
if scheme.Name != SigV4S3Express {
t.Fatalf("expected %s, got %s", SigV4S3Express, scheme.Name)
}
}

View File

@@ -1,43 +0,0 @@
package smithy
import (
"context"
"fmt"
"time"
"github.com/aws/smithy-go"
"github.com/aws/smithy-go/auth"
"github.com/aws/smithy-go/auth/bearer"
)
// BearerTokenAdapter adapts smithy bearer.Token to smithy auth.Identity.
type BearerTokenAdapter struct {
Token bearer.Token
}
var _ auth.Identity = (*BearerTokenAdapter)(nil)
// Expiration returns the time of expiration for the token.
func (v *BearerTokenAdapter) Expiration() time.Time {
return v.Token.Expires
}
// BearerTokenProviderAdapter adapts smithy bearer.TokenProvider to smithy
// auth.IdentityResolver.
type BearerTokenProviderAdapter struct {
Provider bearer.TokenProvider
}
var _ (auth.IdentityResolver) = (*BearerTokenProviderAdapter)(nil)
// GetIdentity retrieves a bearer token using the underlying provider.
func (v *BearerTokenProviderAdapter) GetIdentity(ctx context.Context, _ smithy.Properties) (
auth.Identity, error,
) {
token, err := v.Provider.RetrieveBearerToken(ctx)
if err != nil {
return nil, fmt.Errorf("get token: %w", err)
}
return &BearerTokenAdapter{Token: token}, nil
}

View File

@@ -1,35 +0,0 @@
package smithy
import (
"context"
"fmt"
"github.com/aws/smithy-go"
"github.com/aws/smithy-go/auth"
"github.com/aws/smithy-go/auth/bearer"
smithyhttp "github.com/aws/smithy-go/transport/http"
)
// BearerTokenSignerAdapter adapts smithy bearer.Signer to smithy http
// auth.Signer.
type BearerTokenSignerAdapter struct {
Signer bearer.Signer
}
var _ (smithyhttp.Signer) = (*BearerTokenSignerAdapter)(nil)
// SignRequest signs the request with the provided bearer token.
func (v *BearerTokenSignerAdapter) SignRequest(ctx context.Context, r *smithyhttp.Request, identity auth.Identity, _ smithy.Properties) error {
ca, ok := identity.(*BearerTokenAdapter)
if !ok {
return fmt.Errorf("unexpected identity type: %T", identity)
}
signed, err := v.Signer.SignWithBearerToken(ctx, ca.Token, r)
if err != nil {
return fmt.Errorf("sign request: %w", err)
}
*r = *signed.(*smithyhttp.Request)
return nil
}

View File

@@ -1,46 +0,0 @@
package smithy
import (
"context"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/smithy-go"
"github.com/aws/smithy-go/auth"
)
// CredentialsAdapter adapts aws.Credentials to auth.Identity.
type CredentialsAdapter struct {
Credentials aws.Credentials
}
var _ auth.Identity = (*CredentialsAdapter)(nil)
// Expiration returns the time of expiration for the credentials.
func (v *CredentialsAdapter) Expiration() time.Time {
return v.Credentials.Expires
}
// CredentialsProviderAdapter adapts aws.CredentialsProvider to auth.IdentityResolver.
type CredentialsProviderAdapter struct {
Provider aws.CredentialsProvider
}
var _ (auth.IdentityResolver) = (*CredentialsProviderAdapter)(nil)
// GetIdentity retrieves AWS credentials using the underlying provider.
func (v *CredentialsProviderAdapter) GetIdentity(ctx context.Context, _ smithy.Properties) (
auth.Identity, error,
) {
if v.Provider == nil {
return &CredentialsAdapter{Credentials: aws.Credentials{}}, nil
}
creds, err := v.Provider.Retrieve(ctx)
if err != nil {
return nil, fmt.Errorf("get credentials: %w", err)
}
return &CredentialsAdapter{Credentials: creds}, nil
}

View File

@@ -1,2 +0,0 @@
// Package smithy adapts concrete AWS auth and signing types to the generic smithy versions.
package smithy

View File

@@ -1,53 +0,0 @@
package smithy
import (
"context"
"fmt"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/smithy-go"
"github.com/aws/smithy-go/auth"
"github.com/aws/smithy-go/logging"
smithyhttp "github.com/aws/smithy-go/transport/http"
"github.com/versity/versitygw/aws/internal/sdk"
)
// V4SignerAdapter adapts v4.HTTPSigner to smithy http.Signer.
type V4SignerAdapter struct {
Signer v4.HTTPSigner
Logger logging.Logger
LogSigning bool
}
var _ (smithyhttp.Signer) = (*V4SignerAdapter)(nil)
// SignRequest signs the request with the provided identity.
func (v *V4SignerAdapter) SignRequest(ctx context.Context, r *smithyhttp.Request, identity auth.Identity, props smithy.Properties) error {
ca, ok := identity.(*CredentialsAdapter)
if !ok {
return fmt.Errorf("unexpected identity type: %T", identity)
}
name, ok := smithyhttp.GetSigV4SigningName(&props)
if !ok {
return fmt.Errorf("sigv4 signing name is required")
}
region, ok := smithyhttp.GetSigV4SigningRegion(&props)
if !ok {
return fmt.Errorf("sigv4 signing region is required")
}
hash := v4.GetPayloadHash(ctx)
err := v.Signer.SignHTTP(ctx, ca.Credentials, r.Request, hash, name, region, sdk.NowTime(), func(o *v4.SignerOptions) {
o.DisableURIPathEscaping, _ = smithyhttp.GetDisableDoubleEncoding(&props)
o.Logger = v.Logger
o.LogSigning = v.LogSigning
})
if err != nil {
return fmt.Errorf("sign http: %w", err)
}
return nil
}

View File

@@ -1,222 +0,0 @@
package awstesting
import (
"encoding/json"
"encoding/xml"
"fmt"
"net/url"
"reflect"
"regexp"
"sort"
"strings"
"testing"
)
// Match is a testing helper to test for testing error by comparing expected
// with a regular expression.
func Match(t *testing.T, regex, expected string) {
t.Helper()
if !regexp.MustCompile(regex).Match([]byte(expected)) {
t.Errorf("%q\n\tdoes not match /%s/", expected, regex)
}
}
// AssertURL verifies the expected URL is matches the actual.
func AssertURL(t *testing.T, expect, actual string, msgAndArgs ...interface{}) bool {
t.Helper()
expectURL, err := url.Parse(expect)
if err != nil {
t.Errorf(errMsg("unable to parse expected URL", err, msgAndArgs))
return false
}
actualURL, err := url.Parse(actual)
if err != nil {
t.Errorf(errMsg("unable to parse actual URL", err, msgAndArgs))
return false
}
equal(t, expectURL.Host, actualURL.Host, msgAndArgs...)
equal(t, expectURL.Scheme, actualURL.Scheme, msgAndArgs...)
equal(t, expectURL.Path, actualURL.Path, msgAndArgs...)
return AssertQuery(t, expectURL.Query().Encode(), actualURL.Query().Encode(), msgAndArgs...)
}
var queryMapKey = regexp.MustCompile(`(.*?)\.[0-9]+\.key`)
// AssertQuery verifies the expect HTTP query string matches the actual.
func AssertQuery(t *testing.T, expect, actual string, msgAndArgs ...interface{}) bool {
t.Helper()
expectQ, err := url.ParseQuery(expect)
if err != nil {
t.Errorf(errMsg("unable to parse expected Query", err, msgAndArgs))
return false
}
actualQ, err := url.ParseQuery(actual)
if err != nil {
t.Errorf(errMsg("unable to parse actual Query", err, msgAndArgs))
return false
}
// Make sure the keys are the same
if !equal(t, queryValueKeys(expectQ), queryValueKeys(actualQ), msgAndArgs...) {
return false
}
keys := map[string][]string{}
for key, v := range expectQ {
if queryMapKey.Match([]byte(key)) {
submatch := queryMapKey.FindStringSubmatch(key)
keys[submatch[1]] = append(keys[submatch[1]], v...)
}
}
for k, v := range keys {
// clear all keys that have prefix
for key := range expectQ {
if strings.HasPrefix(key, k) {
delete(expectQ, key)
}
}
sort.Strings(v)
for i, value := range v {
expectQ[fmt.Sprintf("%s.%d.key", k, i+1)] = []string{value}
}
}
for k, expectQVals := range expectQ {
sort.Strings(expectQVals)
actualQVals := actualQ[k]
sort.Strings(actualQVals)
if !equal(t, expectQVals, actualQVals, msgAndArgs...) {
return false
}
}
return true
}
// AssertJSON verifies that the expect json string matches the actual.
func AssertJSON(t *testing.T, expect, actual string, msgAndArgs ...interface{}) bool {
t.Helper()
expectVal := map[string]interface{}{}
if err := json.Unmarshal([]byte(expect), &expectVal); err != nil {
t.Errorf(errMsg("unable to parse expected JSON", err, msgAndArgs...))
return false
}
actualVal := map[string]interface{}{}
if err := json.Unmarshal([]byte(actual), &actualVal); err != nil {
t.Errorf(errMsg("unable to parse actual JSON", err, msgAndArgs...))
return false
}
return equal(t, expectVal, actualVal, msgAndArgs...)
}
// AssertXML verifies that the expect xml string matches the actual.
func AssertXML(t *testing.T, expect, actual string, container interface{}, msgAndArgs ...interface{}) bool {
expectVal := container
if err := xml.Unmarshal([]byte(expect), &expectVal); err != nil {
t.Errorf(errMsg("unable to parse expected XML", err, msgAndArgs...))
}
actualVal := container
if err := xml.Unmarshal([]byte(actual), &actualVal); err != nil {
t.Errorf(errMsg("unable to parse actual XML", err, msgAndArgs...))
}
return equal(t, expectVal, actualVal, msgAndArgs...)
}
// DidPanic returns if the function paniced and returns true if the function paniced.
func DidPanic(fn func()) (bool, interface{}) {
var paniced bool
var msg interface{}
func() {
defer func() {
if msg = recover(); msg != nil {
paniced = true
}
}()
fn()
}()
return paniced, msg
}
// objectsAreEqual determines if two objects are considered equal.
//
// This function does no assertion of any kind.
//
// Based on github.com/stretchr/testify/assert.ObjectsAreEqual
// Copied locally to prevent non-test build dependencies on testify
func objectsAreEqual(expected, actual interface{}) bool {
if expected == nil || actual == nil {
return expected == actual
}
return reflect.DeepEqual(expected, actual)
}
// Equal asserts that two objects are equal.
//
// assert.Equal(t, 123, 123, "123 and 123 should be equal")
//
// Returns whether the assertion was successful (true) or not (false).
//
// Based on github.com/stretchr/testify/assert.Equal
// Copied locally to prevent non-test build dependencies on testify
func equal(t *testing.T, expected, actual interface{}, msgAndArgs ...interface{}) bool {
t.Helper()
if !objectsAreEqual(expected, actual) {
t.Errorf("%s\n%s", messageFromMsgAndArgs(msgAndArgs),
SprintExpectActual(expected, actual))
return false
}
return true
}
func errMsg(baseMsg string, err error, msgAndArgs ...interface{}) string {
message := messageFromMsgAndArgs(msgAndArgs)
if message != "" {
message += ", "
}
return fmt.Sprintf("%s%s, %v", message, baseMsg, err)
}
// Based on github.com/stretchr/testify/assert.messageFromMsgAndArgs
// Copied locally to prevent non-test build dependencies on testify
func messageFromMsgAndArgs(msgAndArgs []interface{}) string {
if len(msgAndArgs) == 0 || msgAndArgs == nil {
return ""
}
if len(msgAndArgs) == 1 {
return msgAndArgs[0].(string)
}
if len(msgAndArgs) > 1 {
return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...)
}
return ""
}
func queryValueKeys(v url.Values) []string {
keys := make([]string, 0, len(v))
for k := range v {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// SprintExpectActual returns a string for test failure cases when the actual
// value is not the same as the expected.
func SprintExpectActual(expect, actual interface{}) string {
return fmt.Sprintf("expect: %+v\nactual: %+v\n", expect, actual)
}

View File

@@ -1,89 +0,0 @@
package awstesting_test
import (
"encoding/xml"
"testing"
"github.com/versity/versitygw/aws/internal/awstesting"
)
func TestAssertJSON(t *testing.T) {
cases := []struct {
e, a string
asserts bool
}{
{
e: `{"RecursiveStruct":{"RecursiveMap":{"foo":{"NoRecurse":"foo"},"bar":{"NoRecurse":"bar"}}}}`,
a: `{"RecursiveStruct":{"RecursiveMap":{"bar":{"NoRecurse":"bar"},"foo":{"NoRecurse":"foo"}}}}`,
asserts: true,
},
}
for i, c := range cases {
mockT := &testing.T{}
if awstesting.AssertJSON(mockT, c.e, c.a) != c.asserts {
t.Error("Assert JSON result was not expected.", i)
}
}
}
func TestAssertXML(t *testing.T) {
cases := []struct {
e, a string
asserts bool
container struct {
XMLName xml.Name `xml:"OperationRequest"`
NS string `xml:"xmlns,attr"`
RecursiveStruct struct {
RecursiveMap struct {
Entries []struct {
XMLName xml.Name `xml:"entries"`
Key string `xml:"key"`
Value struct {
XMLName xml.Name `xml:"value"`
NoRecurse string
}
}
}
}
}
}{
{
e: `<OperationRequest xmlns="https://foo/"><RecursiveStruct xmlns="https://foo/"><RecursiveMap xmlns="https://foo/"><entry xmlns="https://foo/"><key xmlns="https://foo/">foo</key><value xmlns="https://foo/"><NoRecurse xmlns="https://foo/">foo</NoRecurse></value></entry><entry xmlns="https://foo/"><key xmlns="https://foo/">bar</key><value xmlns="https://foo/"><NoRecurse xmlns="https://foo/">bar</NoRecurse></value></entry></RecursiveMap></RecursiveStruct></OperationRequest>`,
a: `<OperationRequest xmlns="https://foo/"><RecursiveStruct xmlns="https://foo/"><RecursiveMap xmlns="https://foo/"><entry xmlns="https://foo/"><key xmlns="https://foo/">bar</key><value xmlns="https://foo/"><NoRecurse xmlns="https://foo/">bar</NoRecurse></value></entry><entry xmlns="https://foo/"><key xmlns="https://foo/">foo</key><value xmlns="https://foo/"><NoRecurse xmlns="https://foo/">foo</NoRecurse></value></entry></RecursiveMap></RecursiveStruct></OperationRequest>`,
asserts: true,
},
}
for i, c := range cases {
// mockT := &testing.T{}
if awstesting.AssertXML(t, c.e, c.a, c.container) != c.asserts {
t.Error("Assert XML result was not expected.", i)
}
}
}
func TestAssertQuery(t *testing.T) {
cases := []struct {
e, a string
asserts bool
}{
{
e: `Action=OperationName&Version=2014-01-01&Foo=val1&Bar=val2`,
a: `Action=OperationName&Version=2014-01-01&Foo=val2&Bar=val3`,
asserts: false,
},
{
e: `Action=OperationName&Version=2014-01-01&Foo=val1&Bar=val2`,
a: `Action=OperationName&Version=2014-01-01&Foo=val1&Bar=val2`,
asserts: true,
},
}
for i, c := range cases {
mockT := &testing.T{}
if awstesting.AssertQuery(mockT, c.e, c.a) != c.asserts {
t.Error("Assert Query result was not expected.", i)
}
}
}

View File

@@ -1,291 +0,0 @@
package awstesting
import (
"bytes"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"io/ioutil"
"math/big"
"net"
"net/http"
"net/http/httptest"
"os"
"strings"
"time"
)
var (
// TLSBundleCA is the CA PEM
TLSBundleCA []byte
// TLSBundleCert is the Server PEM
TLSBundleCert []byte
// TLSBundleKey is the Server private key PEM
TLSBundleKey []byte
// ClientTLSCert is the Client PEM
ClientTLSCert []byte
// ClientTLSKey is the Client private key PEM
ClientTLSKey []byte
)
func init() {
caPEM, _, caCert, caPrivKey, err := generateRootCA()
if err != nil {
panic("failed to generate testing root CA, " + err.Error())
}
TLSBundleCA = caPEM
serverCertPEM, serverCertPrivKeyPEM, err := generateLocalCert(caCert, caPrivKey)
if err != nil {
panic("failed to generate testing server cert, " + err.Error())
}
TLSBundleCert = serverCertPEM
TLSBundleKey = serverCertPrivKeyPEM
clientCertPEM, clientCertPrivKeyPEM, err := generateLocalCert(caCert, caPrivKey)
if err != nil {
panic("failed to generate testing client cert, " + err.Error())
}
ClientTLSCert = clientCertPEM
ClientTLSKey = clientCertPrivKeyPEM
}
func generateRootCA() (
caPEM, caPrivKeyPEM []byte, caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, err error,
) {
caCert = &x509.Certificate{
SerialNumber: big.NewInt(42),
Subject: pkix.Name{
Country: []string{"US"},
Organization: []string{"AWS SDK for Go Test Certificate"},
CommonName: "Test Root CA",
},
NotBefore: time.Now().Add(-time.Minute),
NotAfter: time.Now().AddDate(1, 0, 0),
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{
x509.ExtKeyUsageClientAuth,
x509.ExtKeyUsageServerAuth,
},
BasicConstraintsValid: true,
IsCA: true,
}
// Create CA private and public key
caPrivKey, err = rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("failed generate CA RSA key, %w", err)
}
// Create CA certificate
caBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, &caPrivKey.PublicKey, caPrivKey)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("failed generate CA certificate, %w", err)
}
// PEM encode CA certificate and private key
var caPEMBuf bytes.Buffer
pem.Encode(&caPEMBuf, &pem.Block{
Type: "CERTIFICATE",
Bytes: caBytes,
})
var caPrivKeyPEMBuf bytes.Buffer
pem.Encode(&caPrivKeyPEMBuf, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(caPrivKey),
})
return caPEMBuf.Bytes(), caPrivKeyPEMBuf.Bytes(), caCert, caPrivKey, nil
}
func generateLocalCert(parentCert *x509.Certificate, parentPrivKey *rsa.PrivateKey) (
certPEM, certPrivKeyPEM []byte, err error,
) {
cert := &x509.Certificate{
SerialNumber: big.NewInt(42),
Subject: pkix.Name{
Country: []string{"US"},
Organization: []string{"AWS SDK for Go Test Certificate"},
CommonName: "Test Root CA",
},
IPAddresses: []net.IP{
net.IPv4(127, 0, 0, 1),
net.IPv6loopback,
},
NotBefore: time.Now().Add(-time.Minute),
NotAfter: time.Now().AddDate(1, 0, 0),
ExtKeyUsage: []x509.ExtKeyUsage{
x509.ExtKeyUsageClientAuth,
x509.ExtKeyUsageServerAuth,
},
KeyUsage: x509.KeyUsageDigitalSignature,
}
// Create server private and public key
certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
return nil, nil, fmt.Errorf("failed to generate server RSA private key, %w", err)
}
// Create server certificate
certBytes, err := x509.CreateCertificate(rand.Reader, cert, parentCert, &certPrivKey.PublicKey, parentPrivKey)
if err != nil {
return nil, nil, fmt.Errorf("failed to generate server certificate, %w", err)
}
// PEM encode certificate and private key
var certPEMBuf bytes.Buffer
pem.Encode(&certPEMBuf, &pem.Block{
Type: "CERTIFICATE",
Bytes: certBytes,
})
var certPrivKeyPEMBuf bytes.Buffer
pem.Encode(&certPrivKeyPEMBuf, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
})
return certPEMBuf.Bytes(), certPrivKeyPEMBuf.Bytes(), nil
}
// NewTLSClientCertServer creates a new HTTP test server initialize to require
// HTTP clients authenticate with TLS client certificates.
func NewTLSClientCertServer(handler http.Handler) (*httptest.Server, error) {
server := httptest.NewUnstartedServer(handler)
if server.TLS == nil {
server.TLS = &tls.Config{}
}
server.TLS.ClientAuth = tls.RequireAndVerifyClientCert
if server.TLS.ClientCAs == nil {
server.TLS.ClientCAs = x509.NewCertPool()
}
certPem := append(ClientTLSCert, ClientTLSKey...)
if ok := server.TLS.ClientCAs.AppendCertsFromPEM(certPem); !ok {
return nil, fmt.Errorf("failed to append client certs")
}
return server, nil
}
// CreateClientTLSCertFiles returns a set of temporary files for the client
// certificate and key files.
func CreateClientTLSCertFiles() (cert, key string, err error) {
cert, err = createTmpFile(ClientTLSCert)
if err != nil {
return "", "", err
}
key, err = createTmpFile(ClientTLSKey)
if err != nil {
return "", "", err
}
return cert, key, nil
}
func availableLocalAddr(ip string) (v string, err error) {
l, err := net.Listen("tcp", ip+":0")
if err != nil {
return "", err
}
defer func() {
closeErr := l.Close()
if err == nil {
err = closeErr
} else if closeErr != nil {
err = fmt.Errorf("ip listener close error: %v, original error: %w", closeErr, err)
}
}()
return l.Addr().String(), nil
}
// CreateTLSServer will create the TLS server on an open port using the
// certificate and key. The address will be returned that the server is running on.
func CreateTLSServer(cert, key string, mux *http.ServeMux) (string, error) {
addr, err := availableLocalAddr("127.0.0.1")
if err != nil {
return "", err
}
if mux == nil {
mux = http.NewServeMux()
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {})
}
go func() {
if err := http.ListenAndServeTLS(addr, cert, key, mux); err != nil {
panic(err)
}
}()
for i := 0; i < 60; i++ {
if _, err := http.Get("https://" + addr); err != nil && !strings.Contains(err.Error(), "connection refused") {
break
}
time.Sleep(1 * time.Second)
}
return "https://" + addr, nil
}
// CreateTLSBundleFiles returns the temporary filenames for the certificate
// key, and CA PEM content. These files should be deleted when no longer
// needed. CleanupTLSBundleFiles can be used for this cleanup.
func CreateTLSBundleFiles() (cert, key, ca string, err error) {
cert, err = createTmpFile(TLSBundleCert)
if err != nil {
return "", "", "", err
}
key, err = createTmpFile(TLSBundleKey)
if err != nil {
return "", "", "", err
}
ca, err = createTmpFile(TLSBundleCA)
if err != nil {
return "", "", "", err
}
return cert, key, ca, nil
}
// CleanupTLSBundleFiles takes variadic list of files to be deleted.
func CleanupTLSBundleFiles(files ...string) error {
for _, file := range files {
if err := os.Remove(file); err != nil {
return err
}
}
return nil
}
func createTmpFile(b []byte) (string, error) {
bundleFile, err := ioutil.TempFile(os.TempDir(), "aws-sdk-go-session-test")
if err != nil {
return "", err
}
_, err = bundleFile.Write(b)
if err != nil {
return "", err
}
defer bundleFile.Close()
return bundleFile.Name(), nil
}

View File

@@ -1,11 +0,0 @@
package awstesting
// DiscardAt is an io.WriteAt that discards
// the requested bytes to be written
type DiscardAt struct{}
// WriteAt discards the given []byte slice and returns len(p) bytes
// as having been written at the given offset. It will never return an error.
func (d DiscardAt) WriteAt(p []byte, off int64) (n int, err error) {
return len(p), nil
}

View File

@@ -1,12 +0,0 @@
package awstesting
// EndlessReader is an io.Reader that will always return
// that bytes have been read.
type EndlessReader struct{}
// Read will report that it has read len(p) bytes in p.
// The content in the []byte will be unmodified.
// This will never return an error.
func (e EndlessReader) Read(p []byte) (int, error) {
return len(p), nil
}

View File

@@ -1,43 +0,0 @@
# Based on docker-library's golang 1.6 alpine and wheezy docker files.
# https://github.com/docker-library/golang/blob/master/1.6/alpine/Dockerfile
# https://github.com/docker-library/golang/blob/master/1.6/wheezy/Dockerfile
FROM buildpack-deps:buster-scm
ENV GOLANG_SRC_REPO_URL https://github.com/golang/go
# as of 1.20 Go 1.17 is required to bootstrap
# see https://github.com/golang/go/issues/44505
ENV GOLANG_BOOTSTRAP_URL https://go.dev/dl/go1.17.13.linux-amd64.tar.gz
ENV GOLANG_BOOTSTRAP_SHA256 4cdd2bc664724dc7db94ad51b503512c5ae7220951cac568120f64f8e94399fc
ENV GOLANG_BOOTSTRAP_PATH /usr/local/bootstrap
# gcc for cgo
RUN apt-get update && apt-get install -y --no-install-recommends \
g++ \
gcc \
libc6-dev \
make \
git \
&& rm -rf /var/lib/apt/lists/*
# Setup the Bootstrap
RUN mkdir -p "$GOLANG_BOOTSTRAP_PATH" \
&& curl -fsSL "$GOLANG_BOOTSTRAP_URL" -o golang.tar.gz \
&& echo "$GOLANG_BOOTSTRAP_SHA256 golang.tar.gz" | sha256sum -c - \
&& tar -C "$GOLANG_BOOTSTRAP_PATH" -xzf golang.tar.gz \
&& rm golang.tar.gz
# Get and build Go tip
RUN export GOROOT_BOOTSTRAP=$GOLANG_BOOTSTRAP_PATH/go \
&& git clone "$GOLANG_SRC_REPO_URL" /usr/local/go \
&& cd /usr/local/go/src \
&& ./make.bash \
&& rm -rf "$GOLANG_BOOTSTRAP_PATH" /usr/local/go/pkg/bootstrap
# Build Go workspace and environment
ENV GOPATH /go
ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" \
&& chmod -R 777 "$GOPATH"
WORKDIR $GOPATH

View File

@@ -1,16 +0,0 @@
FROM aws-golang:tip
ENV GOPROXY=direct
RUN apt-get update && apt-get install -y --no-install-recommends \
software-properties-common \
&& wget -O- https://apt.corretto.aws/corretto.key | apt-key add - \
&& add-apt-repository 'deb https://apt.corretto.aws stable main' \
&& apt-get update && apt-get install -y --no-install-recommends \
vim \
java-17-amazon-corretto-jdk \
&& rm -rf /var/list/apt/lists/*
ADD . /go/src/github.com/aws/aws-sdk-go-v2
WORKDIR /go/src/github.com/aws/aws-sdk-go-v2
CMD ["make", "unit"]

View File

@@ -1,18 +0,0 @@
ARG GO_VERSION
FROM golang:${GO_VERSION}
ENV GOPROXY=direct
RUN apt-get update && apt-get install -y --no-install-recommends \
software-properties-common \
&& wget -O- https://apt.corretto.aws/corretto.key | apt-key add - \
&& add-apt-repository 'deb https://apt.corretto.aws stable main' \
&& apt-get update && apt-get install -y --no-install-recommends \
vim \
java-17-amazon-corretto-jdk \
&& rm -rf /var/list/apt/lists/*
ADD . /go/src/github.com/aws/aws-sdk-go-v2
WORKDIR /go/src/github.com/aws/aws-sdk-go-v2
CMD ["make", "unit"]

View File

@@ -1,61 +0,0 @@
// Package unit performs initialization and validation for unit tests
package unit
import (
"context"
"crypto/rsa"
"math/big"
"github.com/aws/aws-sdk-go-v2/aws"
)
func init() {
config = aws.Config{}
config.Region = "mock-region"
config.Credentials = StubCredentialsProvider{}
}
// StubCredentialsProvider provides a stub credential provider that returns
// static credentials that never expire.
type StubCredentialsProvider struct{}
// Retrieve satisfies the CredentialsProvider interface. Returns stub
// credential value, and never error.
func (StubCredentialsProvider) Retrieve(context.Context) (aws.Credentials, error) {
return aws.Credentials{
AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION",
Source: "unit test credentials",
}, nil
}
var config aws.Config
// Config returns a copy of the mock configuration for unit tests.
func Config() aws.Config { return config.Copy() }
// RSAPrivateKey is used for testing functionality that requires some
// sort of private key. Taken from crypto/rsa/rsa_test.go
//
// Credit to golang 1.11
var RSAPrivateKey = &rsa.PrivateKey{
PublicKey: rsa.PublicKey{
N: fromBase10("14314132931241006650998084889274020608918049032671858325988396851334124245188214251956198731333464217832226406088020736932173064754214329009979944037640912127943488972644697423190955557435910767690712778463524983667852819010259499695177313115447116110358524558307947613422897787329221478860907963827160223559690523660574329011927531289655711860504630573766609239332569210831325633840174683944553667352219670930408593321661375473885147973879086994006440025257225431977751512374815915392249179976902953721486040787792801849818254465486633791826766873076617116727073077821584676715609985777563958286637185868165868520557"),
E: 3,
},
D: fromBase10("9542755287494004433998723259516013739278699355114572217325597900889416163458809501304132487555642811888150937392013824621448709836142886006653296025093941418628992648429798282127303704957273845127141852309016655778568546006839666463451542076964744073572349705538631742281931858219480985907271975884773482372966847639853897890615456605598071088189838676728836833012254065983259638538107719766738032720239892094196108713378822882383694456030043492571063441943847195939549773271694647657549658603365629458610273821292232646334717612674519997533901052790334279661754176490593041941863932308687197618671528035670452762731"),
Primes: []*big.Int{
fromBase10("130903255182996722426771613606077755295583329135067340152947172868415809027537376306193179624298874215608270802054347609836776473930072411958753044562214537013874103802006369634761074377213995983876788718033850153719421695468704276694983032644416930879093914927146648402139231293035971427838068945045019075433"),
fromBase10("109348945610485453577574767652527472924289229538286649661240938988020367005475727988253438647560958573506159449538793540472829815903949343191091817779240101054552748665267574271163617694640513549693841337820602726596756351006149518830932261246698766355347898158548465400674856021497190430791824869615170301029"),
},
}
// Taken from crypto/rsa/rsa_test.go
//
// Credit to golang 1.11
func fromBase10(base10 string) *big.Int {
i, ok := new(big.Int).SetString(base10, 10)
if !ok {
panic("bad number: " + base10)
}
return i
}

View File

@@ -1,201 +0,0 @@
package awstesting
import (
"context"
"io"
"net/http"
"os"
"runtime"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
)
// ZeroReader is a io.Reader which will always write zeros to the byte slice provided.
type ZeroReader struct{}
// Read fills the provided byte slice with zeros returning the number of bytes written.
func (r *ZeroReader) Read(b []byte) (int, error) {
for i := 0; i < len(b); i++ {
b[i] = 0
}
return len(b), nil
}
// ReadCloser is a io.ReadCloser for unit testing.
// Designed to test for leaks and whether a handle has
// been closed
type ReadCloser struct {
Size int
Closed bool
set bool
FillData func(bool, []byte, int, int)
}
// Read will call FillData and fill it with whatever data needed.
// Decrements the size until zero, then return io.EOF.
func (r *ReadCloser) Read(b []byte) (int, error) {
if r.Closed {
return 0, io.EOF
}
delta := len(b)
if delta > r.Size {
delta = r.Size
}
r.Size -= delta
for i := 0; i < delta; i++ {
b[i] = 'a'
}
if r.FillData != nil {
r.FillData(r.set, b, r.Size, delta)
}
r.set = true
if r.Size > 0 {
return delta, nil
}
return delta, io.EOF
}
// Close sets Closed to true and returns no error
func (r *ReadCloser) Close() error {
r.Closed = true
return nil
}
// A FakeContext provides a simple stub implementation of a Context
type FakeContext struct {
Error error
DoneCh chan struct{}
}
// Deadline always will return not set
func (c *FakeContext) Deadline() (deadline time.Time, ok bool) {
return time.Time{}, false
}
// Done returns a read channel for listening to the Done event
func (c *FakeContext) Done() <-chan struct{} {
return c.DoneCh
}
// Err returns the error, is nil if not set.
func (c *FakeContext) Err() error {
return c.Error
}
// Value ignores the Value and always returns nil
func (c *FakeContext) Value(key interface{}) interface{} {
return nil
}
// StashEnv stashes the current environment variables except variables listed in envToKeepx
// Returns an function to pop out old environment
func StashEnv(envToKeep ...string) []string {
if runtime.GOOS == "windows" {
envToKeep = append(envToKeep, "ComSpec")
envToKeep = append(envToKeep, "SYSTEM32")
envToKeep = append(envToKeep, "SYSTEMROOT")
}
envToKeep = append(envToKeep, "PATH", "HOME", "USERPROFILE")
extraEnv := getEnvs(envToKeep)
originalEnv := os.Environ()
os.Clearenv() // clear env
for key, val := range extraEnv {
os.Setenv(key, val)
}
return originalEnv
}
// PopEnv takes the list of the environment values and injects them into the
// process's environment variable data. Clears any existing environment values
// that may already exist.
func PopEnv(env []string) {
os.Clearenv()
for _, e := range env {
p := strings.SplitN(e, "=", 2)
k, v := p[0], ""
if len(p) > 1 {
v = p[1]
}
os.Setenv(k, v)
}
}
// MockCredentialsProvider is a type that can be used to mock out credentials
// providers
type MockCredentialsProvider struct {
RetrieveFn func(ctx context.Context) (aws.Credentials, error)
InvalidateFn func()
}
// Retrieve calls the RetrieveFn
func (p MockCredentialsProvider) Retrieve(ctx context.Context) (aws.Credentials, error) {
return p.RetrieveFn(ctx)
}
// Invalidate calls the InvalidateFn
func (p MockCredentialsProvider) Invalidate() {
p.InvalidateFn()
}
func getEnvs(envs []string) map[string]string {
extraEnvs := make(map[string]string)
for _, env := range envs {
if val, ok := os.LookupEnv(env); ok && len(val) > 0 {
extraEnvs[env] = val
}
}
return extraEnvs
}
const (
signaturePreambleSigV4 = "AWS4-HMAC-SHA256"
signaturePreambleSigV4A = "AWS4-ECDSA-P256-SHA256"
)
// SigV4Signature represents a parsed sigv4 or sigv4a signature.
type SigV4Signature struct {
Preamble string // e.g. AWS4-HMAC-SHA256, AWS4-ECDSA-P256-SHA256
SigningName string // generally the service name e.g. "s3"
SigningRegion string // for sigv4a this is the region-set header as-is
SignedHeaders []string // list of signed headers
Signature string // calculated signature
}
// ParseSigV4Signature deconstructs a sigv4 or sigv4a signature from a set of
// request headers.
func ParseSigV4Signature(header http.Header) *SigV4Signature {
auth := header.Get("Authorization")
preamble, after, _ := strings.Cut(auth, " ")
credential, after, _ := strings.Cut(after, ", ")
signedHeaders, signature, _ := strings.Cut(after, ", ")
credentialParts := strings.Split(credential, "/")
// sigv4 : AccessKeyID/DateString/SigningRegion/SigningName/SignatureID
// sigv4a : AccessKeyID/DateString/SigningName/SignatureID, region set on
// header
var signingName, signingRegion string
if preamble == signaturePreambleSigV4 {
signingName = credentialParts[3]
signingRegion = credentialParts[2]
} else if preamble == signaturePreambleSigV4A {
signingName = credentialParts[2]
signingRegion = header.Get("X-Amz-Region-Set")
}
return &SigV4Signature{
Preamble: preamble,
SigningName: signingName,
SigningRegion: signingRegion,
SignedHeaders: strings.Split(signedHeaders, ";"),
Signature: signature,
}
}

View File

@@ -1,75 +0,0 @@
package awstesting_test
import (
"io"
"testing"
"github.com/versity/versitygw/aws/internal/awstesting"
)
func TestReadCloserClose(t *testing.T) {
rc := awstesting.ReadCloser{Size: 1}
err := rc.Close()
if err != nil {
t.Errorf("expect nil, got %v", err)
}
if !rc.Closed {
t.Errorf("expect closed, was not")
}
if e, a := rc.Size, 1; e != a {
t.Errorf("expect %v, got %v", e, a)
}
}
func TestReadCloserRead(t *testing.T) {
rc := awstesting.ReadCloser{Size: 5}
b := make([]byte, 2)
n, err := rc.Read(b)
if err != nil {
t.Errorf("expect nil, got %v", err)
}
if e, a := n, 2; e != a {
t.Errorf("expect %v, got %v", e, a)
}
if rc.Closed {
t.Errorf("expect not to be closed")
}
if e, a := rc.Size, 3; e != a {
t.Errorf("expect %v, got %v", e, a)
}
err = rc.Close()
if err != nil {
t.Errorf("expect nil, got %v", err)
}
n, err = rc.Read(b)
if e, a := err, io.EOF; e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := n, 0; e != a {
t.Errorf("expect %v, got %v", e, a)
}
}
func TestReadCloserReadAll(t *testing.T) {
rc := awstesting.ReadCloser{Size: 5}
b := make([]byte, 5)
n, err := rc.Read(b)
if e, a := err, io.EOF; e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := n, 5; e != a {
t.Errorf("expect %v, got %v", e, a)
}
if rc.Closed {
t.Errorf("expect not to be closed")
}
if e, a := rc.Size, 0; e != a {
t.Errorf("expect %v, got %v", e, a)
}
}

View File

@@ -1,9 +0,0 @@
package sdk
// Invalidator provides access to a type's invalidate method to make it
// invalidate it cache.
//
// e.g aws.SafeCredentialsProvider's Invalidate method.
type Invalidator interface {
Invalidate()
}

View File

@@ -1,74 +0,0 @@
package sdk
import (
"context"
"time"
)
func init() {
NowTime = time.Now
Sleep = time.Sleep
SleepWithContext = sleepWithContext
}
// NowTime is a value for getting the current time. This value can be overridden
// for testing mocking out current time.
var NowTime func() time.Time
// Sleep is a value for sleeping for a duration. This value can be overridden
// for testing and mocking out sleep duration.
var Sleep func(time.Duration)
// SleepWithContext will wait for the timer duration to expire, or the context
// is canceled. Which ever happens first. If the context is canceled the Context's
// error will be returned.
//
// This value can be overridden for testing and mocking out sleep duration.
var SleepWithContext func(context.Context, time.Duration) error
// sleepWithContext will wait for the timer duration to expire, or the context
// is canceled. Which ever happens first. If the context is canceled the
// Context's error will be returned.
func sleepWithContext(ctx context.Context, dur time.Duration) error {
t := time.NewTimer(dur)
defer t.Stop()
select {
case <-t.C:
break
case <-ctx.Done():
return ctx.Err()
}
return nil
}
// noOpSleepWithContext does nothing, returns immediately.
func noOpSleepWithContext(context.Context, time.Duration) error {
return nil
}
func noOpSleep(time.Duration) {}
// TestingUseNopSleep is a utility for disabling sleep across the SDK for
// testing.
func TestingUseNopSleep() func() {
SleepWithContext = noOpSleepWithContext
Sleep = noOpSleep
return func() {
SleepWithContext = sleepWithContext
Sleep = time.Sleep
}
}
// TestingUseReferenceTime is a utility for swapping the time function across the SDK to return a specific reference time
// for testing purposes.
func TestingUseReferenceTime(referenceTime time.Time) func() {
NowTime = func() time.Time {
return referenceTime
}
return func() {
NowTime = time.Now
}
}

View File

@@ -1,32 +0,0 @@
package sdk
import (
"context"
"strings"
"testing"
"time"
)
func TestSleepWithContext(t *testing.T) {
ctx, cancelFn := context.WithCancel(context.Background())
defer cancelFn()
err := sleepWithContext(ctx, 1*time.Millisecond)
if err != nil {
t.Errorf("expect context to not be canceled, got %v", err)
}
}
func TestSleepWithContext_Canceled(t *testing.T) {
ctx, cancelFn := context.WithCancel(context.Background())
cancelFn()
err := sleepWithContext(ctx, 10*time.Second)
if err == nil {
t.Fatalf("expect error, did not get one")
}
if e, a := "context canceled", err.Error(); !strings.Contains(a, e) {
t.Errorf("expect %v error, got %v", e, a)
}
}

View File

@@ -1,11 +0,0 @@
package strings
import (
"strings"
)
// HasPrefixFold tests whether the string s begins with prefix, interpreted as UTF-8 strings,
// under Unicode case-folding.
func HasPrefixFold(s, prefix string) bool {
return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix)
}

View File

@@ -1,81 +0,0 @@
package strings
import (
"strings"
"testing"
)
func TestHasPrefixFold(t *testing.T) {
type args struct {
s string
prefix string
}
tests := map[string]struct {
args args
want bool
}{
"empty strings and prefix": {
args: args{
s: "",
prefix: "",
},
want: true,
},
"strings starts with prefix": {
args: args{
s: "some string",
prefix: "some",
},
want: true,
},
"prefix longer then string": {
args: args{
s: "some",
prefix: "some string",
},
},
"equal length string and prefix": {
args: args{
s: "short string",
prefix: "short string",
},
want: true,
},
"different cases": {
args: args{
s: "ShOrT StRING",
prefix: "short",
},
want: true,
},
"empty prefix not empty string": {
args: args{
s: "ShOrT StRING",
prefix: "",
},
want: true,
},
"mixed-case prefixes": {
args: args{
s: "SoMe String",
prefix: "sOme",
},
want: true,
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
if got := HasPrefixFold(tt.args.s, tt.args.prefix); got != tt.want {
t.Errorf("HasPrefixFold() = %v, want %v", got, tt.want)
}
})
}
}
func BenchmarkHasPrefixFold(b *testing.B) {
HasPrefixFold("SoME string", "sOmE")
}
func BenchmarkHasPrefix(b *testing.B) {
strings.HasPrefix(strings.ToLower("SoME string"), strings.ToLower("sOmE"))
}

View File

@@ -1,115 +0,0 @@
package v4
import (
"strings"
"sync"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
)
func lookupKey(service, region string) string {
var s strings.Builder
s.Grow(len(region) + len(service) + 3)
s.WriteString(region)
s.WriteRune('/')
s.WriteString(service)
return s.String()
}
type derivedKey struct {
AccessKey string
Date time.Time
Credential []byte
}
type derivedKeyCache struct {
values map[string]derivedKey
mutex sync.RWMutex
}
func newDerivedKeyCache() derivedKeyCache {
return derivedKeyCache{
values: make(map[string]derivedKey),
}
}
func (s *derivedKeyCache) Get(credentials aws.Credentials, service, region string, signingTime SigningTime) []byte {
key := lookupKey(service, region)
s.mutex.RLock()
if cred, ok := s.get(key, credentials, signingTime.Time); ok {
s.mutex.RUnlock()
return cred
}
s.mutex.RUnlock()
s.mutex.Lock()
if cred, ok := s.get(key, credentials, signingTime.Time); ok {
s.mutex.Unlock()
return cred
}
cred := deriveKey(credentials.SecretAccessKey, service, region, signingTime)
entry := derivedKey{
AccessKey: credentials.AccessKeyID,
Date: signingTime.Time,
Credential: cred,
}
s.values[key] = entry
s.mutex.Unlock()
return cred
}
func (s *derivedKeyCache) get(key string, credentials aws.Credentials, signingTime time.Time) ([]byte, bool) {
cacheEntry, ok := s.retrieveFromCache(key)
if ok && cacheEntry.AccessKey == credentials.AccessKeyID && isSameDay(signingTime, cacheEntry.Date) {
return cacheEntry.Credential, true
}
return nil, false
}
func (s *derivedKeyCache) retrieveFromCache(key string) (derivedKey, bool) {
if v, ok := s.values[key]; ok {
return v, true
}
return derivedKey{}, false
}
// SigningKeyDeriver derives a signing key from a set of credentials
type SigningKeyDeriver struct {
cache derivedKeyCache
}
// NewSigningKeyDeriver returns a new SigningKeyDeriver
func NewSigningKeyDeriver() *SigningKeyDeriver {
return &SigningKeyDeriver{
cache: newDerivedKeyCache(),
}
}
// DeriveKey returns a derived signing key from the given credentials to be used with SigV4 signing.
func (k *SigningKeyDeriver) DeriveKey(credential aws.Credentials, service, region string, signingTime SigningTime) []byte {
return k.cache.Get(credential, service, region, signingTime)
}
func deriveKey(secret, service, region string, t SigningTime) []byte {
hmacDate := HMACSHA256([]byte("AWS4"+secret), []byte(t.ShortTimeFormat()))
hmacRegion := HMACSHA256(hmacDate, []byte(region))
hmacService := HMACSHA256(hmacRegion, []byte(service))
return HMACSHA256(hmacService, []byte("aws4_request"))
}
func isSameDay(x, y time.Time) bool {
xYear, xMonth, xDay := x.Date()
yYear, yMonth, yDay := y.Date()
if xYear != yYear {
return false
}
if xMonth != yMonth {
return false
}
return xDay == yDay
}

View File

@@ -1,40 +0,0 @@
package v4
// Signature Version 4 (SigV4) Constants
const (
// EmptyStringSHA256 is the hex encoded sha256 value of an empty string
EmptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`
// UnsignedPayload indicates that the request payload body is unsigned
UnsignedPayload = "UNSIGNED-PAYLOAD"
// AmzAlgorithmKey indicates the signing algorithm
AmzAlgorithmKey = "X-Amz-Algorithm"
// AmzSecurityTokenKey indicates the security token to be used with temporary credentials
AmzSecurityTokenKey = "X-Amz-Security-Token"
// AmzDateKey is the UTC timestamp for the request in the format YYYYMMDD'T'HHMMSS'Z'
AmzDateKey = "X-Amz-Date"
// AmzCredentialKey is the access key ID and credential scope
AmzCredentialKey = "X-Amz-Credential"
// AmzSignedHeadersKey is the set of headers signed for the request
AmzSignedHeadersKey = "X-Amz-SignedHeaders"
// AmzSignatureKey is the query parameter to store the SigV4 signature
AmzSignatureKey = "X-Amz-Signature"
// TimeFormat is the time format to be used in the X-Amz-Date header or query parameter
TimeFormat = "20060102T150405Z"
// ShortTimeFormat is the shorten time format used in the credential scope
ShortTimeFormat = "20060102"
// ContentSHAKey is the SHA256 of request body
ContentSHAKey = "X-Amz-Content-Sha256"
// StreamingEventsPayload indicates that the request payload body is a signed event stream.
StreamingEventsPayload = "STREAMING-AWS4-HMAC-SHA256-EVENTS"
)

View File

@@ -1,82 +0,0 @@
package v4
import (
sdkstrings "github.com/versity/versitygw/aws/internal/strings"
)
// Rules houses a set of Rule needed for validation of a
// string value
type Rules []Rule
// Rule interface allows for more flexible rules and just simply
// checks whether or not a value adheres to that Rule
type Rule interface {
IsValid(value string) bool
}
// IsValid will iterate through all rules and see if any rules
// apply to the value and supports nested rules
func (r Rules) IsValid(value string) bool {
for _, rule := range r {
if rule.IsValid(value) {
return true
}
}
return false
}
// MapRule generic Rule for maps
type MapRule map[string]struct{}
// IsValid for the map Rule satisfies whether it exists in the map
func (m MapRule) IsValid(value string) bool {
_, ok := m[value]
return ok
}
// AllowList is a generic Rule for include listing
type AllowList struct {
Rule
}
// IsValid for AllowList checks if the value is within the AllowList
func (w AllowList) IsValid(value string) bool {
return w.Rule.IsValid(value)
}
// ExcludeList is a generic Rule for exclude listing
type ExcludeList struct {
Rule
}
// IsValid for AllowList checks if the value is within the AllowList
func (b ExcludeList) IsValid(value string) bool {
return !b.Rule.IsValid(value)
}
// Patterns is a list of strings to match against
type Patterns []string
// IsValid for Patterns checks each pattern and returns if a match has
// been found
func (p Patterns) IsValid(value string) bool {
for _, pattern := range p {
if sdkstrings.HasPrefixFold(value, pattern) {
return true
}
}
return false
}
// InclusiveRules rules allow for rules to depend on one another
type InclusiveRules []Rule
// IsValid will return true if all rules are true
func (r InclusiveRules) IsValid(value string) bool {
for _, rule := range r {
if !rule.IsValid(value) {
return false
}
}
return true
}

View File

@@ -1,72 +0,0 @@
package v4
// IgnoredHeaders is a list of headers that are ignored during signing
var IgnoredHeaders = Rules{
ExcludeList{
MapRule{
"Authorization": struct{}{},
// some clients use user-agent in signed headers
// "User-Agent": struct{}{},
"X-Amzn-Trace-Id": struct{}{},
"Expect": struct{}{},
},
},
}
// RequiredSignedHeaders is a allow list for Build canonical headers.
var RequiredSignedHeaders = Rules{
AllowList{
MapRule{
"Cache-Control": struct{}{},
"Content-Disposition": struct{}{},
"Content-Encoding": struct{}{},
"Content-Language": struct{}{},
"Content-Md5": struct{}{},
"Content-Type": struct{}{},
"Expires": struct{}{},
"If-Match": struct{}{},
"If-Modified-Since": struct{}{},
"If-None-Match": struct{}{},
"If-Unmodified-Since": struct{}{},
"Range": struct{}{},
"X-Amz-Acl": struct{}{},
"X-Amz-Copy-Source": struct{}{},
"X-Amz-Copy-Source-If-Match": struct{}{},
"X-Amz-Copy-Source-If-Modified-Since": struct{}{},
"X-Amz-Copy-Source-If-None-Match": struct{}{},
"X-Amz-Copy-Source-If-Unmodified-Since": struct{}{},
"X-Amz-Copy-Source-Range": struct{}{},
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": struct{}{},
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": struct{}{},
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": struct{}{},
"X-Amz-Expected-Bucket-Owner": struct{}{},
"X-Amz-Grant-Full-control": struct{}{},
"X-Amz-Grant-Read": struct{}{},
"X-Amz-Grant-Read-Acp": struct{}{},
"X-Amz-Grant-Write": struct{}{},
"X-Amz-Grant-Write-Acp": struct{}{},
"X-Amz-Metadata-Directive": struct{}{},
"X-Amz-Mfa": struct{}{},
"X-Amz-Request-Payer": struct{}{},
"X-Amz-Server-Side-Encryption": struct{}{},
"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": struct{}{},
"X-Amz-Server-Side-Encryption-Context": struct{}{},
"X-Amz-Server-Side-Encryption-Customer-Algorithm": struct{}{},
"X-Amz-Server-Side-Encryption-Customer-Key": struct{}{},
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": struct{}{},
"X-Amz-Storage-Class": struct{}{},
"X-Amz-Website-Redirect-Location": struct{}{},
"X-Amz-Content-Sha256": struct{}{},
"X-Amz-Tagging": struct{}{},
},
},
Patterns{"X-Amz-Object-Lock-"},
Patterns{"X-Amz-Meta-"},
}
// AllowedQueryHoisting is a allowed list for Build query headers. The boolean value
// represents whether or not it is a pattern.
var AllowedQueryHoisting = InclusiveRules{
ExcludeList{RequiredSignedHeaders},
Patterns{"X-Amz-"},
}

View File

@@ -1,63 +0,0 @@
package v4
import "testing"
func TestAllowedQueryHoisting(t *testing.T) {
cases := map[string]struct {
Header string
ExpectHoist bool
}{
"object-lock": {
Header: "X-Amz-Object-Lock-Mode",
ExpectHoist: false,
},
"s3 metadata": {
Header: "X-Amz-Meta-SomeName",
ExpectHoist: false,
},
"another header": {
Header: "X-Amz-SomeOtherHeader",
ExpectHoist: true,
},
"non X-AMZ header": {
Header: "X-SomeOtherHeader",
ExpectHoist: false,
},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
if e, a := c.ExpectHoist, AllowedQueryHoisting.IsValid(c.Header); e != a {
t.Errorf("expect hoist %v, was %v", e, a)
}
})
}
}
func TestIgnoredHeaders(t *testing.T) {
cases := map[string]struct {
Header string
ExpectIgnored bool
}{
"expect": {
Header: "Expect",
ExpectIgnored: true,
},
"authorization": {
Header: "Authorization",
ExpectIgnored: true,
},
"X-AMZ header": {
Header: "X-Amz-Content-Sha256",
ExpectIgnored: false,
},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
if e, a := c.ExpectIgnored, IgnoredHeaders.IsValid(c.Header); e == a {
t.Errorf("expect ignored %v, was %v", e, a)
}
})
}
}

View File

@@ -1,13 +0,0 @@
package v4
import (
"crypto/hmac"
"crypto/sha256"
)
// HMACSHA256 computes a HMAC-SHA256 of data given the provided key.
func HMACSHA256(key []byte, data []byte) []byte {
hash := hmac.New(sha256.New, key)
hash.Write(data)
return hash.Sum(nil)
}

View File

@@ -1,75 +0,0 @@
package v4
import (
"net/http"
"strings"
)
// SanitizeHostForHeader removes default port from host and updates request.Host
func SanitizeHostForHeader(r *http.Request) {
host := getHost(r)
port := portOnly(host)
if port != "" && isDefaultPort(r.URL.Scheme, port) {
r.Host = stripPort(host)
}
}
// Returns host from request
func getHost(r *http.Request) string {
if r.Host != "" {
return r.Host
}
return r.URL.Host
}
// Hostname returns u.Host, without any port number.
//
// If Host is an IPv6 literal with a port number, Hostname returns the
// IPv6 literal without the square brackets. IPv6 literals may include
// a zone identifier.
//
// Copied from the Go 1.8 standard library (net/url)
func stripPort(hostport string) string {
colon := strings.IndexByte(hostport, ':')
if colon == -1 {
return hostport
}
if i := strings.IndexByte(hostport, ']'); i != -1 {
return strings.TrimPrefix(hostport[:i], "[")
}
return hostport[:colon]
}
// Port returns the port part of u.Host, without the leading colon.
// If u.Host doesn't contain a port, Port returns an empty string.
//
// Copied from the Go 1.8 standard library (net/url)
func portOnly(hostport string) string {
colon := strings.IndexByte(hostport, ':')
if colon == -1 {
return ""
}
if i := strings.Index(hostport, "]:"); i != -1 {
return hostport[i+len("]:"):]
}
if strings.Contains(hostport, "]") {
return ""
}
return hostport[colon+len(":"):]
}
// Returns true if the specified URI is using the standard port
// (i.e. port 80 for HTTP URIs or 443 for HTTPS URIs)
func isDefaultPort(scheme, port string) bool {
if port == "" {
return true
}
lowerCaseScheme := strings.ToLower(scheme)
if (lowerCaseScheme == "http" && port == "80") || (lowerCaseScheme == "https" && port == "443") {
return true
}
return false
}

View File

@@ -1,13 +0,0 @@
package v4
import "strings"
// BuildCredentialScope builds the Signature Version 4 (SigV4) signing scope
func BuildCredentialScope(signingTime SigningTime, region, service string) string {
return strings.Join([]string{
signingTime.ShortTimeFormat(),
region,
service,
"aws4_request",
}, "/")
}

View File

@@ -1,36 +0,0 @@
package v4
import "time"
// SigningTime provides a wrapper around a time.Time which provides cached values for SigV4 signing.
type SigningTime struct {
time.Time
timeFormat string
shortTimeFormat string
}
// NewSigningTime creates a new SigningTime given a time.Time
func NewSigningTime(t time.Time) SigningTime {
return SigningTime{
Time: t,
}
}
// TimeFormat provides a time formatted in the X-Amz-Date format.
func (m *SigningTime) TimeFormat() string {
return m.format(&m.timeFormat, TimeFormat)
}
// ShortTimeFormat provides a time formatted of 20060102.
func (m *SigningTime) ShortTimeFormat() string {
return m.format(&m.shortTimeFormat, ShortTimeFormat)
}
func (m *SigningTime) format(target *string, format string) string {
if len(*target) > 0 {
return *target
}
v := m.Time.Format(format)
*target = v
return v
}

View File

@@ -1,80 +0,0 @@
package v4
import (
"net/url"
"strings"
)
const doubleSpace = " "
// StripExcessSpaces will rewrite the passed in slice's string values to not
// contain multiple side-by-side spaces.
func StripExcessSpaces(str string) string {
var j, k, l, m, spaces int
// Trim trailing spaces
for j = len(str) - 1; j >= 0 && str[j] == ' '; j-- {
}
// Trim leading spaces
for k = 0; k < j && str[k] == ' '; k++ {
}
str = str[k : j+1]
// Strip multiple spaces.
j = strings.Index(str, doubleSpace)
if j < 0 {
return str
}
buf := []byte(str)
for k, m, l = j, j, len(buf); k < l; k++ {
if buf[k] == ' ' {
if spaces == 0 {
// First space.
buf[m] = buf[k]
m++
}
spaces++
} else {
// End of multiple spaces.
spaces = 0
buf[m] = buf[k]
m++
}
}
return string(buf[:m])
}
// GetURIPath returns the escaped URI component from the provided URL.
func GetURIPath(u *url.URL) string {
var uriPath string
if len(u.Opaque) > 0 {
const schemeSep, pathSep, queryStart = "//", "/", "?"
opaque := u.Opaque
// Cut off the query string if present.
if idx := strings.Index(opaque, queryStart); idx >= 0 {
opaque = opaque[:idx]
}
// Cutout the scheme separator if present.
if strings.Index(opaque, schemeSep) == 0 {
opaque = opaque[len(schemeSep):]
}
// capture URI path starting with first path separator.
if idx := strings.Index(opaque, pathSep); idx >= 0 {
uriPath = opaque[idx:]
}
} else {
uriPath = u.EscapedPath()
}
if len(uriPath) == 0 {
uriPath = "/"
}
return uriPath
}

View File

@@ -1,158 +0,0 @@
package v4
import (
"net/http"
"net/url"
"testing"
)
func lazyURLParse(v string) func() (*url.URL, error) {
return func() (*url.URL, error) {
return url.Parse(v)
}
}
func TestGetURIPath(t *testing.T) {
cases := map[string]struct {
getURL func() (*url.URL, error)
expect string
}{
// Cases
"with scheme": {
getURL: lazyURLParse("https://localhost:9000"),
expect: "/",
},
"no port, with scheme": {
getURL: lazyURLParse("https://localhost"),
expect: "/",
},
"without scheme": {
getURL: lazyURLParse("localhost:9000"),
expect: "/",
},
"without scheme, with path": {
getURL: lazyURLParse("localhost:9000/abc123"),
expect: "/abc123",
},
"without scheme, with separator": {
getURL: lazyURLParse("//localhost:9000"),
expect: "/",
},
"no port, without scheme, with separator": {
getURL: lazyURLParse("//localhost"),
expect: "/",
},
"without scheme, with separator, with path": {
getURL: lazyURLParse("//localhost:9000/abc123"),
expect: "/abc123",
},
"no port, without scheme, with separator, with path": {
getURL: lazyURLParse("//localhost/abc123"),
expect: "/abc123",
},
"opaque with query string": {
getURL: lazyURLParse("localhost:9000/abc123?efg=456"),
expect: "/abc123",
},
"failing test": {
getURL: func() (*url.URL, error) {
endpoint := "https://service.region.amazonaws.com"
req, _ := http.NewRequest("POST", endpoint, nil)
u := req.URL
u.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
query := u.Query()
query.Set("some-query-key", "value")
u.RawQuery = query.Encode()
return u, nil
},
expect: "/bucket/key-._~,!@#$%^&*()",
},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
u, err := c.getURL()
if err != nil {
t.Fatalf("failed to get URL, %v", err)
}
actual := GetURIPath(u)
if e, a := c.expect, actual; e != a {
t.Errorf("expect %v path, got %v", e, a)
}
})
}
}
func TestStripExcessHeaders(t *testing.T) {
vals := []string{
"",
"123",
"1 2 3",
"1 2 3 ",
" 1 2 3",
"1 2 3",
"1 23",
"1 2 3",
"1 2 ",
" 1 2 ",
"12 3",
"12 3 1",
"12 3 1",
"12 3 1abc123",
}
expected := []string{
"",
"123",
"1 2 3",
"1 2 3",
"1 2 3",
"1 2 3",
"1 23",
"1 2 3",
"1 2",
"1 2",
"12 3",
"12 3 1",
"12 3 1",
"12 3 1abc123",
}
for i := 0; i < len(vals); i++ {
r := StripExcessSpaces(vals[i])
if e, a := expected[i], r; e != a {
t.Errorf("%d, expect %v, got %v", i, e, a)
}
}
}
var stripExcessSpaceCases = []string{
`AWS4-HMAC-SHA256 Credential=AKIDFAKEIDFAKEID/20160628/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=1234567890abcdef1234567890abcdef1234567890abcdef`,
`123 321 123 321`,
` 123 321 123 321 `,
` 123 321 123 321 `,
"123",
"1 2 3",
" 1 2 3",
"1 2 3",
"1 23",
"1 2 3",
"1 2 ",
" 1 2 ",
"12 3",
"12 3 1",
"12 3 1",
"12 3 1abc123",
}
func BenchmarkStripExcessSpaces(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, v := range stripExcessSpaceCases {
StripExcessSpaces(v)
}
}
}

View File

@@ -1,139 +0,0 @@
package v4_test
import (
"context"
"fmt"
"net/http"
"testing"
"time"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/versity/versitygw/aws/internal/awstesting/unit"
v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
)
var standaloneSignCases = []struct {
OrigURI string
OrigQuery string
Region, Service, SubDomain string
ExpSig string
EscapedURI string
}{
{
OrigURI: `/logs-*/_search`,
OrigQuery: `pretty=true`,
Region: "us-west-2", Service: "es", SubDomain: "hostname-clusterkey",
EscapedURI: `/logs-%2A/_search`,
ExpSig: `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-west-2/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=79d0760751907af16f64a537c1242416dacf51204a7dd5284492d15577973b91`,
},
}
func TestStandaloneSign_CustomURIEscape(t *testing.T) {
var expectSig = `AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/es/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=6601e883cc6d23871fd6c2a394c5677ea2b8c82b04a6446786d64cd74f520967`
creds, err := unit.Config().Credentials.Retrieve(context.Background())
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
signer := v4.NewSigner(func(signer *v4.SignerOptions) {
signer.DisableURIPathEscaping = true
})
host := "https://subdomain.us-east-1.es.amazonaws.com"
req, err := http.NewRequest("GET", host, nil)
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
req.URL.Path = `/log-*/_search`
req.URL.Opaque = "//subdomain.us-east-1.es.amazonaws.com/log-%2A/_search"
err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, "es", "us-east-1", time.Unix(0, 0))
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
actual := req.Header.Get("Authorization")
if e, a := expectSig, actual; e != a {
t.Errorf("expect %v, got %v", e, a)
}
}
func TestStandaloneSign(t *testing.T) {
creds, err := unit.Config().Credentials.Retrieve(context.Background())
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
signer := v4.NewSigner()
for _, c := range standaloneSignCases {
host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
c.SubDomain, c.Region, c.Service)
req, err := http.NewRequest("GET", host, nil)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
// URL.EscapedPath() will be used by the signer to get the
// escaped form of the request's URI path.
req.URL.Path = c.OrigURI
req.URL.RawQuery = c.OrigQuery
err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
actual := req.Header.Get("Authorization")
if e, a := c.ExpSig, actual; e != a {
t.Errorf("expected %v, but recieved %v", e, a)
}
if e, a := c.OrigURI, req.URL.Path; e != a {
t.Errorf("expected %v, but recieved %v", e, a)
}
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
t.Errorf("expected %v, but recieved %v", e, a)
}
}
}
func TestStandaloneSign_RawPath(t *testing.T) {
creds, err := unit.Config().Credentials.Retrieve(context.Background())
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
signer := v4.NewSigner()
for _, c := range standaloneSignCases {
host := fmt.Sprintf("https://%s.%s.%s.amazonaws.com",
c.SubDomain, c.Region, c.Service)
req, err := http.NewRequest("GET", host, nil)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
// URL.EscapedPath() will be used by the signer to get the
// escaped form of the request's URI path.
req.URL.Path = c.OrigURI
req.URL.RawPath = c.EscapedURI
req.URL.RawQuery = c.OrigQuery
err = signer.SignHTTP(context.Background(), creds, req, v4Internal.EmptyStringSHA256, c.Service, c.Region, time.Unix(0, 0))
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
actual := req.Header.Get("Authorization")
if e, a := c.ExpSig, actual; e != a {
t.Errorf("expected %v, but recieved %v", e, a)
}
if e, a := c.OrigURI, req.URL.Path; e != a {
t.Errorf("expected %v, but recieved %v", e, a)
}
if e, a := c.EscapedURI, req.URL.EscapedPath(); e != a {
t.Errorf("expected %v, but recieved %v", e, a)
}
}
}

View File

@@ -1,443 +0,0 @@
package v4
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"strings"
"github.com/aws/aws-sdk-go-v2/aws"
awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
"github.com/aws/aws-sdk-go-v2/aws/middleware/private/metrics"
"github.com/aws/smithy-go/middleware"
smithyhttp "github.com/aws/smithy-go/transport/http"
internalauth "github.com/versity/versitygw/aws/internal/auth"
"github.com/versity/versitygw/aws/internal/sdk"
v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
)
const computePayloadHashMiddlewareID = "ComputePayloadHash"
// HashComputationError indicates an error occurred while computing the signing hash
type HashComputationError struct {
Err error
}
// Error is the error message
func (e *HashComputationError) Error() string {
return fmt.Sprintf("failed to compute payload hash: %v", e.Err)
}
// Unwrap returns the underlying error if one is set
func (e *HashComputationError) Unwrap() error {
return e.Err
}
// SigningError indicates an error condition occurred while performing SigV4 signing
type SigningError struct {
Err error
}
func (e *SigningError) Error() string {
return fmt.Sprintf("failed to sign request: %v", e.Err)
}
// Unwrap returns the underlying error cause
func (e *SigningError) Unwrap() error {
return e.Err
}
// UseDynamicPayloadSigningMiddleware swaps the compute payload sha256 middleware with a resolver middleware that
// switches between unsigned and signed payload based on TLS state for request.
// This middleware should not be used for AWS APIs that do not support unsigned payload signing auth.
// By default, SDK uses this middleware for known AWS APIs that support such TLS based auth selection .
//
// Usage example -
// S3 PutObject API allows unsigned payload signing auth usage when TLS is enabled, and uses this middleware to
// dynamically switch between unsigned and signed payload based on TLS state for request.
func UseDynamicPayloadSigningMiddleware(stack *middleware.Stack) error {
_, err := stack.Finalize.Swap(computePayloadHashMiddlewareID, &dynamicPayloadSigningMiddleware{})
return err
}
// dynamicPayloadSigningMiddleware dynamically resolves the middleware that computes and set payload sha256 middleware.
type dynamicPayloadSigningMiddleware struct {
}
// ID returns the resolver identifier
func (m *dynamicPayloadSigningMiddleware) ID() string {
return computePayloadHashMiddlewareID
}
// HandleFinalize delegates SHA256 computation according to whether the request
// is TLS-enabled.
func (m *dynamicPayloadSigningMiddleware) HandleFinalize(
ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
) (
out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
) {
req, ok := in.Request.(*smithyhttp.Request)
if !ok {
return out, metadata, fmt.Errorf("unknown transport type %T", in.Request)
}
if req.IsHTTPS() {
return (&UnsignedPayload{}).HandleFinalize(ctx, in, next)
}
return (&ComputePayloadSHA256{}).HandleFinalize(ctx, in, next)
}
// UnsignedPayload sets the SigV4 request payload hash to unsigned.
//
// Will not set the Unsigned Payload magic SHA value, if a SHA has already been
// stored in the context. (e.g. application pre-computed SHA256 before making
// API call).
//
// This middleware does not check the X-Amz-Content-Sha256 header, if that
// header is serialized a middleware must translate it into the context.
type UnsignedPayload struct{}
// AddUnsignedPayloadMiddleware adds unsignedPayload to the operation
// middleware stack
func AddUnsignedPayloadMiddleware(stack *middleware.Stack) error {
return stack.Finalize.Insert(&UnsignedPayload{}, "ResolveEndpointV2", middleware.After)
}
// ID returns the unsignedPayload identifier
func (m *UnsignedPayload) ID() string {
return computePayloadHashMiddlewareID
}
// HandleFinalize sets the payload hash magic value to the unsigned sentinel.
func (m *UnsignedPayload) HandleFinalize(
ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
) (
out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
) {
if GetPayloadHash(ctx) == "" {
ctx = SetPayloadHash(ctx, v4Internal.UnsignedPayload)
}
return next.HandleFinalize(ctx, in)
}
// ComputePayloadSHA256 computes SHA256 payload hash to sign.
//
// Will not set the Unsigned Payload magic SHA value, if a SHA has already been
// stored in the context. (e.g. application pre-computed SHA256 before making
// API call).
//
// This middleware does not check the X-Amz-Content-Sha256 header, if that
// header is serialized a middleware must translate it into the context.
type ComputePayloadSHA256 struct{}
// AddComputePayloadSHA256Middleware adds computePayloadSHA256 to the
// operation middleware stack
func AddComputePayloadSHA256Middleware(stack *middleware.Stack) error {
return stack.Finalize.Insert(&ComputePayloadSHA256{}, "ResolveEndpointV2", middleware.After)
}
// RemoveComputePayloadSHA256Middleware removes computePayloadSHA256 from the
// operation middleware stack
func RemoveComputePayloadSHA256Middleware(stack *middleware.Stack) error {
_, err := stack.Finalize.Remove(computePayloadHashMiddlewareID)
return err
}
// ID is the middleware name
func (m *ComputePayloadSHA256) ID() string {
return computePayloadHashMiddlewareID
}
// HandleFinalize computes the payload hash for the request, storing it to the
// context. This is a no-op if a caller has previously set that value.
func (m *ComputePayloadSHA256) HandleFinalize(
ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
) (
out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
) {
if GetPayloadHash(ctx) != "" {
return next.HandleFinalize(ctx, in)
}
req, ok := in.Request.(*smithyhttp.Request)
if !ok {
return out, metadata, &HashComputationError{
Err: fmt.Errorf("unexpected request middleware type %T", in.Request),
}
}
hash := sha256.New()
if stream := req.GetStream(); stream != nil {
_, err = io.Copy(hash, stream)
if err != nil {
return out, metadata, &HashComputationError{
Err: fmt.Errorf("failed to compute payload hash, %w", err),
}
}
if err := req.RewindStream(); err != nil {
return out, metadata, &HashComputationError{
Err: fmt.Errorf("failed to seek body to start, %w", err),
}
}
}
ctx = SetPayloadHash(ctx, hex.EncodeToString(hash.Sum(nil)))
return next.HandleFinalize(ctx, in)
}
// SwapComputePayloadSHA256ForUnsignedPayloadMiddleware replaces the
// ComputePayloadSHA256 middleware with the UnsignedPayload middleware.
//
// Use this to disable computing the Payload SHA256 checksum and instead use
// UNSIGNED-PAYLOAD for the SHA256 value.
func SwapComputePayloadSHA256ForUnsignedPayloadMiddleware(stack *middleware.Stack) error {
_, err := stack.Finalize.Swap(computePayloadHashMiddlewareID, &UnsignedPayload{})
return err
}
// ContentSHA256Header sets the X-Amz-Content-Sha256 header value to
// the Payload hash stored in the context.
type ContentSHA256Header struct{}
// AddContentSHA256HeaderMiddleware adds ContentSHA256Header to the
// operation middleware stack
func AddContentSHA256HeaderMiddleware(stack *middleware.Stack) error {
return stack.Finalize.Insert(&ContentSHA256Header{}, computePayloadHashMiddlewareID, middleware.After)
}
// RemoveContentSHA256HeaderMiddleware removes contentSHA256Header middleware
// from the operation middleware stack
func RemoveContentSHA256HeaderMiddleware(stack *middleware.Stack) error {
_, err := stack.Finalize.Remove((*ContentSHA256Header)(nil).ID())
return err
}
// ID returns the ContentSHA256HeaderMiddleware identifier
func (m *ContentSHA256Header) ID() string {
return "SigV4ContentSHA256Header"
}
// HandleFinalize sets the X-Amz-Content-Sha256 header value to the Payload hash
// stored in the context.
func (m *ContentSHA256Header) HandleFinalize(
ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
) (
out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
) {
req, ok := in.Request.(*smithyhttp.Request)
if !ok {
return out, metadata, &HashComputationError{Err: fmt.Errorf("unexpected request middleware type %T", in.Request)}
}
req.Header.Set(v4Internal.ContentSHAKey, GetPayloadHash(ctx))
return next.HandleFinalize(ctx, in)
}
// SignHTTPRequestMiddlewareOptions is the configuration options for
// [SignHTTPRequestMiddleware].
//
// Deprecated: [SignHTTPRequestMiddleware] is deprecated.
type SignHTTPRequestMiddlewareOptions struct {
CredentialsProvider aws.CredentialsProvider
Signer HTTPSigner
LogSigning bool
}
// SignHTTPRequestMiddleware is a `FinalizeMiddleware` implementation for SigV4
// HTTP Signing.
//
// Deprecated: AWS service clients no longer use this middleware. Signing as an
// SDK operation is now performed through an internal per-service middleware
// which opaquely selects and uses the signer from the resolved auth scheme.
type SignHTTPRequestMiddleware struct {
credentialsProvider aws.CredentialsProvider
signer HTTPSigner
logSigning bool
}
// NewSignHTTPRequestMiddleware constructs a [SignHTTPRequestMiddleware] using
// the given [Signer] for signing requests.
//
// Deprecated: SignHTTPRequestMiddleware is deprecated.
func NewSignHTTPRequestMiddleware(options SignHTTPRequestMiddlewareOptions) *SignHTTPRequestMiddleware {
return &SignHTTPRequestMiddleware{
credentialsProvider: options.CredentialsProvider,
signer: options.Signer,
logSigning: options.LogSigning,
}
}
// ID is the SignHTTPRequestMiddleware identifier.
//
// Deprecated: SignHTTPRequestMiddleware is deprecated.
func (s *SignHTTPRequestMiddleware) ID() string {
return "Signing"
}
// HandleFinalize will take the provided input and sign the request using the
// SigV4 authentication scheme.
//
// Deprecated: SignHTTPRequestMiddleware is deprecated.
func (s *SignHTTPRequestMiddleware) HandleFinalize(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (
out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
) {
if !haveCredentialProvider(s.credentialsProvider) {
return next.HandleFinalize(ctx, in)
}
req, ok := in.Request.(*smithyhttp.Request)
if !ok {
return out, metadata, &SigningError{Err: fmt.Errorf("unexpected request middleware type %T", in.Request)}
}
signingName, signingRegion := awsmiddleware.GetSigningName(ctx), awsmiddleware.GetSigningRegion(ctx)
payloadHash := GetPayloadHash(ctx)
if len(payloadHash) == 0 {
return out, metadata, &SigningError{Err: fmt.Errorf("computed payload hash missing from context")}
}
mctx := metrics.Context(ctx)
if mctx != nil {
if attempt, err := mctx.Data().LatestAttempt(); err == nil {
attempt.CredentialFetchStartTime = sdk.NowTime()
}
}
credentials, err := s.credentialsProvider.Retrieve(ctx)
if mctx != nil {
if attempt, err := mctx.Data().LatestAttempt(); err == nil {
attempt.CredentialFetchEndTime = sdk.NowTime()
}
}
if err != nil {
return out, metadata, &SigningError{Err: fmt.Errorf("failed to retrieve credentials: %w", err)}
}
signerOptions := []func(o *SignerOptions){
func(o *SignerOptions) {
o.Logger = middleware.GetLogger(ctx)
o.LogSigning = s.logSigning
},
}
// existing DisableURIPathEscaping is equivalent in purpose
// to authentication scheme property DisableDoubleEncoding
disableDoubleEncoding, overridden := internalauth.GetDisableDoubleEncoding(ctx)
if overridden {
signerOptions = append(signerOptions, func(o *SignerOptions) {
o.DisableURIPathEscaping = disableDoubleEncoding
})
}
if mctx != nil {
if attempt, err := mctx.Data().LatestAttempt(); err == nil {
attempt.SignStartTime = sdk.NowTime()
}
}
err = s.signer.SignHTTP(ctx, credentials, req.Request, payloadHash, signingName, signingRegion, sdk.NowTime(), signerOptions...)
if mctx != nil {
if attempt, err := mctx.Data().LatestAttempt(); err == nil {
attempt.SignEndTime = sdk.NowTime()
}
}
if err != nil {
return out, metadata, &SigningError{Err: fmt.Errorf("failed to sign http request, %w", err)}
}
ctx = awsmiddleware.SetSigningCredentials(ctx, credentials)
return next.HandleFinalize(ctx, in)
}
// StreamingEventsPayload signs input event stream messages.
type StreamingEventsPayload struct{}
// AddStreamingEventsPayload adds the streamingEventsPayload middleware to the stack.
func AddStreamingEventsPayload(stack *middleware.Stack) error {
return stack.Finalize.Add(&StreamingEventsPayload{}, middleware.Before)
}
// ID identifies the middleware.
func (s *StreamingEventsPayload) ID() string {
return computePayloadHashMiddlewareID
}
// HandleFinalize marks the input stream to be signed with SigV4.
func (s *StreamingEventsPayload) HandleFinalize(
ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
) (
out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
) {
contentSHA := GetPayloadHash(ctx)
if len(contentSHA) == 0 {
contentSHA = v4Internal.StreamingEventsPayload
}
ctx = SetPayloadHash(ctx, contentSHA)
return next.HandleFinalize(ctx, in)
}
// GetSignedRequestSignature attempts to extract the signature of the request.
// Returning an error if the request is unsigned, or unable to extract the
// signature.
func GetSignedRequestSignature(r *http.Request) ([]byte, error) {
const authHeaderSignatureElem = "Signature="
if auth := r.Header.Get(authorizationHeader); len(auth) != 0 {
ps := strings.Split(auth, ", ")
for _, p := range ps {
if idx := strings.Index(p, authHeaderSignatureElem); idx >= 0 {
sig := p[len(authHeaderSignatureElem):]
if len(sig) == 0 {
return nil, fmt.Errorf("invalid request signature authorization header")
}
return hex.DecodeString(sig)
}
}
}
if sig := r.URL.Query().Get("X-Amz-Signature"); len(sig) != 0 {
return hex.DecodeString(sig)
}
return nil, fmt.Errorf("request not signed")
}
func haveCredentialProvider(p aws.CredentialsProvider) bool {
if p == nil {
return false
}
return !aws.IsCredentialsProvider(p, (*aws.AnonymousCredentials)(nil))
}
type payloadHashKey struct{}
// GetPayloadHash retrieves the payload hash to use for signing
//
// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
// to clear all stack values.
func GetPayloadHash(ctx context.Context) (v string) {
v, _ = middleware.GetStackValue(ctx, payloadHashKey{}).(string)
return v
}
// SetPayloadHash sets the payload hash to be used for signing the request
//
// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
// to clear all stack values.
func SetPayloadHash(ctx context.Context, hash string) context.Context {
return middleware.WithStackValue(ctx, payloadHashKey{}, hash)
}

View File

@@ -1,415 +0,0 @@
package v4
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
"github.com/aws/smithy-go/logging"
"github.com/aws/smithy-go/middleware"
smithyhttp "github.com/aws/smithy-go/transport/http"
"github.com/google/go-cmp/cmp"
"github.com/versity/versitygw/aws/internal/awstesting/unit"
)
func TestComputePayloadHashMiddleware(t *testing.T) {
cases := []struct {
content io.Reader
expectedHash string
expectedErr interface{}
}{
0: {
content: func() io.Reader {
br := bytes.NewReader([]byte("some content"))
return br
}(),
expectedHash: "290f493c44f5d63d06b374d0a5abd292fae38b92cab2fae5efefe1b0e9347f56",
},
1: {
content: func() io.Reader {
return &nonSeeker{}
}(),
expectedErr: &HashComputationError{},
},
2: {
content: func() io.Reader {
return &semiSeekable{}
}(),
expectedErr: &HashComputationError{},
},
}
for i, tt := range cases {
t.Run(strconv.Itoa(i), func(t *testing.T) {
c := &ComputePayloadSHA256{}
next := middleware.FinalizeHandlerFunc(func(ctx context.Context, in middleware.FinalizeInput) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
value := GetPayloadHash(ctx)
if len(value) == 0 {
t.Fatalf("expected payload hash value to be on context")
}
if e, a := tt.expectedHash, value; e != a {
t.Errorf("expected %v, got %v", e, a)
}
return out, metadata, err
})
stream, err := smithyhttp.NewStackRequest().(*smithyhttp.Request).SetStream(tt.content)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
_, _, err = c.HandleFinalize(context.Background(), middleware.FinalizeInput{Request: stream}, next)
if err != nil && tt.expectedErr == nil {
t.Errorf("expected no error, got %v", err)
} else if err != nil && tt.expectedErr != nil {
e, a := tt.expectedErr, err
if !errors.As(a, &e) {
t.Errorf("expected error type %T, got %T", e, a)
}
} else if err == nil && tt.expectedErr != nil {
t.Errorf("expected error, got nil")
}
})
}
}
type httpSignerFunc func(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error
func (f httpSignerFunc) SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error {
return f(ctx, credentials, r, payloadHash, service, region, signingTime, optFns...)
}
func TestSignHTTPRequestMiddleware(t *testing.T) {
cases := map[string]struct {
creds aws.CredentialsProvider
hash string
logSigning bool
expectedErr interface{}
}{
"success": {
creds: unit.StubCredentialsProvider{},
hash: "0123456789abcdef",
},
"error": {
creds: unit.StubCredentialsProvider{},
hash: "",
expectedErr: &SigningError{},
},
"anonymous creds": {
creds: aws.AnonymousCredentials{},
},
"nil creds": {
creds: nil,
},
"with log signing": {
creds: unit.StubCredentialsProvider{},
hash: "0123456789abcdef",
logSigning: true,
},
}
const (
signingName = "serviceId"
signingRegion = "regionName"
)
for name, tt := range cases {
t.Run(name, func(t *testing.T) {
c := &SignHTTPRequestMiddleware{
credentialsProvider: tt.creds,
signer: httpSignerFunc(
func(ctx context.Context,
credentials aws.Credentials, r *http.Request, payloadHash string,
service string, region string, signingTime time.Time,
optFns ...func(*SignerOptions),
) error {
var options SignerOptions
for _, fn := range optFns {
fn(&options)
}
if options.Logger == nil {
t.Errorf("expect logger, got none")
}
if options.LogSigning {
options.Logger.Logf(logging.Debug, t.Name())
}
expectCreds, _ := unit.StubCredentialsProvider{}.Retrieve(context.Background())
if e, a := expectCreds, credentials; e != a {
t.Errorf("expected %v, got %v", e, a)
}
if e, a := tt.hash, payloadHash; e != a {
t.Errorf("expected %v, got %v", e, a)
}
if e, a := signingName, service; e != a {
t.Errorf("expected %v, got %v", e, a)
}
if e, a := signingRegion, region; e != a {
t.Errorf("expected %v, got %v", e, a)
}
return nil
}),
logSigning: tt.logSigning,
}
next := middleware.FinalizeHandlerFunc(func(ctx context.Context, in middleware.FinalizeInput) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
return out, metadata, err
})
ctx := awsmiddleware.SetSigningRegion(
awsmiddleware.SetSigningName(context.Background(), signingName),
signingRegion)
var loggerBuf bytes.Buffer
logger := logging.NewStandardLogger(&loggerBuf)
ctx = middleware.SetLogger(ctx, logger)
if len(tt.hash) != 0 {
ctx = SetPayloadHash(ctx, tt.hash)
}
_, _, err := c.HandleFinalize(ctx, middleware.FinalizeInput{
Request: &smithyhttp.Request{Request: &http.Request{}},
}, next)
if err != nil && tt.expectedErr == nil {
t.Errorf("expected no error, got %v", err)
} else if err != nil && tt.expectedErr != nil {
e, a := tt.expectedErr, err
if !errors.As(a, &e) {
t.Errorf("expected error type %T, got %T", e, a)
}
} else if err == nil && tt.expectedErr != nil {
t.Errorf("expected error, got nil")
}
if tt.logSigning {
if e, a := t.Name(), loggerBuf.String(); !strings.Contains(a, e) {
t.Errorf("expect %v logged in %v", e, a)
}
} else {
if loggerBuf.Len() != 0 {
t.Errorf("expect no log, got %v", loggerBuf.String())
}
}
})
}
}
func TestSwapComputePayloadSHA256ForUnsignedPayloadMiddleware(t *testing.T) {
cases := map[string]struct {
InitStep func(*middleware.Stack) error
Mutator func(*middleware.Stack) error
ExpectErr string
ExpectIDs []string
}{
"swap in place": {
InitStep: func(s *middleware.Stack) (err error) {
err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("before", nil), middleware.After)
if err != nil {
return err
}
err = AddComputePayloadSHA256Middleware(s)
if err != nil {
return err
}
err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("after", nil), middleware.After)
if err != nil {
return err
}
return nil
},
Mutator: SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
ExpectIDs: []string{
"ResolveEndpointV2",
computePayloadHashMiddlewareID, // should snap to after resolve endpoint
"before",
"after",
},
},
"already unsigned payload exists": {
InitStep: func(s *middleware.Stack) (err error) {
err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("before", nil), middleware.After)
if err != nil {
return err
}
err = AddUnsignedPayloadMiddleware(s)
if err != nil {
return err
}
err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("after", nil), middleware.After)
if err != nil {
return err
}
return nil
},
Mutator: SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
ExpectIDs: []string{
"ResolveEndpointV2",
computePayloadHashMiddlewareID,
"before",
"after",
},
},
"no compute payload": {
InitStep: func(s *middleware.Stack) (err error) {
err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("before", nil), middleware.After)
if err != nil {
return err
}
err = s.Finalize.Add(middleware.FinalizeMiddlewareFunc("after", nil), middleware.After)
if err != nil {
return err
}
return nil
},
Mutator: SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
ExpectErr: "not found, " + computePayloadHashMiddlewareID,
},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
stack := middleware.NewStack(t.Name(), smithyhttp.NewStackRequest)
stack.Finalize.Add(&nopResolveEndpoint{}, middleware.After)
if err := c.InitStep(stack); err != nil {
t.Fatalf("expect no error, got %v", err)
}
err := c.Mutator(stack)
if len(c.ExpectErr) != 0 {
if err == nil {
t.Fatalf("expect error, got none")
}
if e, a := c.ExpectErr, err.Error(); !strings.Contains(a, e) {
t.Fatalf("expect error to contain %v, got %v", e, a)
}
return
}
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if diff := cmp.Diff(c.ExpectIDs, stack.Finalize.List()); len(diff) != 0 {
t.Errorf("expect match\n%v", diff)
}
})
}
}
func TestUseDynamicPayloadSigningMiddleware(t *testing.T) {
cases := map[string]struct {
content io.Reader
url string
expectedHash string
expectedErr interface{}
}{
"TLS disabled": {
content: func() io.Reader {
br := bytes.NewReader([]byte("some content"))
return br
}(),
url: "http://localhost.com/",
expectedHash: "290f493c44f5d63d06b374d0a5abd292fae38b92cab2fae5efefe1b0e9347f56",
},
"TLS enabled": {
content: func() io.Reader {
br := bytes.NewReader([]byte("some content"))
return br
}(),
url: "https://localhost.com/",
expectedHash: "UNSIGNED-PAYLOAD",
},
}
for name, tt := range cases {
t.Run(name, func(t *testing.T) {
c := &dynamicPayloadSigningMiddleware{}
next := middleware.FinalizeHandlerFunc(func(ctx context.Context, in middleware.FinalizeInput) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
value := GetPayloadHash(ctx)
if len(value) == 0 {
t.Fatalf("expected payload hash value to be on context")
}
if e, a := tt.expectedHash, value; e != a {
t.Errorf("expected %v, got %v", e, a)
}
return out, metadata, err
})
req := smithyhttp.NewStackRequest().(*smithyhttp.Request)
req.URL, _ = url.Parse(tt.url)
stream, err := req.SetStream(tt.content)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
_, _, err = c.HandleFinalize(context.Background(), middleware.FinalizeInput{Request: stream}, next)
if err != nil && tt.expectedErr == nil {
t.Errorf("expected no error, got %v", err)
} else if err != nil && tt.expectedErr != nil {
e, a := tt.expectedErr, err
if !errors.As(a, &e) {
t.Errorf("expected error type %T, got %T", e, a)
}
} else if err == nil && tt.expectedErr != nil {
t.Errorf("expected error, got nil")
}
})
}
}
type nonSeeker struct{}
func (nonSeeker) Read(p []byte) (n int, err error) {
return 0, io.EOF
}
type semiSeekable struct {
hasSeeked bool
}
func (s *semiSeekable) Seek(offset int64, whence int) (int64, error) {
if !s.hasSeeked {
s.hasSeeked = true
return 0, nil
}
return 0, fmt.Errorf("io seek error")
}
func (*semiSeekable) Read(p []byte) (n int, err error) {
return 0, io.EOF
}
type nopResolveEndpoint struct{}
func (*nopResolveEndpoint) ID() string { return "ResolveEndpointV2" }
func (*nopResolveEndpoint) HandleFinalize(
ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
) (
out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
) {
return out, metadata, err
}
var (
_ middleware.FinalizeMiddleware = &UnsignedPayload{}
_ middleware.FinalizeMiddleware = &ComputePayloadSHA256{}
_ middleware.FinalizeMiddleware = &ContentSHA256Header{}
_ middleware.FinalizeMiddleware = &SignHTTPRequestMiddleware{}
)

View File

@@ -1,127 +0,0 @@
package v4
import (
"context"
"fmt"
"net/http"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
"github.com/aws/smithy-go/middleware"
smithyHTTP "github.com/aws/smithy-go/transport/http"
"github.com/versity/versitygw/aws/internal/sdk"
)
// HTTPPresigner is an interface to a SigV4 signer that can sign create a
// presigned URL for a HTTP requests.
type HTTPPresigner interface {
PresignHTTP(
ctx context.Context, credentials aws.Credentials, r *http.Request,
payloadHash string, service string, region string, signingTime time.Time,
optFns ...func(*SignerOptions),
) (url string, signedHeader http.Header, err error)
}
// PresignedHTTPRequest provides the URL and signed headers that are included
// in the presigned URL.
type PresignedHTTPRequest struct {
URL string
Method string
SignedHeader http.Header
}
// PresignHTTPRequestMiddlewareOptions is the options for the PresignHTTPRequestMiddleware middleware.
type PresignHTTPRequestMiddlewareOptions struct {
CredentialsProvider aws.CredentialsProvider
Presigner HTTPPresigner
LogSigning bool
}
// PresignHTTPRequestMiddleware provides the Finalize middleware for creating a
// presigned URL for an HTTP request.
//
// Will short circuit the middleware stack and not forward onto the next
// Finalize handler.
type PresignHTTPRequestMiddleware struct {
credentialsProvider aws.CredentialsProvider
presigner HTTPPresigner
logSigning bool
}
// NewPresignHTTPRequestMiddleware returns a new PresignHTTPRequestMiddleware
// initialized with the presigner.
func NewPresignHTTPRequestMiddleware(options PresignHTTPRequestMiddlewareOptions) *PresignHTTPRequestMiddleware {
return &PresignHTTPRequestMiddleware{
credentialsProvider: options.CredentialsProvider,
presigner: options.Presigner,
logSigning: options.LogSigning,
}
}
// ID provides the middleware ID.
func (*PresignHTTPRequestMiddleware) ID() string { return "PresignHTTPRequest" }
// HandleFinalize will take the provided input and create a presigned url for
// the http request using the SigV4 presign authentication scheme.
//
// Since the signed request is not a valid HTTP request
func (s *PresignHTTPRequestMiddleware) HandleFinalize(
ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler,
) (
out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
) {
req, ok := in.Request.(*smithyHTTP.Request)
if !ok {
return out, metadata, &SigningError{
Err: fmt.Errorf("unexpected request middleware type %T", in.Request),
}
}
httpReq := req.Build(ctx)
if !haveCredentialProvider(s.credentialsProvider) {
out.Result = &PresignedHTTPRequest{
URL: httpReq.URL.String(),
Method: httpReq.Method,
SignedHeader: http.Header{},
}
return out, metadata, nil
}
signingName := awsmiddleware.GetSigningName(ctx)
signingRegion := awsmiddleware.GetSigningRegion(ctx)
payloadHash := GetPayloadHash(ctx)
if len(payloadHash) == 0 {
return out, metadata, &SigningError{
Err: fmt.Errorf("computed payload hash missing from context"),
}
}
credentials, err := s.credentialsProvider.Retrieve(ctx)
if err != nil {
return out, metadata, &SigningError{
Err: fmt.Errorf("failed to retrieve credentials: %w", err),
}
}
u, h, err := s.presigner.PresignHTTP(ctx, credentials,
httpReq, payloadHash, signingName, signingRegion, sdk.NowTime(),
func(o *SignerOptions) {
o.Logger = middleware.GetLogger(ctx)
o.LogSigning = s.logSigning
})
if err != nil {
return out, metadata, &SigningError{
Err: fmt.Errorf("failed to sign http request, %w", err),
}
}
out.Result = &PresignedHTTPRequest{
URL: u,
Method: httpReq.Method,
SignedHeader: h,
}
return out, metadata, nil
}

View File

@@ -1,224 +0,0 @@
package v4
import (
"bytes"
"context"
"net/http"
"net/url"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
"github.com/aws/smithy-go/logging"
"github.com/aws/smithy-go/middleware"
smithyhttp "github.com/aws/smithy-go/transport/http"
"github.com/google/go-cmp/cmp"
"github.com/versity/versitygw/aws/internal/awstesting/unit"
)
type httpPresignerFunc func(
ctx context.Context, credentials aws.Credentials, r *http.Request,
payloadHash string, service string, region string, signingTime time.Time,
optFns ...func(*SignerOptions),
) (url string, signedHeader http.Header, err error)
func (f httpPresignerFunc) PresignHTTP(
ctx context.Context, credentials aws.Credentials, r *http.Request,
payloadHash string, service string, region string, signingTime time.Time,
optFns ...func(*SignerOptions),
) (
url string, signedHeader http.Header, err error,
) {
return f(ctx, credentials, r, payloadHash, service, region, signingTime, optFns...)
}
func TestPresignHTTPRequestMiddleware(t *testing.T) {
cases := map[string]struct {
Request *http.Request
Creds aws.CredentialsProvider
PayloadHash string
LogSigning bool
ExpectResult *PresignedHTTPRequest
ExpectErr string
}{
"success": {
Request: &http.Request{
URL: func() *url.URL {
u, _ := url.Parse("https://example.aws/path?query=foo")
return u
}(),
Header: http.Header{},
},
Creds: unit.StubCredentialsProvider{},
PayloadHash: "0123456789abcdef",
ExpectResult: &PresignedHTTPRequest{
URL: "https://example.aws/path?query=foo",
SignedHeader: http.Header{},
},
},
"error": {
Request: func() *http.Request {
return &http.Request{}
}(),
Creds: unit.StubCredentialsProvider{},
PayloadHash: "",
ExpectErr: "failed to sign request",
},
"anonymous creds": {
Request: &http.Request{
URL: func() *url.URL {
u, _ := url.Parse("https://example.aws/path?query=foo")
return u
}(),
Header: http.Header{},
},
Creds: unit.StubCredentialsProvider{},
PayloadHash: "",
ExpectErr: "failed to sign request",
ExpectResult: &PresignedHTTPRequest{
URL: "https://example.aws/path?query=foo",
SignedHeader: http.Header{},
},
},
"nil creds": {
Request: &http.Request{
URL: func() *url.URL {
u, _ := url.Parse("https://example.aws/path?query=foo")
return u
}(),
Header: http.Header{},
},
Creds: nil,
ExpectResult: &PresignedHTTPRequest{
URL: "https://example.aws/path?query=foo",
SignedHeader: http.Header{},
},
},
"with log signing": {
Request: &http.Request{
URL: func() *url.URL {
u, _ := url.Parse("https://example.aws/path?query=foo")
return u
}(),
Header: http.Header{},
},
Creds: unit.StubCredentialsProvider{},
PayloadHash: "0123456789abcdef",
ExpectResult: &PresignedHTTPRequest{
URL: "https://example.aws/path?query=foo",
SignedHeader: http.Header{},
},
LogSigning: true,
},
}
const (
signingName = "serviceId"
signingRegion = "regionName"
)
for name, c := range cases {
t.Run(name, func(t *testing.T) {
m := &PresignHTTPRequestMiddleware{
credentialsProvider: c.Creds,
presigner: httpPresignerFunc(func(
ctx context.Context, credentials aws.Credentials, r *http.Request,
payloadHash string, service string, region string, signingTime time.Time,
optFns ...func(*SignerOptions),
) (url string, signedHeader http.Header, err error) {
var options SignerOptions
for _, fn := range optFns {
fn(&options)
}
if options.Logger == nil {
t.Errorf("expect logger, got none")
}
if options.LogSigning {
options.Logger.Logf(logging.Debug, t.Name())
}
if !haveCredentialProvider(c.Creds) {
t.Errorf("expect presigner not to be called for not credentials provider")
}
expectCreds, _ := unit.StubCredentialsProvider{}.Retrieve(context.Background())
if e, a := expectCreds, credentials; e != a {
t.Errorf("expected %v, got %v", e, a)
}
if e, a := c.PayloadHash, payloadHash; e != a {
t.Errorf("expected %v, got %v", e, a)
}
if e, a := signingName, service; e != a {
t.Errorf("expected %v, got %v", e, a)
}
if e, a := signingRegion, region; e != a {
t.Errorf("expected %v, got %v", e, a)
}
return c.ExpectResult.URL, c.ExpectResult.SignedHeader, nil
}),
logSigning: c.LogSigning,
}
next := middleware.FinalizeHandlerFunc(
func(ctx context.Context, in middleware.FinalizeInput) (
out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
) {
t.Errorf("expect next handler not to be called")
return out, metadata, err
})
ctx := awsmiddleware.SetSigningRegion(
awsmiddleware.SetSigningName(context.Background(), signingName),
signingRegion)
var loggerBuf bytes.Buffer
logger := logging.NewStandardLogger(&loggerBuf)
ctx = middleware.SetLogger(ctx, logger)
if len(c.PayloadHash) != 0 {
ctx = SetPayloadHash(ctx, c.PayloadHash)
}
result, _, err := m.HandleFinalize(ctx, middleware.FinalizeInput{
Request: &smithyhttp.Request{
Request: c.Request,
},
}, next)
if len(c.ExpectErr) != 0 {
if err == nil {
t.Fatalf("expect error, got none")
}
if e, a := c.ExpectErr, err.Error(); !strings.Contains(a, e) {
t.Fatalf("expect error to contain %v, got %v", e, a)
}
return
}
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if diff := cmp.Diff(c.ExpectResult, result.Result); len(diff) != 0 {
t.Errorf("expect result match\n%v", diff)
}
if c.LogSigning {
if e, a := t.Name(), loggerBuf.String(); !strings.Contains(a, e) {
t.Errorf("expect %v logged in %v", e, a)
}
} else {
if loggerBuf.Len() != 0 {
t.Errorf("expect no log, got %v", loggerBuf.String())
}
}
})
}
}
var (
_ middleware.FinalizeMiddleware = &PresignHTTPRequestMiddleware{}
)

View File

@@ -1,87 +0,0 @@
package v4
import (
"context"
"crypto/sha256"
"encoding/hex"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
)
// EventStreamSigner is an AWS EventStream protocol signer.
type EventStreamSigner interface {
GetSignature(ctx context.Context, headers, payload []byte, signingTime time.Time, optFns ...func(*StreamSignerOptions)) ([]byte, error)
}
// StreamSignerOptions is the configuration options for StreamSigner.
type StreamSignerOptions struct{}
// StreamSigner implements Signature Version 4 (SigV4) signing of event stream encoded payloads.
type StreamSigner struct {
options StreamSignerOptions
credentials aws.Credentials
service string
region string
prevSignature []byte
signingKeyDeriver *v4Internal.SigningKeyDeriver
}
// NewStreamSigner returns a new AWS EventStream protocol signer.
func NewStreamSigner(credentials aws.Credentials, service, region string, seedSignature []byte, optFns ...func(*StreamSignerOptions)) *StreamSigner {
o := StreamSignerOptions{}
for _, fn := range optFns {
fn(&o)
}
return &StreamSigner{
options: o,
credentials: credentials,
service: service,
region: region,
signingKeyDeriver: v4Internal.NewSigningKeyDeriver(),
prevSignature: seedSignature,
}
}
// GetSignature signs the provided header and payload bytes.
func (s *StreamSigner) GetSignature(ctx context.Context, headers, payload []byte, signingTime time.Time, optFns ...func(*StreamSignerOptions)) ([]byte, error) {
options := s.options
for _, fn := range optFns {
fn(&options)
}
prevSignature := s.prevSignature
st := v4Internal.NewSigningTime(signingTime)
sigKey := s.signingKeyDeriver.DeriveKey(s.credentials, s.service, s.region, st)
scope := v4Internal.BuildCredentialScope(st, s.region, s.service)
stringToSign := s.buildEventStreamStringToSign(headers, payload, prevSignature, scope, &st)
signature := v4Internal.HMACSHA256(sigKey, []byte(stringToSign))
s.prevSignature = signature
return signature, nil
}
func (s *StreamSigner) buildEventStreamStringToSign(headers, payload, previousSignature []byte, credentialScope string, signingTime *v4Internal.SigningTime) string {
hash := sha256.New()
return strings.Join([]string{
"AWS4-HMAC-SHA256-PAYLOAD",
signingTime.TimeFormat(),
credentialScope,
hex.EncodeToString(previousSignature),
hex.EncodeToString(makeHash(hash, headers)),
hex.EncodeToString(makeHash(hash, payload)),
}, "\n")
}

View File

@@ -1,565 +0,0 @@
// Package v4 implements signing for AWS V4 signer
//
// Provides request signing for request that need to be signed with
// AWS V4 Signatures.
//
// # Standalone Signer
//
// Generally using the signer outside of the SDK should not require any additional
//
// The signer does this by taking advantage of the URL.EscapedPath method. If your request URI requires
//
// additional escaping you many need to use the URL.Opaque to define what the raw URI should be sent
// to the service as.
//
// The signer will first check the URL.Opaque field, and use its value if set.
// The signer does require the URL.Opaque field to be set in the form of:
//
// "//<hostname>/<path>"
//
// // e.g.
// "//example.com/some/path"
//
// The leading "//" and hostname are required or the URL.Opaque escaping will
// not work correctly.
//
// If URL.Opaque is not set the signer will fallback to the URL.EscapedPath()
// method and using the returned value.
//
// AWS v4 signature validation requires that the canonical string's URI path
// element must be the URI escaped form of the HTTP request's path.
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
//
// The Go HTTP client will perform escaping automatically on the request. Some
// of these escaping may cause signature validation errors because the HTTP
// request differs from the URI path or query that the signature was generated.
// https://golang.org/pkg/net/url/#URL.EscapedPath
//
// Because of this, it is recommended that when using the signer outside of the
// SDK that explicitly escaping the request prior to being signed is preferable,
// and will help prevent signature validation errors. This can be done by setting
// the URL.Opaque or URL.RawPath. The SDK will use URL.Opaque first and then
// call URL.EscapedPath() if Opaque is not set.
//
// Test `TestStandaloneSign` provides a complete example of using the signer
// outside of the SDK and pre-escaping the URI path.
package v4
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"hash"
"net/http"
"net/textproto"
"net/url"
"slices"
"sort"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/smithy-go/encoding/httpbinding"
"github.com/aws/smithy-go/logging"
v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
)
const (
signingAlgorithm = "AWS4-HMAC-SHA256"
authorizationHeader = "Authorization"
// Version of signing v4
Version = "SigV4"
)
// HTTPSigner is an interface to a SigV4 signer that can sign HTTP requests
type HTTPSigner interface {
SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*SignerOptions)) error
}
type keyDerivator interface {
DeriveKey(credential aws.Credentials, service, region string, signingTime v4Internal.SigningTime) []byte
}
// SignerOptions is the SigV4 Signer options.
type SignerOptions struct {
// Disables the Signer's moving HTTP header key/value pairs from the HTTP
// request header to the request's query string. This is most commonly used
// with pre-signed requests preventing headers from being added to the
// request's query string.
DisableHeaderHoisting bool
// Disables the automatic escaping of the URI path of the request for the
// siganture's canonical string's path. For services that do not need additional
// escaping then use this to disable the signer escaping the path.
//
// S3 is an example of a service that does not need additional escaping.
//
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
DisableURIPathEscaping bool
// The logger to send log messages to.
Logger logging.Logger
// Enable logging of signed requests.
// This will enable logging of the canonical request, the string to sign, and for presigning the subsequent
// presigned URL.
LogSigning bool
// Disables setting the session token on the request as part of signing
// through X-Amz-Security-Token. This is needed for variations of v4 that
// present the token elsewhere.
DisableSessionToken bool
}
// Signer applies AWS v4 signing to given request. Use this to sign requests
// that need to be signed with AWS V4 Signatures.
type Signer struct {
options SignerOptions
keyDerivator keyDerivator
}
// NewSigner returns a new SigV4 Signer
func NewSigner(optFns ...func(signer *SignerOptions)) *Signer {
options := SignerOptions{}
for _, fn := range optFns {
fn(&options)
}
return &Signer{options: options, keyDerivator: v4Internal.NewSigningKeyDeriver()}
}
type httpSigner struct {
Request *http.Request
ServiceName string
Region string
Time v4Internal.SigningTime
Credentials aws.Credentials
KeyDerivator keyDerivator
IsPreSign bool
SignedHdrs []string
PayloadHash string
DisableHeaderHoisting bool
DisableURIPathEscaping bool
DisableSessionToken bool
}
func (s *httpSigner) Build() (signedRequest, error) {
req := s.Request
query := req.URL.Query()
headers := req.Header
s.setRequiredSigningFields(headers, query)
// Sort Each Query Key's Values
for key := range query {
sort.Strings(query[key])
}
v4Internal.SanitizeHostForHeader(req)
credentialScope := s.buildCredentialScope()
credentialStr := s.Credentials.AccessKeyID + "/" + credentialScope
if s.IsPreSign {
query.Set(v4Internal.AmzCredentialKey, credentialStr)
}
unsignedHeaders := headers
if s.IsPreSign && !s.DisableHeaderHoisting {
var urlValues url.Values
urlValues, unsignedHeaders = buildQuery(v4Internal.AllowedQueryHoisting, headers)
for k := range urlValues {
query[k] = urlValues[k]
}
}
host := req.URL.Host
if len(req.Host) > 0 {
host = req.Host
}
signedHeaders, signedHeadersStr, canonicalHeaderStr := s.buildCanonicalHeaders(host, v4Internal.IgnoredHeaders, unsignedHeaders, s.Request.ContentLength)
if s.IsPreSign {
query.Set(v4Internal.AmzSignedHeadersKey, signedHeadersStr)
}
var rawQuery strings.Builder
rawQuery.WriteString(strings.Replace(query.Encode(), "+", "%20", -1))
canonicalURI := v4Internal.GetURIPath(req.URL)
if !s.DisableURIPathEscaping {
canonicalURI = httpbinding.EscapePath(canonicalURI, false)
}
canonicalString := s.buildCanonicalString(
req.Method,
canonicalURI,
rawQuery.String(),
signedHeadersStr,
canonicalHeaderStr,
)
strToSign := s.buildStringToSign(credentialScope, canonicalString)
signingSignature, err := s.buildSignature(strToSign)
if err != nil {
return signedRequest{}, err
}
if s.IsPreSign {
rawQuery.WriteString("&X-Amz-Signature=")
rawQuery.WriteString(signingSignature)
} else {
headers[authorizationHeader] = append(headers[authorizationHeader][:0], buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature))
}
req.URL.RawQuery = rawQuery.String()
return signedRequest{
Request: req,
SignedHeaders: signedHeaders,
CanonicalString: canonicalString,
StringToSign: strToSign,
PreSigned: s.IsPreSign,
}, nil
}
func buildAuthorizationHeader(credentialStr, signedHeadersStr, signingSignature string) string {
const credential = "Credential="
const signedHeaders = "SignedHeaders="
const signature = "Signature="
const commaSpace = ", "
var parts strings.Builder
parts.Grow(len(signingAlgorithm) + 1 +
len(credential) + len(credentialStr) + 2 +
len(signedHeaders) + len(signedHeadersStr) + 2 +
len(signature) + len(signingSignature),
)
parts.WriteString(signingAlgorithm)
parts.WriteRune(' ')
parts.WriteString(credential)
parts.WriteString(credentialStr)
parts.WriteString(commaSpace)
parts.WriteString(signedHeaders)
parts.WriteString(signedHeadersStr)
parts.WriteString(commaSpace)
parts.WriteString(signature)
parts.WriteString(signingSignature)
return parts.String()
}
// SignHTTP signs AWS v4 requests with the provided payload hash, service name, region the
// request is made to, and time the request is signed at. The signTime allows
// you to specify that a request is signed for the future, and cannot be
// used until then.
//
// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
// must be provided. Even if the request has no payload (aka body). If the
// request has no payload you should use the hex encoded SHA-256 of an empty
// string as the payloadHash value.
//
// "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
//
// Some services such as Amazon S3 accept alternative values for the payload
// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
// included in the request signature.
//
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
//
// Sign differs from Presign in that it will sign the request using HTTP
// header values. This type of signing is intended for http.Request values that
// will not be shared, or are shared in a way the header values on the request
// will not be lost.
//
// The passed in request will be modified in place.
func (s Signer) SignHTTP(ctx context.Context, credentials aws.Credentials, r *http.Request, payloadHash string, service string, region string, signingTime time.Time, signedHdrs []string, optFns ...func(options *SignerOptions)) error {
options := s.options
for _, fn := range optFns {
fn(&options)
}
signer := &httpSigner{
Request: r,
PayloadHash: payloadHash,
ServiceName: service,
Region: region,
Credentials: credentials,
Time: v4Internal.NewSigningTime(signingTime.UTC()),
DisableHeaderHoisting: options.DisableHeaderHoisting,
DisableURIPathEscaping: options.DisableURIPathEscaping,
DisableSessionToken: options.DisableSessionToken,
KeyDerivator: s.keyDerivator,
SignedHdrs: signedHdrs,
}
signedRequest, err := signer.Build()
if err != nil {
return err
}
logSigningInfo(ctx, options, &signedRequest, false)
return nil
}
// PresignHTTP signs AWS v4 requests with the payload hash, service name, region
// the request is made to, and time the request is signed at. The signTime
// allows you to specify that a request is signed for the future, and cannot
// be used until then.
//
// Returns the signed URL and the map of HTTP headers that were included in the
// signature or an error if signing the request failed. For presigned requests
// these headers and their values must be included on the HTTP request when it
// is made. This is helpful to know what header values need to be shared with
// the party the presigned request will be distributed to.
//
// The payloadHash is the hex encoded SHA-256 hash of the request payload, and
// must be provided. Even if the request has no payload (aka body). If the
// request has no payload you should use the hex encoded SHA-256 of an empty
// string as the payloadHash value.
//
// "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
//
// Some services such as Amazon S3 accept alternative values for the payload
// hash, such as "UNSIGNED-PAYLOAD" for requests where the body will not be
// included in the request signature.
//
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
//
// PresignHTTP differs from SignHTTP in that it will sign the request using
// query string instead of header values. This allows you to share the
// Presigned Request's URL with third parties, or distribute it throughout your
// system with minimal dependencies.
//
// PresignHTTP will not set the expires time of the presigned request
// automatically. To specify the expire duration for a request add the
// "X-Amz-Expires" query parameter on the request with the value as the
// duration in seconds the presigned URL should be considered valid for. This
// parameter is not used by all AWS services, and is most notable used by
// Amazon S3 APIs.
//
// expires := 20 * time.Minute
// query := req.URL.Query()
// query.Set("X-Amz-Expires", strconv.FormatInt(int64(expires/time.Second), 10))
// req.URL.RawQuery = query.Encode()
//
// This method does not modify the provided request.
func (s *Signer) PresignHTTP(
ctx context.Context, credentials aws.Credentials, r *http.Request,
payloadHash string, service string, region string, signingTime time.Time,
signedHdrs []string,
optFns ...func(*SignerOptions),
) (signedURI string, signedHeaders http.Header, err error) {
options := s.options
for _, fn := range optFns {
fn(&options)
}
signer := &httpSigner{
Request: r.Clone(r.Context()),
PayloadHash: payloadHash,
ServiceName: service,
Region: region,
Credentials: credentials,
Time: v4Internal.NewSigningTime(signingTime.UTC()),
IsPreSign: true,
DisableHeaderHoisting: options.DisableHeaderHoisting,
DisableURIPathEscaping: options.DisableURIPathEscaping,
DisableSessionToken: options.DisableSessionToken,
KeyDerivator: s.keyDerivator,
SignedHdrs: signedHdrs,
}
signedRequest, err := signer.Build()
if err != nil {
return "", nil, err
}
logSigningInfo(ctx, options, &signedRequest, true)
signedHeaders = make(http.Header)
// For the signed headers we canonicalize the header keys in the returned map.
// This avoids situations where can standard library double headers like host header. For example the standard
// library will set the Host header, even if it is present in lower-case form.
for k, v := range signedRequest.SignedHeaders {
key := textproto.CanonicalMIMEHeaderKey(k)
signedHeaders[key] = append(signedHeaders[key], v...)
}
return signedRequest.Request.URL.String(), signedHeaders, nil
}
func (s *httpSigner) buildCredentialScope() string {
return v4Internal.BuildCredentialScope(s.Time, s.Region, s.ServiceName)
}
func buildQuery(r v4Internal.Rule, header http.Header) (url.Values, http.Header) {
query := url.Values{}
unsignedHeaders := http.Header{}
for k, h := range header {
if r.IsValid(k) {
query[k] = h
} else {
unsignedHeaders[k] = h
}
}
return query, unsignedHeaders
}
func (s *httpSigner) buildCanonicalHeaders(host string, rule v4Internal.Rule, header http.Header, length int64) (signed http.Header, signedHeaders, canonicalHeadersStr string) {
signed = make(http.Header)
var headers []string
const hostHeader = "host"
headers = append(headers, hostHeader)
signed[hostHeader] = append(signed[hostHeader], host)
const contentLengthHeader = "content-length"
if slices.Contains(s.SignedHdrs, contentLengthHeader) {
headers = append(headers, contentLengthHeader)
signed[contentLengthHeader] = append(signed[contentLengthHeader], strconv.FormatInt(length, 10))
}
for k, v := range header {
if !rule.IsValid(k) {
continue // ignored header
}
if strings.EqualFold(k, contentLengthHeader) {
// prevent signing already handled content-length header.
continue
}
lowerCaseKey := strings.ToLower(k)
if _, ok := signed[lowerCaseKey]; ok {
// include additional values
signed[lowerCaseKey] = append(signed[lowerCaseKey], v...)
continue
}
headers = append(headers, lowerCaseKey)
signed[lowerCaseKey] = v
}
sort.Strings(headers)
signedHeaders = strings.Join(headers, ";")
var canonicalHeaders strings.Builder
n := len(headers)
const colon = ':'
for i := 0; i < n; i++ {
if headers[i] == hostHeader {
canonicalHeaders.WriteString(hostHeader)
canonicalHeaders.WriteRune(colon)
canonicalHeaders.WriteString(v4Internal.StripExcessSpaces(host))
} else {
canonicalHeaders.WriteString(headers[i])
canonicalHeaders.WriteRune(colon)
// Trim out leading, trailing, and dedup inner spaces from signed header values.
values := signed[headers[i]]
for j, v := range values {
cleanedValue := strings.TrimSpace(v4Internal.StripExcessSpaces(v))
canonicalHeaders.WriteString(cleanedValue)
if j < len(values)-1 {
canonicalHeaders.WriteRune(',')
}
}
}
canonicalHeaders.WriteRune('\n')
}
canonicalHeadersStr = canonicalHeaders.String()
return signed, signedHeaders, canonicalHeadersStr
}
func (s *httpSigner) buildCanonicalString(method, uri, query, signedHeaders, canonicalHeaders string) string {
return strings.Join([]string{
method,
uri,
query,
canonicalHeaders,
signedHeaders,
s.PayloadHash,
}, "\n")
}
func (s *httpSigner) buildStringToSign(credentialScope, canonicalRequestString string) string {
return strings.Join([]string{
signingAlgorithm,
s.Time.TimeFormat(),
credentialScope,
hex.EncodeToString(makeHash(sha256.New(), []byte(canonicalRequestString))),
}, "\n")
}
func makeHash(hash hash.Hash, b []byte) []byte {
hash.Reset()
hash.Write(b)
return hash.Sum(nil)
}
func (s *httpSigner) buildSignature(strToSign string) (string, error) {
key := s.KeyDerivator.DeriveKey(s.Credentials, s.ServiceName, s.Region, s.Time)
return hex.EncodeToString(v4Internal.HMACSHA256(key, []byte(strToSign))), nil
}
func (s *httpSigner) setRequiredSigningFields(headers http.Header, query url.Values) {
amzDate := s.Time.TimeFormat()
if s.IsPreSign {
query.Set(v4Internal.AmzAlgorithmKey, signingAlgorithm)
sessionToken := s.Credentials.SessionToken
if !s.DisableSessionToken && len(sessionToken) > 0 {
query.Set("X-Amz-Security-Token", sessionToken)
}
query.Set(v4Internal.AmzDateKey, amzDate)
return
}
headers[v4Internal.AmzDateKey] = append(headers[v4Internal.AmzDateKey][:0], amzDate)
if !s.DisableSessionToken && len(s.Credentials.SessionToken) > 0 {
headers[v4Internal.AmzSecurityTokenKey] = append(headers[v4Internal.AmzSecurityTokenKey][:0], s.Credentials.SessionToken)
}
}
func logSigningInfo(ctx context.Context, options SignerOptions, request *signedRequest, isPresign bool) {
if !options.LogSigning {
return
}
signedURLMsg := ""
if isPresign {
signedURLMsg = fmt.Sprintf(logSignedURLMsg, request.Request.URL.String())
}
logger := logging.WithContext(ctx, options.Logger)
logger.Logf(logging.Debug, logSignInfoMsg, request.CanonicalString, request.StringToSign, signedURLMsg)
}
type signedRequest struct {
Request *http.Request
SignedHeaders http.Header
CanonicalString string
StringToSign string
PreSigned bool
}
const logSignInfoMsg = `Request Signature:
---[ CANONICAL STRING ]-----------------------------
%s
---[ STRING TO SIGN ]--------------------------------
%s%s
-----------------------------------------------------`
const logSignedURLMsg = `
---[ SIGNED URL ]------------------------------------
%s`

View File

@@ -1,359 +0,0 @@
package v4
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/google/go-cmp/cmp"
v4Internal "github.com/versity/versitygw/aws/signer/internal/v4"
)
var testCredentials = aws.Credentials{AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "SESSION"}
func buildRequest(serviceName, region, body string) (*http.Request, string) {
reader := strings.NewReader(body)
return buildRequestWithBodyReader(serviceName, region, reader)
}
func buildRequestWithBodyReader(serviceName, region string, body io.Reader) (*http.Request, string) {
var bodyLen int
type lenner interface {
Len() int
}
if lr, ok := body.(lenner); ok {
bodyLen = lr.Len()
}
endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
req, _ := http.NewRequest("POST", endpoint, body)
req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()"
req.Header.Set("X-Amz-Target", "prefix.Operation")
req.Header.Set("Content-Type", "application/x-amz-json-1.0")
if bodyLen > 0 {
req.ContentLength = int64(bodyLen)
}
req.Header.Set("X-Amz-Meta-Other-Header", "some-value=!@#$%^&* (+)")
req.Header.Add("X-Amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
req.Header.Add("X-amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)")
h := sha256.New()
_, _ = io.Copy(h, body)
payloadHash := hex.EncodeToString(h.Sum(nil))
return req, payloadHash
}
func TestPresignRequest(t *testing.T) {
req, body := buildRequest("dynamodb", "us-east-1", "{}")
query := req.URL.Query()
query.Set("X-Amz-Expires", "300")
req.URL.RawQuery = query.Encode()
signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
signer := NewSigner()
signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
expectedDate := "19700101T000000Z"
expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
expectedSig := "122f0b9e091e4ba84286097e2b3404a1f1f4c4aad479adda95b7dff0ccbe5581"
expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
expectedTarget := "prefix.Operation"
q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
if err != nil {
t.Errorf("expect no error, got %v", err)
}
if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
t.Errorf("expect %v to be empty", a)
}
if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
for _, h := range strings.Split(expectedHeaders, ";") {
v := headers.Get(h)
if len(v) == 0 {
t.Errorf("expect %v, to be present in header map", h)
}
}
}
func TestPresignBodyWithArrayRequest(t *testing.T) {
req, body := buildRequest("dynamodb", "us-east-1", "{}")
req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
query := req.URL.Query()
query.Set("X-Amz-Expires", "300")
req.URL.RawQuery = query.Encode()
signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
signer := NewSigner()
signed, headers, err := signer.PresignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
q, err := url.ParseQuery(signed[strings.Index(signed, "?"):])
if err != nil {
t.Errorf("expect no error, got %v", err)
}
expectedDate := "19700101T000000Z"
expectedHeaders := "content-length;content-type;host;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore"
expectedSig := "e3ac55addee8711b76c6d608d762cff285fe8b627a057f8b5ec9268cf82c08b1"
expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request"
expectedTarget := "prefix.Operation"
if e, a := expectedSig, q.Get("X-Amz-Signature"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedCred, q.Get("X-Amz-Credential"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedHeaders, q.Get("X-Amz-SignedHeaders"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if a := q.Get("X-Amz-Meta-Other-Header"); len(a) != 0 {
t.Errorf("expect %v to be empty, was not", a)
}
if e, a := expectedTarget, q.Get("X-Amz-Target"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
for _, h := range strings.Split(expectedHeaders, ";") {
v := headers.Get(h)
if len(v) == 0 {
t.Errorf("expect %v, to be present in header map", h)
}
}
}
func TestSignRequest(t *testing.T) {
req, body := buildRequest("dynamodb", "us-east-1", "{}")
signer := NewSigner()
signedHdrs := []string{"content-length", "content-type", "host", "x-amz-date", "x-amz-meta-other-header", "x-amz-meta-other-header_with_underscore", "x-amz-security-token", "x-amz-target"}
err := signer.SignHTTP(context.Background(), testCredentials, req, body, "dynamodb", "us-east-1", time.Unix(0, 0), signedHdrs)
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
expectedDate := "19700101T000000Z"
expectedSig := "AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/dynamodb/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-meta-other-header;x-amz-meta-other-header_with_underscore;x-amz-security-token;x-amz-target, Signature=a518299330494908a70222cec6899f6f32f297f8595f6df1776d998936652ad9"
q := req.Header
if e, a := expectedSig, q.Get("Authorization"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
if e, a := expectedDate, q.Get("X-Amz-Date"); e != a {
t.Errorf("expect %v, got %v", e, a)
}
}
func TestBuildCanonicalRequest(t *testing.T) {
req, _ := buildRequest("dynamodb", "us-east-1", "{}")
req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
ctx := &httpSigner{
ServiceName: "dynamodb",
Region: "us-east-1",
Request: req,
Time: v4Internal.NewSigningTime(time.Now()),
KeyDerivator: v4Internal.NewSigningKeyDeriver(),
}
build, err := ctx.Build()
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
expected := "https://example.org/bucket/key-._~,!@#$%^&*()?Foo=a&Foo=m&Foo=o&Foo=z"
if e, a := expected, build.Request.URL.String(); e != a {
t.Errorf("expect %v, got %v", e, a)
}
}
func TestSigner_SignHTTP_NoReplaceRequestBody(t *testing.T) {
req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
req.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
s := NewSigner()
origBody := req.Body
err := s.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if req.Body != origBody {
t.Errorf("expect request body to not be chagned")
}
}
func TestRequestHost(t *testing.T) {
req, _ := buildRequest("dynamodb", "us-east-1", "{}")
req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
req.Host = "myhost"
query := req.URL.Query()
query.Set("X-Amz-Expires", "5")
req.URL.RawQuery = query.Encode()
ctx := &httpSigner{
ServiceName: "dynamodb",
Region: "us-east-1",
Request: req,
Time: v4Internal.NewSigningTime(time.Now()),
KeyDerivator: v4Internal.NewSigningKeyDeriver(),
}
build, err := ctx.Build()
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if !strings.Contains(build.CanonicalString, "host:"+req.Host) {
t.Errorf("canonical host header invalid")
}
}
func TestSign_buildCanonicalHeadersContentLengthPresent(t *testing.T) {
body := `{"description": "this is a test"}`
req, _ := buildRequest("dynamodb", "us-east-1", body)
req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a"
req.Host = "myhost"
contentLength := fmt.Sprintf("%d", len([]byte(body)))
req.Header.Add("Content-Length", contentLength)
query := req.URL.Query()
query.Set("X-Amz-Expires", "5")
req.URL.RawQuery = query.Encode()
ctx := &httpSigner{
ServiceName: "dynamodb",
Region: "us-east-1",
Request: req,
Time: v4Internal.NewSigningTime(time.Now()),
KeyDerivator: v4Internal.NewSigningKeyDeriver(),
}
_, err := ctx.Build()
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
//if !strings.Contains(build.CanonicalString, "content-length:"+contentLength+"\n") {
// t.Errorf("canonical header content-length invalid")
//}
}
func TestSign_buildCanonicalHeaders(t *testing.T) {
serviceName := "mockAPI"
region := "mock-region"
endpoint := "https://" + serviceName + "." + region + ".amazonaws.com"
req, err := http.NewRequest("POST", endpoint, nil)
if err != nil {
t.Fatalf("failed to create request, %v", err)
}
req.Header.Set("FooInnerSpace", " inner space ")
req.Header.Set("FooLeadingSpace", " leading-space")
req.Header.Add("FooMultipleSpace", "no-space")
req.Header.Add("FooMultipleSpace", "\ttab-space")
req.Header.Add("FooMultipleSpace", "trailing-space ")
req.Header.Set("FooNoSpace", "no-space")
req.Header.Set("FooTabSpace", "\ttab-space\t")
req.Header.Set("FooTrailingSpace", "trailing-space ")
req.Header.Set("FooWrappedSpace", " wrapped-space ")
ctx := &httpSigner{
ServiceName: serviceName,
Region: region,
Request: req,
Time: v4Internal.NewSigningTime(time.Date(2021, 10, 20, 12, 42, 0, 0, time.UTC)),
KeyDerivator: v4Internal.NewSigningKeyDeriver(),
}
build, err := ctx.Build()
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
expectCanonicalString := strings.Join([]string{
`POST`,
`/`,
``,
`fooinnerspace:inner space`,
`fooleadingspace:leading-space`,
`foomultiplespace:no-space,tab-space,trailing-space`,
`foonospace:no-space`,
`footabspace:tab-space`,
`footrailingspace:trailing-space`,
`foowrappedspace:wrapped-space`,
`host:mockAPI.mock-region.amazonaws.com`,
`x-amz-date:20211020T124200Z`,
``,
`fooinnerspace;fooleadingspace;foomultiplespace;foonospace;footabspace;footrailingspace;foowrappedspace;host;x-amz-date`,
``,
}, "\n")
if diff := cmp.Diff(expectCanonicalString, build.CanonicalString); diff != "" {
t.Errorf("expect match, got\n%s", diff)
}
}
func BenchmarkPresignRequest(b *testing.B) {
signer := NewSigner()
req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
query := req.URL.Query()
query.Set("X-Amz-Expires", "5")
req.URL.RawQuery = query.Encode()
for i := 0; i < b.N; i++ {
signer.PresignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
}
}
func BenchmarkSignRequest(b *testing.B) {
signer := NewSigner()
req, bodyHash := buildRequest("dynamodb", "us-east-1", "{}")
for i := 0; i < b.N; i++ {
signer.SignHTTP(context.Background(), testCredentials, req, bodyHash, "dynamodb", "us-east-1", time.Now(), []string{})
}
}

View File

@@ -1 +0,0 @@
checks = []

File diff suppressed because it is too large Load Diff

View File

@@ -1,63 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package azure
import (
"errors"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/versity/versitygw/s3err"
)
// Parses azure ResponseError into AWS APIError
func azureErrToS3Err(apiErr error) error {
var azErr *azcore.ResponseError
if !errors.As(apiErr, &azErr) {
return apiErr
}
return azErrToS3err(azErr)
}
func azErrToS3err(azErr *azcore.ResponseError) s3err.APIError {
switch azErr.ErrorCode {
case "ContainerAlreadyExists":
return s3err.GetAPIError(s3err.ErrBucketAlreadyExists)
case "InvalidResourceName", "ContainerNotFound":
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
case "BlobNotFound":
return s3err.GetAPIError(s3err.ErrNoSuchKey)
case "TagsTooLarge":
return s3err.GetAPIError(s3err.ErrInvalidTag)
case "Requested Range Not Satisfiable":
return s3err.GetAPIError(s3err.ErrInvalidRange)
}
return s3err.APIError{
Code: azErr.ErrorCode,
Description: azErr.RawResponse.Status,
HTTPStatusCode: azErr.StatusCode,
}
}
func parseMpError(mpErr error) error {
err := azureErrToS3Err(mpErr)
serr, ok := err.(s3err.APIError)
if !ok || serr.Code != "NoSuchKey" {
return mpErr
}
return s3err.GetAPIError(s3err.ErrNoSuchUpload)
}

View File

@@ -15,7 +15,6 @@
package backend
import (
"bufio"
"context"
"fmt"
"io"
@@ -23,7 +22,6 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
"github.com/versity/versitygw/s3select"
)
//go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
@@ -35,11 +33,9 @@ type Backend interface {
ListBuckets(_ context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error)
HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error)
CreateBucket(_ context.Context, _ *s3.CreateBucketInput, defaultACL []byte) error
CreateBucket(context.Context, *s3.CreateBucketInput) error
PutBucketAcl(_ context.Context, bucket string, data []byte) error
DeleteBucket(context.Context, *s3.DeleteBucketInput) error
PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error
GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
// multipart operations
CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
@@ -62,18 +58,12 @@ type Backend interface {
DeleteObject(context.Context, *s3.DeleteObjectInput) error
DeleteObjects(context.Context, *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error)
PutObjectAcl(context.Context, *s3.PutObjectAclInput) error
ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
// special case object operations
RestoreObject(context.Context, *s3.RestoreObjectInput) error
SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
// bucket tagging operations
GetBucketTagging(_ context.Context, bucket string) (map[string]string, error)
PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error
DeleteBucketTagging(_ context.Context, bucket string) error
// object tagging operations
// object tags operations
GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
PutObjectTagging(_ context.Context, bucket, object string, tags map[string]string) error
DeleteObjectTagging(_ context.Context, bucket, object string) error
@@ -103,7 +93,7 @@ func (BackendUnsupported) HeadBucket(context.Context, *s3.HeadBucketInput) (*s3.
func (BackendUnsupported) GetBucketAcl(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput, []byte) error {
func (BackendUnsupported) CreateBucket(context.Context, *s3.CreateBucketInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []byte) error {
@@ -112,12 +102,6 @@ func (BackendUnsupported) PutBucketAcl(_ context.Context, bucket string, data []
func (BackendUnsupported) DeleteBucket(context.Context, *s3.DeleteBucketInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketVersioning(context.Context, *s3.PutBucketVersioningInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketVersioning(_ context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
@@ -178,33 +162,8 @@ func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) e
func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
return func(w *bufio.Writer) {
var getProgress s3select.GetProgress
progress := input.RequestProgress
if progress != nil && *progress.Enabled {
getProgress = func() (bytesScanned int64, bytesProcessed int64) {
return -1, -1
}
}
mh := s3select.NewMessageHandler(ctx, w, getProgress)
apiErr := s3err.GetAPIError(s3err.ErrNotImplemented)
mh.FinishWithError(apiErr.Code, apiErr.Description)
}
}
func (BackendUnsupported) ListObjectVersions(context.Context, *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) DeleteBucketTagging(_ context.Context, bucket string) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
func (BackendUnsupported) SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
return s3response.SelectObjectContentResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {

View File

@@ -146,10 +146,6 @@ func (p *Posix) ListBuckets(_ context.Context, owner string, isAdmin bool) (s3re
}
func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
_, err := os.Lstat(*input.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -161,12 +157,9 @@ func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.He
return &s3.HeadBucketOutput{}, nil
}
func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput, acl []byte) error {
if input.Bucket == nil {
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput) error {
bucket := *input.Bucket
owner := string(input.ObjectOwnership)
err := os.Mkdir(bucket, 0777)
if err != nil && os.IsExist(err) {
@@ -176,7 +169,13 @@ func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput, acl
return fmt.Errorf("mkdir bucket: %w", err)
}
if err := xattr.Set(bucket, aclkey, acl); err != nil {
acl := auth.ACL{ACL: "private", Owner: owner, Grantees: []auth.Grantee{}}
jsonACL, err := json.Marshal(acl)
if err != nil {
return fmt.Errorf("marshal acl: %w", err)
}
if err := xattr.Set(bucket, aclkey, jsonACL); err != nil {
return fmt.Errorf("set acl: %w", err)
}
@@ -184,10 +183,6 @@ func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput, acl
}
func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) error {
if input.Bucket == nil {
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
names, err := os.ReadDir(*input.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -217,13 +212,6 @@ func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) err
}
func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
if mpu.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if mpu.Key == nil {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
bucket := *mpu.Bucket
object := *mpu.Key
@@ -281,19 +269,6 @@ func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipart
}
func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if input.UploadId == nil {
return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
if input.MultipartUpload == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
@@ -319,7 +294,7 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
partsize := int64(0)
var totalsize int64
for i, p := range parts {
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))
fi, err := os.Lstat(partPath)
if err != nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
@@ -351,7 +326,7 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
defer f.cleanup()
for _, p := range parts {
pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)))
pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)))
if err != nil {
return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err)
}
@@ -537,16 +512,6 @@ func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
}
func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUploadInput) error {
if mpu.Bucket == nil {
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if mpu.Key == nil {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if mpu.UploadId == nil {
return s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
bucket := *mpu.Bucket
object := *mpu.Key
uploadID := *mpu.UploadId
@@ -577,12 +542,6 @@ func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUp
}
func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
var lmu s3response.ListMultipartUploadsResult
if mpu.Bucket == nil {
return lmu, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
bucket := *mpu.Bucket
var delimiter string
if mpu.Delimiter != nil {
@@ -593,6 +552,8 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
prefix = *mpu.Prefix
}
var lmu s3response.ListMultipartUploadsResult
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return lmu, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -665,16 +626,12 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
}
}
maxUploads := 0
if mpu.MaxUploads != nil {
maxUploads = int(*mpu.MaxUploads)
}
if (uploadIDMarker != "" && !uploadIdMarkerFound) || (keyMarker != "" && keyMarkerInd == -1) {
return s3response.ListMultipartUploadsResult{
Bucket: bucket,
Delimiter: delimiter,
KeyMarker: keyMarker,
MaxUploads: maxUploads,
MaxUploads: int(mpu.MaxUploads),
Prefix: prefix,
UploadIDMarker: uploadIDMarker,
Uploads: []s3response.Upload{},
@@ -686,18 +643,18 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
})
for i := keyMarkerInd + 1; i < len(uploads); i++ {
if maxUploads == 0 {
if mpu.MaxUploads == 0 {
break
}
if keyMarker != "" && uploadIDMarker != "" && uploads[i].UploadID < uploadIDMarker {
continue
}
if i != len(uploads)-1 && len(resultUpds) == maxUploads {
if i != len(uploads)-1 && len(resultUpds) == int(mpu.MaxUploads) {
return s3response.ListMultipartUploadsResult{
Bucket: bucket,
Delimiter: delimiter,
KeyMarker: keyMarker,
MaxUploads: maxUploads,
MaxUploads: int(mpu.MaxUploads),
NextKeyMarker: resultUpds[i-1].Key,
NextUploadIDMarker: resultUpds[i-1].UploadID,
IsTruncated: true,
@@ -714,7 +671,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
Bucket: bucket,
Delimiter: delimiter,
KeyMarker: keyMarker,
MaxUploads: maxUploads,
MaxUploads: int(mpu.MaxUploads),
Prefix: prefix,
UploadIDMarker: uploadIDMarker,
Uploads: resultUpds,
@@ -722,29 +679,13 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
}
func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
var lpr s3response.ListPartsResult
if input.Bucket == nil {
return lpr, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return lpr, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if input.UploadId == nil {
return lpr, s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
stringMarker := ""
if input.PartNumberMarker != nil {
stringMarker = *input.PartNumberMarker
}
maxParts := 0
if input.MaxParts != nil {
maxParts = int(*input.MaxParts)
}
stringMarker := *input.PartNumberMarker
maxParts := int(input.MaxParts)
var lpr s3response.ListPartsResult
var partNumberMarker int
if stringMarker != "" {
@@ -836,21 +777,11 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon
}
func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string, error) {
if input.Bucket == nil {
return "", s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return "", s3err.GetAPIError(s3err.ErrNoSuchKey)
}
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
part := input.PartNumber
length := int64(0)
if input.ContentLength != nil {
length = *input.ContentLength
}
length := input.ContentLength
r := input.Body
_, err := os.Stat(bucket)
@@ -872,7 +803,7 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
return "", fmt.Errorf("stat uploadid: %w", err)
}
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part))
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", part))
f, err := openTmpFile(filepath.Join(bucket, objdir),
bucket, partPath, length)
@@ -902,13 +833,6 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
}
func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
if upi.Bucket == nil {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if upi.Key == nil {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
_, err := os.Stat(*upi.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -928,7 +852,7 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
return s3response.CopyObjectResult{}, fmt.Errorf("stat uploadid: %w", err)
}
partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", *upi.PartNumber))
partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", upi.PartNumber))
substrs := strings.SplitN(*upi.CopySource, "/", 2)
if len(substrs) != 2 {
@@ -1014,13 +938,6 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
}
func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) {
if po.Bucket == nil {
return "", s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if po.Key == nil {
return "", s3err.GetAPIError(s3err.ErrNoSuchKey)
}
tagsStr := getString(po.Tagging)
tags := make(map[string]string)
_, err := os.Stat(*po.Bucket)
@@ -1047,13 +964,9 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
name := filepath.Join(*po.Bucket, *po.Key)
contentLength := int64(0)
if po.ContentLength != nil {
contentLength = *po.ContentLength
}
if strings.HasSuffix(*po.Key, "/") {
// object is directory
if contentLength != 0 {
if po.ContentLength != 0 {
// posix directories can't contain data, send error
// if reuests has a data payload associated with a
// directory object
@@ -1082,7 +995,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
}
f, err := openTmpFile(filepath.Join(*po.Bucket, metaTmpDir),
*po.Bucket, *po.Key, contentLength)
*po.Bucket, *po.Key, po.ContentLength)
if err != nil {
return "", fmt.Errorf("open temp file: %w", err)
}
@@ -1126,13 +1039,6 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
}
func (p *Posix) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) error {
if input.Bucket == nil {
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
bucket := *input.Bucket
object := *input.Key
@@ -1225,16 +1131,6 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput)
}
func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if input.Range == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidRange)
}
bucket := *input.Bucket
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
@@ -1296,17 +1192,15 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
return nil, fmt.Errorf("get object tags: %w", err)
}
tagCount := int32(len(tags))
return &s3.GetObjectOutput{
AcceptRanges: &acceptRange,
ContentLength: &length,
ContentLength: length,
ContentEncoding: &contentEncoding,
ContentType: &contentType,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
TagCount: &tagCount,
TagCount: int32(len(tags)),
ContentRange: &contentRange,
}, nil
}
@@ -1341,28 +1235,20 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
return nil, fmt.Errorf("get object tags: %w", err)
}
tagCount := int32(len(tags))
return &s3.GetObjectOutput{
AcceptRanges: &acceptRange,
ContentLength: &length,
ContentLength: length,
ContentEncoding: &contentEncoding,
ContentType: &contentType,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
TagCount: &tagCount,
TagCount: int32(len(tags)),
ContentRange: &contentRange,
}, nil
}
func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
bucket := *input.Bucket
object := *input.Key
@@ -1392,10 +1278,8 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He
etag = ""
}
size := fi.Size()
return &s3.HeadObjectOutput{
ContentLength: &size,
ContentLength: fi.Size(),
ContentType: &contentType,
ContentEncoding: &contentEncoding,
ETag: &etag,
@@ -1405,18 +1289,6 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He
}
func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
}
if input.CopySource == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource)
}
if input.ExpectedBucketOwner == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
srcBucket, srcObject, ok := strings.Cut(*input.CopySource, "/")
if !ok {
return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource)
@@ -1489,16 +1361,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
}
}
contentLength := fInfo.Size()
etag, err := p.PutObject(ctx,
&s3.PutObjectInput{
Bucket: &dstBucket,
Key: &dstObject,
Body: f,
ContentLength: &contentLength,
Metadata: meta,
})
etag, err := p.PutObject(ctx, &s3.PutObjectInput{Bucket: &dstBucket, Key: &dstObject, Body: f, ContentLength: fInfo.Size(), Metadata: meta})
if err != nil {
return nil, err
}
@@ -1517,26 +1380,11 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
}
func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
bucket := *input.Bucket
prefix := ""
if input.Prefix != nil {
prefix = *input.Prefix
}
marker := ""
if input.Marker != nil {
marker = *input.Marker
}
delim := ""
if input.Delimiter != nil {
delim = *input.Delimiter
}
maxkeys := int32(0)
if input.MaxKeys != nil {
maxkeys = *input.MaxKeys
}
prefix := *input.Prefix
marker := *input.Marker
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
@@ -1557,9 +1405,9 @@ func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.
CommonPrefixes: results.CommonPrefixes,
Contents: results.Objects,
Delimiter: &delim,
IsTruncated: &results.Truncated,
IsTruncated: results.Truncated,
Marker: &marker,
MaxKeys: &maxkeys,
MaxKeys: maxkeys,
Name: &bucket,
NextMarker: &results.NextMarker,
Prefix: &prefix,
@@ -1618,46 +1466,21 @@ func fileToObj(bucket string) backend.GetObjFunc {
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
size := fi.Size()
return types.Object{
ETag: &etag,
Key: &path,
LastModified: backend.GetTimePtr(fi.ModTime()),
Size: &size,
Size: fi.Size(),
}, nil
}
}
func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
bucket := *input.Bucket
prefix := ""
if input.Prefix != nil {
prefix = *input.Prefix
}
marker := ""
if input.ContinuationToken != nil {
if input.StartAfter != nil {
if *input.StartAfter > *input.ContinuationToken {
marker = *input.StartAfter
} else {
marker = *input.ContinuationToken
}
} else {
marker = *input.ContinuationToken
}
}
delim := ""
if input.Delimiter != nil {
delim = *input.Delimiter
}
maxkeys := int32(0)
if input.MaxKeys != nil {
maxkeys = *input.MaxKeys
}
prefix := *input.Prefix
marker := *input.ContinuationToken
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
@@ -1668,25 +1491,23 @@ func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (
}
fileSystem := os.DirFS(bucket)
results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
fileToObj(bucket), []string{metaTmpDir})
if err != nil {
return nil, fmt.Errorf("walk %v: %w", bucket, err)
}
count := int32(len(results.Objects))
return &s3.ListObjectsV2Output{
CommonPrefixes: results.CommonPrefixes,
Contents: results.Objects,
Delimiter: &delim,
IsTruncated: &results.Truncated,
IsTruncated: results.Truncated,
ContinuationToken: &marker,
MaxKeys: &maxkeys,
MaxKeys: int32(maxkeys),
Name: &bucket,
NextContinuationToken: &results.NextMarker,
Prefix: &prefix,
KeyCount: &count,
KeyCount: int32(len(results.Objects)),
}, nil
}
@@ -1707,9 +1528,6 @@ func (p *Posix) PutBucketAcl(_ context.Context, bucket string, data []byte) erro
}
func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
_, err := os.Stat(*input.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -1728,57 +1546,6 @@ func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]
return b, nil
}
func (p *Posix) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error {
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return fmt.Errorf("stat bucket: %w", err)
}
if tags == nil {
err = xattr.Remove(bucket, "user."+tagHdr)
if err != nil {
return fmt.Errorf("remove tags: %w", err)
}
return nil
}
b, err := json.Marshal(tags)
if err != nil {
return fmt.Errorf("marshal tags: %w", err)
}
err = xattr.Set(bucket, "user."+tagHdr, b)
if err != nil {
return fmt.Errorf("set tags: %w", err)
}
return nil
}
func (p *Posix) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) {
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
}
if err != nil {
return nil, fmt.Errorf("stat bucket: %w", err)
}
tags, err := p.getXattrTags(bucket, "")
if err != nil {
return nil, err
}
return tags, nil
}
func (p *Posix) DeleteBucketTagging(ctx context.Context, bucket string) error {
return p.PutBucketTagging(ctx, bucket, nil)
}
func (p *Posix) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {

View File

@@ -17,6 +17,7 @@ package s3proxy
import (
"context"
"crypto/tls"
"fmt"
"net/http"
"github.com/aws/aws-sdk-go-v2/aws"
@@ -25,10 +26,16 @@ import (
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/smithy-go/middleware"
"github.com/versity/versitygw/auth"
)
func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
cfg, err := s.getConfig(ctx, s.access, s.secret)
func (s *S3be) getClientFromCtx(ctx context.Context) (*s3.Client, error) {
acct, ok := ctx.Value("account").(auth.Account)
if !ok {
return nil, fmt.Errorf("invalid account in context")
}
cfg, err := s.getConfig(ctx, acct.Access, acct.Secret)
if err != nil {
return nil, err
}
@@ -36,7 +43,7 @@ func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
return s3.NewFromConfig(cfg), nil
}
func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
func (s *S3be) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
creds := credentials.NewStaticCredentialsProvider(access, secret, "")
tr := &http.Transport{
@@ -69,7 +76,7 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
}
// ResolveEndpoint is used for on prem or non-aws endpoints
func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
func (s *S3be) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
URL: s.endpoint,

View File

@@ -16,37 +16,21 @@ package s3proxy
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
const aclKey string = "versitygwAcl"
type S3Proxy struct {
type S3be struct {
backend.BackendUnsupported
client *s3.Client
access string
secret string
endpoint string
awsRegion string
disableChecksum bool
@@ -54,28 +38,25 @@ type S3Proxy struct {
debug bool
}
func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
s := &S3Proxy{
access: access,
secret: secret,
func New(endpoint, region string, disableChecksum, sslSkipVerify, debug bool) *S3be {
return &S3be{
endpoint: endpoint,
awsRegion: region,
disableChecksum: disableChecksum,
sslSkipVerify: sslSkipVerify,
debug: debug,
}
client, err := s.getClientWithCtx(context.Background())
if err != nil {
return nil, err
}
s.client = client
return s, nil
}
func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{})
func (s *S3be) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return s3response.ListAllMyBucketsResult{}, handleError(err)
return s3response.ListAllMyBucketsResult{}, err
}
output, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
if err != nil {
return s3response.ListAllMyBucketsResult{}, err
}
var buckets []s3response.ListAllMyBucketsEntry
@@ -88,7 +69,8 @@ func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (
return s3response.ListAllMyBucketsResult{
Owner: s3response.CanonicalUser{
ID: *output.Owner.ID,
ID: *output.Owner.ID,
DisplayName: *output.Owner.DisplayName,
},
Buckets: s3response.ListAllMyBucketsList{
Bucket: buckets,
@@ -96,56 +78,76 @@ func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (
}, nil
}
func (s *S3Proxy) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
out, err := s.client.HeadBucket(ctx, input)
return out, handleError(err)
}
func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput, acl []byte) error {
_, err := s.client.CreateBucket(ctx, input)
func (s *S3be) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return handleError(err)
return nil, err
}
var tagSet []types.Tag
tagSet = append(tagSet, types.Tag{
Key: backend.GetStringPtr(aclKey),
Value: backend.GetStringPtr(base64Encode(acl)),
})
_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
Bucket: input.Bucket,
Tagging: &types.Tagging{
TagSet: tagSet,
},
})
return handleError(err)
return client.HeadBucket(ctx, input)
}
func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
_, err := s.client.DeleteBucket(ctx, input)
return handleError(err)
}
func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
out, err := s.client.CreateMultipartUpload(ctx, input)
return out, handleError(err)
}
func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
out, err := s.client.CompleteMultipartUpload(ctx, input)
return out, handleError(err)
}
func (s *S3Proxy) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
_, err := s.client.AbortMultipartUpload(ctx, input)
return handleError(err)
}
func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
output, err := s.client.ListMultipartUploads(ctx, input)
func (s *S3be) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return s3response.ListMultipartUploadsResult{}, handleError(err)
return err
}
_, err = client.CreateBucket(ctx, input)
return err
}
func (s *S3be) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return err
}
_, err = client.DeleteBucket(ctx, input)
return err
}
func (s *S3be) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
return client.CreateMultipartUpload(ctx, input)
}
func (s *S3be) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
return client.CompleteMultipartUpload(ctx, input)
}
func (s *S3be) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return err
}
_, err = client.AbortMultipartUpload(ctx, input)
return err
}
const (
iso8601Format = "20060102T150405Z"
)
func (s *S3be) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return s3response.ListMultipartUploadsResult{}, err
}
output, err := client.ListMultipartUploads(ctx, input)
if err != nil {
return s3response.ListMultipartUploadsResult{}, err
}
var uploads []s3response.Upload
@@ -162,7 +164,7 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
DisplayName: *u.Owner.DisplayName,
},
StorageClass: string(u.StorageClass),
Initiated: u.Initiated.Format(backend.RFC3339TimeFormat),
Initiated: u.Initiated.Format(iso8601Format),
})
}
@@ -182,26 +184,31 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
Delimiter: *output.Delimiter,
Prefix: *output.Prefix,
EncodingType: string(output.EncodingType),
MaxUploads: int(*output.MaxUploads),
IsTruncated: *output.IsTruncated,
MaxUploads: int(output.MaxUploads),
IsTruncated: output.IsTruncated,
Uploads: uploads,
CommonPrefixes: cps,
}, nil
}
func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
output, err := s.client.ListParts(ctx, input)
func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return s3response.ListPartsResult{}, handleError(err)
return s3response.ListPartsResult{}, err
}
output, err := client.ListParts(ctx, input)
if err != nil {
return s3response.ListPartsResult{}, err
}
var parts []s3response.Part
for _, p := range output.Parts {
parts = append(parts, s3response.Part{
PartNumber: int(*p.PartNumber),
LastModified: p.LastModified.Format(backend.RFC3339TimeFormat),
PartNumber: int(p.PartNumber),
LastModified: p.LastModified.Format(iso8601Format),
ETag: *p.ETag,
Size: *p.Size,
Size: p.Size,
})
}
pnm, err := strconv.Atoi(*output.PartNumberMarker)
@@ -231,29 +238,35 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
StorageClass: string(output.StorageClass),
PartNumberMarker: pnm,
NextPartNumberMarker: npmn,
MaxParts: int(*output.MaxParts),
IsTruncated: *output.IsTruncated,
MaxParts: int(output.MaxParts),
IsTruncated: output.IsTruncated,
Parts: parts,
}, nil
}
func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
// streaming backend is not seekable,
// use unsigned payload for streaming ops
output, err := s.client.UploadPart(ctx, input, s3.WithAPIOptions(
v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
))
func (s *S3be) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return "", handleError(err)
return "", err
}
output, err := client.UploadPart(ctx, input)
if err != nil {
return "", err
}
return *output.ETag, nil
}
func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
output, err := s.client.UploadPartCopy(ctx, input)
func (s *S3be) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return s3response.CopyObjectResult{}, handleError(err)
return s3response.CopyObjectResult{}, err
}
output, err := client.UploadPartCopy(ctx, input)
if err != nil {
return s3response.CopyObjectResult{}, err
}
return s3response.CopyObjectResult{
@@ -262,28 +275,38 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn
}, nil
}
func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
// streaming backend is not seekable,
// use unsigned payload for streaming ops
output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions(
v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
))
func (s *S3be) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return "", handleError(err)
return "", err
}
output, err := client.PutObject(ctx, input)
if err != nil {
return "", err
}
return *output.ETag, nil
}
func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
out, err := s.client.HeadObject(ctx, input)
return out, handleError(err)
func (s *S3be) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
return client.HeadObject(ctx, input)
}
func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
output, err := s.client.GetObject(ctx, input)
func (s *S3be) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, handleError(err)
return nil, err
}
output, err := client.GetObject(ctx, input)
if err != nil {
return nil, err
}
defer output.Body.Close()
@@ -295,39 +318,61 @@ func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.
return output, nil
}
func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
out, err := s.client.GetObjectAttributes(ctx, input)
return out, handleError(err)
}
func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
out, err := s.client.CopyObject(ctx, input)
return out, handleError(err)
}
func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
out, err := s.client.ListObjects(ctx, input)
return out, handleError(err)
}
func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
out, err := s.client.ListObjectsV2(ctx, input)
return out, handleError(err)
}
func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
_, err := s.client.DeleteObject(ctx, input)
return handleError(err)
}
func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
if len(input.Delete.Objects) == 0 {
input.Delete.Objects = []types.ObjectIdentifier{}
func (s *S3be) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
output, err := s.client.DeleteObjects(ctx, input)
return client.GetObjectAttributes(ctx, input)
}
func (s *S3be) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return s3response.DeleteObjectsResult{}, handleError(err)
return nil, err
}
return client.CopyObject(ctx, input)
}
func (s *S3be) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
return client.ListObjects(ctx, input)
}
func (s *S3be) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
return client.ListObjectsV2(ctx, input)
}
func (s *S3be) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return err
}
_, err = client.DeleteObject(ctx, input)
return err
}
func (s *S3be) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return s3response.DeleteObjectsResult{}, err
}
output, err := client.DeleteObjects(ctx, input)
if err != nil {
return s3response.DeleteObjectsResult{}, err
}
return s3response.DeleteObjectsResult{
@@ -336,63 +381,71 @@ func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInpu
}, nil
}
func (s *S3Proxy) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
tagout, err := s.client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
Bucket: input.Bucket,
})
func (s *S3be) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, handleError(err)
return nil, err
}
for _, tag := range tagout.TagSet {
if *tag.Key == aclKey {
acl, err := base64Decode(*tag.Value)
if err != nil {
return nil, handleError(err)
}
return acl, nil
}
}
return []byte{}, nil
}
func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
tagout, err := s.client.GetBucketTagging(ctx, &s3.GetBucketTaggingInput{
Bucket: &bucket,
})
output, err := client.GetBucketAcl(ctx, input)
if err != nil {
return handleError(err)
return nil, err
}
var found bool
for i, tag := range tagout.TagSet {
if *tag.Key == aclKey {
tagout.TagSet[i] = types.Tag{
Key: backend.GetStringPtr(aclKey),
Value: backend.GetStringPtr(base64Encode(data)),
}
found = true
break
}
}
if !found {
tagout.TagSet = append(tagout.TagSet, types.Tag{
Key: backend.GetStringPtr(aclKey),
Value: backend.GetStringPtr(base64Encode(data)),
var acl auth.ACL
acl.Owner = *output.Owner.ID
for _, el := range output.Grants {
acl.Grantees = append(acl.Grantees, auth.Grantee{
Permission: el.Permission,
Access: *el.Grantee.ID,
})
}
_, err = s.client.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
Bucket: &bucket,
Tagging: &types.Tagging{
TagSet: tagout.TagSet,
},
})
return handleError(err)
return json.Marshal(acl)
}
func (s *S3Proxy) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
func (s S3be) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return err
}
acl, err := auth.ParseACL(data)
if err != nil {
return err
}
input := &s3.PutBucketAclInput{
Bucket: &bucket,
ACL: acl.ACL,
AccessControlPolicy: &types.AccessControlPolicy{
Owner: &types.Owner{
ID: &acl.Owner,
},
},
}
for _, el := range acl.Grantees {
input.AccessControlPolicy.Grants = append(input.AccessControlPolicy.Grants, types.Grant{
Permission: el.Permission,
Grantee: &types.Grantee{
ID: &el.Access,
Type: types.TypeCanonicalUser,
},
})
}
_, err = client.PutBucketAcl(ctx, input)
return err
}
func (s *S3be) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return err
}
tagging := &types.Tagging{
TagSet: []types.Tag{},
}
@@ -403,21 +456,26 @@ func (s *S3Proxy) PutObjectTagging(ctx context.Context, bucket, object string, t
})
}
_, err := s.client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
_, err = client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
Bucket: &bucket,
Key: &object,
Tagging: tagging,
})
return handleError(err)
return err
}
func (s *S3Proxy) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
output, err := s.client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
func (s *S3be) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
output, err := client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
Bucket: &bucket,
Key: &object,
})
if err != nil {
return nil, handleError(err)
return nil, err
}
tags := make(map[string]string)
@@ -428,118 +486,15 @@ func (s *S3Proxy) GetObjectTagging(ctx context.Context, bucket, object string) (
return tags, nil
}
func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
_, err := s.client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
func (s *S3be) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return err
}
_, err = client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
Bucket: &bucket,
Key: &object,
})
return handleError(err)
}
func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, newOwner), nil)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
if resp.StatusCode > 300 {
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
defer resp.Body.Close()
return fmt.Errorf(string(body))
}
return nil
}
func (s *S3Proxy) ListBucketsAndOwners(ctx context.Context) ([]s3response.Bucket, error) {
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", s.endpoint), nil)
if err != nil {
return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
if signErr != nil {
return []s3response.Bucket{}, fmt.Errorf("failed to sign the request: %w", err)
}
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return []s3response.Bucket{}, err
}
defer resp.Body.Close()
var buckets []s3response.Bucket
if err := json.Unmarshal(body, &buckets); err != nil {
return []s3response.Bucket{}, err
}
return buckets, nil
}
func handleError(err error) error {
if err == nil {
return nil
}
var ae smithy.APIError
if errors.As(err, &ae) {
apiErr := s3err.APIError{
Code: ae.ErrorCode(),
Description: ae.ErrorMessage(),
}
var re *awshttp.ResponseError
if errors.As(err, &re) {
apiErr.HTTPStatusCode = re.Response.StatusCode
}
return apiErr
}
return err
}
func base64Encode(input []byte) string {
return base64.StdEncoding.EncodeToString(input)
}
func base64Decode(encoded string) ([]byte, error) {
decoded, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return nil, err
}
return decoded, nil
}

View File

@@ -415,10 +415,8 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.
}
}
contentLength := fi.Size()
return &s3.HeadObjectOutput{
ContentLength: &contentLength,
ContentLength: fi.Size(),
ContentType: &contentType,
ContentEncoding: &contentEncoding,
ETag: &etag,
@@ -509,17 +507,15 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
return nil, fmt.Errorf("get object tags: %w", err)
}
tagCount := int32(len(tags))
return &s3.GetObjectOutput{
AcceptRanges: &acceptRange,
ContentLength: &length,
ContentLength: length,
ContentEncoding: &contentEncoding,
ContentType: &contentType,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
TagCount: &tagCount,
TagCount: int32(len(tags)),
StorageClass: types.StorageClassStandard,
}, nil
}
@@ -546,26 +542,11 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error)
}
func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
bucket := *input.Bucket
prefix := ""
if input.Prefix != nil {
prefix = *input.Prefix
}
marker := ""
if input.Marker != nil {
marker = *input.Marker
}
delim := ""
if input.Delimiter != nil {
delim = *input.Delimiter
}
maxkeys := int32(0)
if input.MaxKeys != nil {
maxkeys = *input.MaxKeys
}
prefix := *input.Prefix
marker := *input.Marker
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
@@ -586,9 +567,9 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s
CommonPrefixes: results.CommonPrefixes,
Contents: results.Objects,
Delimiter: &delim,
IsTruncated: &results.Truncated,
IsTruncated: results.Truncated,
Marker: &marker,
MaxKeys: &maxkeys,
MaxKeys: maxkeys,
Name: &bucket,
NextMarker: &results.NextMarker,
Prefix: &prefix,
@@ -596,26 +577,11 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s
}
func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
bucket := *input.Bucket
prefix := ""
if input.Prefix != nil {
prefix = *input.Prefix
}
marker := ""
if input.ContinuationToken != nil {
marker = *input.ContinuationToken
}
delim := ""
if input.Delimiter != nil {
delim = *input.Delimiter
}
maxkeys := int32(0)
if input.MaxKeys != nil {
maxkeys = *input.MaxKeys
}
prefix := *input.Prefix
marker := *input.ContinuationToken
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
@@ -636,9 +602,9 @@ func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input)
CommonPrefixes: results.CommonPrefixes,
Contents: results.Objects,
Delimiter: &delim,
IsTruncated: &results.Truncated,
IsTruncated: results.Truncated,
ContinuationToken: &marker,
MaxKeys: &maxkeys,
MaxKeys: int32(maxkeys),
Name: &bucket,
NextContinuationToken: &results.NextMarker,
Prefix: &prefix,
@@ -711,13 +677,11 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
}
}
size := fi.Size()
return types.Object{
ETag: &etag,
Key: &path,
LastModified: backend.GetTimePtr(fi.ModTime()),
Size: &size,
Size: fi.Size(),
StorageClass: sc,
}, nil
}

View File

@@ -55,13 +55,11 @@ func getObj(path string, d fs.DirEntry) (types.Object, error) {
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
size := fi.Size()
return types.Object{
ETag: &etag,
Key: &path,
LastModified: backend.GetTimePtr(fi.ModTime()),
Size: &size,
Size: fi.Size(),
}, nil
}

View File

@@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDUaEoThzf6bWNU
R7+V0sYGND0iArE8dx33jCqWBzUCeeAnf8VjsKTVKWK8Cernjot7jP/GlKHCXKVC
Dpaw2SuekVn84gXr0UDD0clrPYYcytm8TWrfNmcPnmLARH6bes+5Glt21gug5fc7
gvd/EFI/eNTAmU6ZPzVjKVP5plvkSZYXePgmgKzY8hs8j9IjjOZB+TzalRlQoMa4
hPYTzNpvdS3aQCM/8PTefC8sxI0H9inx1utax22MPNCaGl+BvD9yqvFwQl7n0d9u
3ZLJ8Br9UKRPX47NDerpsYsmJAAtjevei84v4liCKtxIzC6b/KuKpgEDhiza+TL/
VoJ34xK9AgMBAAECggEBAISIPhBJQshjEKM50XTuetjMJ4jdHTGZMX2QW9IY1R6i
ZRbARq2ZPoAyvoSNu6CX9Lg2ljGV9AiOPh8lcykNmIXsM7XyYbdubXbBo2fij5fP
imRP+eskytGYBg3prwXcb1gT9hYEIGVYmBbt9Pe3e1pXToiOH9jG88zXsKoI/zVF
ASfE73L9xlO9roaFWe+sQlNsHJ5SVhKVNq4FcjvIbZTi3JqMJvuGqgIjyOeKlRdH
FszPjmn8tmlbT7BQIfeo4uIJwDrUxdp4Do9txScSOhV4xcdVjpOrbD+HaTIl18u8
J32pf0n8E90ntKJlPyfSt68VIkIT1VtE0SH42UgxcAECgYEA2Ctw58hSKFSQtSDk
HAOuxh9gSM4eIvWG8Ni4L+S6n66hCPYYj+1ZGBM1xIKqfug25STP2qV54zB34CMM
RfPEJhKglgzN1VdTvTkdLfqSnrEDyF+67/X0IhomV54CLN52i7S6ra39OVClirQc
5FbjX7Dwy57vanNW7E7ZKHyxRwECgYEA+4tiCw8+NeCIjLiOldB/R9P6CguFCIFY
B+MUXc2K690ysRcVnHCVLGVrdYlmlAjiEEOo6e8nhl0pLJecOLUkLnEdZkC9x5je
oy1itan7Y/WjdSPIpLE5sBnrWzzpy5YVoKi4rjIkPXYDHcgbvEcKUxN/qUpOpKk6
rTeaAPOwp70CgYEAwPIFVNz4eAcDIqi48khXN3/J8TIItCtyxoap4BXIfb7g/Z6r
TcwMOfDrjPsUMzIRzXWOERqiMKaSWPzvd4CdE16M92F2V3YayEqyQNfnBr35ImBP
+t8NiWLN1mayiloGdaxa86rY2s+g8qzRHP5w9Hh6dUTnbZyFeWbnbbvegAECgYAG
T1zKQjuhRlymiwqon25R8vNWxSs1J2l56SxdngZaHFZlMtsL7ZcQYgrsC+JS3FYv
akMWezVWnYem4ra8hW6+6399TSp1k1Qia8UKIZV40HSlP5yM5RU5Ya0RwNlsjftE
6HaZiBB4qjkxyg9IDdAofVi6em62mrgqGDb4xyQrUQKBgHfzBE/pxKTZVCyjJgq+
XKtvm7awq6tVByZzKVbFb+kqle7n0Kb+MjDSiH0gXqrntkhquEEpo8ViXenHeDq1
I2Fgp6Tw8pKW7RANcWaxd2KLYDxUkqwNWCMYGQZoW9lhTy1/e5stXSTdqd+3CJkp
qBbZOasECIEGL8XUpFEBmrTA
-----END PRIVATE KEY-----

View File

@@ -1,25 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIEMzCCApugAwIBAgIQbYDa23tko31nfiGrCJ4XfDANBgkqhkiG9w0BAQsFADBj
MR4wHAYDVQQKExVta2NlcnQgZGV2ZWxvcG1lbnQgQ0ExHDAaBgNVBAsME2JlbkBC
eXRlTmV4dXMubG9jYWwxIzAhBgNVBAMMGm1rY2VydCBiZW5AQnl0ZU5leHVzLmxv
Y2FsMB4XDTI0MDIyMTA2MDY1OFoXDTI2MDUyMTA1MDY1OFowRzEnMCUGA1UEChMe
bWtjZXJ0IGRldmVsb3BtZW50IGNlcnRpZmljYXRlMRwwGgYDVQQLDBNiZW5AQnl0
ZU5leHVzLmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1GhK
E4c3+m1jVEe/ldLGBjQ9IgKxPHcd94wqlgc1AnngJ3/FY7Ck1SlivAnq546Le4z/
xpShwlylQg6WsNkrnpFZ/OIF69FAw9HJaz2GHMrZvE1q3zZnD55iwER+m3rPuRpb
dtYLoOX3O4L3fxBSP3jUwJlOmT81YylT+aZb5EmWF3j4JoCs2PIbPI/SI4zmQfk8
2pUZUKDGuIT2E8zab3Ut2kAjP/D03nwvLMSNB/Yp8dbrWsdtjDzQmhpfgbw/cqrx
cEJe59Hfbt2SyfAa/VCkT1+OzQ3q6bGLJiQALY3r3ovOL+JYgircSMwum/yriqYB
A4Ys2vky/1aCd+MSvQIDAQABo38wfTAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAww
CgYIKwYBBQUHAwEwHwYDVR0jBBgwFoAUHWzgJTYXxrKTuaJyaD2ddH4g43QwNQYD
VR0RBC4wLIIJbG9jYWxob3N0ggdhenVyaXRlhwR/AAABhxAAAAAAAAAAAAAAAAAA
AAABMA0GCSqGSIb3DQEBCwUAA4IBgQBANjSWFEE2XrQfCtqKjHRtYn+WnjfeUmJW
4qfNl971spR+ui+WMNob53EKGV9O55Ai8zWUKhL14GSDo4bqygcqCSP7esUaqW5x
LgqGkIDPiPN18OfWxZonlvcSJYaWJ9dL4Xix7OmPXQV88oNUYh5zASJRZ8mN3vZK
v0BElS5w7X4pM8jWkIEVUjmgSQJAxapDnbDPCIntH13rjYPHZW3zF1wjWW/mfW59
+thLVpG0U9w8knDU9TJQWPX9rQB8lLBHYfBy14ebuMWmDl25Rr+wcUjFp76HP8f9
D8wX7ZTJlMlaH60vbTi8ylsKMlu6eHxJ8TBNdRI5+Xa94Pi6A81a9EiagpU2uZs1
P/v/3ug9AtCFoEBB2groj3waC6QTOrJArHhvioLKBz7yMsfGRQCSA+VK+KGuqE6f
E/Knv1Phf8HIe6OpDNXqlJo9+5m0sz/y6F2OaeNqM643Y78OQRFijtDKu+TmB+t0
4lo/rMVz6A0vIOBBQkiJz82xezvCiHQ=
-----END CERTIFICATE-----

View File

@@ -164,14 +164,14 @@ func createUser(ctx *cli.Context) error {
if access == "" || secret == "" {
return fmt.Errorf("invalid input parameters for the new user")
}
if role != string(auth.RoleAdmin) && role != string(auth.RoleUser) && role != string(auth.RoleUserPlus) {
return fmt.Errorf("invalid input parameter for role: %v", role)
if role != "admin" && role != "user" {
return fmt.Errorf("invalid input parameter for role")
}
acc := auth.Account{
Access: access,
Secret: secret,
Role: auth.Role(role),
Role: role,
UserID: userID,
GroupID: groupID,
ProjectID: projectID,
@@ -289,14 +289,11 @@ func listUsers(ctx *cli.Context) error {
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return fmt.Errorf("%s", body)
}
var accs []auth.Account
if err := json.Unmarshal(body, &accs); err != nil {
return err
}
fmt.Println(accs)
printAcctTable(accs)
@@ -405,7 +402,7 @@ func listBuckets(ctx *cli.Context) error {
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return fmt.Errorf("%s", body)
return fmt.Errorf(string(body))
}
var buckets []s3response.Bucket

View File

@@ -1,74 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package main
import (
"fmt"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/azure"
)
var (
azAccount, azKey, azServiceURL, azSASToken string
)
func azureCommand() *cli.Command {
return &cli.Command{
Name: "azure",
Usage: "azure blob storage backend",
Description: `direct translation from s3 objects to azure blobs`,
Action: runAzure,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "account",
Usage: "azure account name",
EnvVars: []string{"AZ_ACCOUNT_NAME"},
Aliases: []string{"a"},
Destination: &azAccount,
},
&cli.StringFlag{
Name: "access-key",
Usage: "azure account key",
EnvVars: []string{"AZ_ACCESS_KEY"},
Aliases: []string{"k"},
Destination: &azKey,
},
&cli.StringFlag{
Name: "sas-token",
Usage: "azure blob storage SAS token",
EnvVars: []string{"AZ_SAS_TOKEN"},
Aliases: []string{"st"},
Destination: &azSASToken,
},
&cli.StringFlag{
Name: "url",
Usage: "azure service URL",
EnvVars: []string{"AZ_ENDPOINT"},
Aliases: []string{"u"},
Destination: &azServiceURL,
},
},
}
}
func runAzure(ctx *cli.Context) error {
be, err := azure.New(azAccount, azKey, azServiceURL, azSASToken)
if err != nil {
return fmt.Errorf("init azure: %w", err)
}
return runGateway(ctx.Context, be)
}

View File

@@ -1,104 +0,0 @@
package main
import (
"context"
"log"
"os"
"path/filepath"
"sync"
"testing"
"github.com/versity/versitygw/backend/posix"
"github.com/versity/versitygw/integration"
)
const (
tdir = "tempdir"
)
var (
wg sync.WaitGroup
)
func initEnv(dir string) {
// both
debug = true
region = "us-east-1"
// server
rootUserAccess = "user"
rootUserSecret = "pass"
iamDir = dir
port = "127.0.0.1:7070"
// client
awsID = "user"
awsSecret = "pass"
endpoint = "http://127.0.0.1:7070"
}
func initPosix(ctx context.Context) {
path, err := os.Getwd()
if err != nil {
log.Fatalf("get current directory: %v", err)
}
tempdir := filepath.Join(path, tdir)
initEnv(tempdir)
err = os.RemoveAll(tempdir)
if err != nil {
log.Fatalf("remove temp directory: %v", err)
}
err = os.Mkdir(tempdir, 0755)
if err != nil {
log.Fatalf("make temp directory: %v", err)
}
be, err := posix.New(tempdir)
if err != nil {
log.Fatalf("init posix: %v", err)
}
wg.Add(1)
go func() {
err = runGateway(ctx, be)
if err != nil && err != context.Canceled {
log.Fatalf("run gateway: %v", err)
}
err := os.RemoveAll(tempdir)
if err != nil {
log.Fatalf("remove temp directory: %v", err)
}
wg.Done()
}()
}
func TestIntegration(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
initPosix(ctx)
opts := []integration.Option{
integration.WithAccess(awsID),
integration.WithSecret(awsSecret),
integration.WithRegion(region),
integration.WithEndpoint(endpoint),
}
if debug {
opts = append(opts, integration.WithDebug())
}
s := integration.NewS3Conf(opts...)
// replace below with desired test
err := integration.HeadBucket_non_existing_bucket(s)
if err != nil {
t.Error(err)
}
cancel()
wg.Wait()
}

View File

@@ -20,6 +20,7 @@ import (
"fmt"
"log"
"os"
"strconv"
"github.com/gofiber/fiber/v2"
"github.com/urfave/cli/v2"
@@ -42,17 +43,11 @@ var (
natsURL, natsTopic string
logWebhookURL string
accessLog string
healthPath string
debug bool
quiet bool
iamDir string
ldapURL, ldapBindDN, ldapPassword string
ldapQueryBase, ldapObjClasses string
ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
s3IamAccess, s3IamSecret string
s3IamRegion, s3IamBucket string
s3IamEndpoint string
s3IamSslNoVerify, s3IamDebug bool
iamCacheDisable bool
iamCacheTTL int
iamCachePrune int
@@ -76,7 +71,6 @@ func main() {
posixCommand(),
scoutfsCommand(),
s3Command(),
azureCommand(),
adminCommand(),
testCommand(),
}
@@ -123,7 +117,6 @@ func initFlags() []cli.Flag {
&cli.StringFlag{
Name: "port",
Usage: "gateway listen address <ip>:<port> or :<port>",
EnvVars: []string{"VGW_PORT"},
Value: ":7070",
Destination: &port,
Aliases: []string{"p"},
@@ -145,7 +138,6 @@ func initFlags() []cli.Flag {
&cli.StringFlag{
Name: "region",
Usage: "s3 region string",
EnvVars: []string{"VGW_REGION"},
Value: "us-east-1",
Destination: &region,
Aliases: []string{"r"},
@@ -153,47 +145,34 @@ func initFlags() []cli.Flag {
&cli.StringFlag{
Name: "cert",
Usage: "TLS cert file",
EnvVars: []string{"VGW_CERT"},
Destination: &certFile,
},
&cli.StringFlag{
Name: "key",
Usage: "TLS key file",
EnvVars: []string{"VGW_KEY"},
Destination: &keyFile,
},
&cli.StringFlag{
Name: "admin-port",
Usage: "gateway admin server listen address <ip>:<port> or :<port>",
EnvVars: []string{"VGW_ADMIN_PORT"},
Destination: &admPort,
Aliases: []string{"ap"},
},
&cli.StringFlag{
Name: "admin-cert",
Usage: "TLS cert file for admin server",
EnvVars: []string{"VGW_ADMIN_CERT"},
Destination: &admCertFile,
},
&cli.StringFlag{
Name: "admin-cert-key",
Usage: "TLS key file for admin server",
EnvVars: []string{"VGW_ADMIN_CERT_KEY"},
Destination: &admKeyFile,
},
&cli.BoolFlag{
Name: "debug",
Usage: "enable debug output",
EnvVars: []string{"VGW_DEBUG"},
Destination: &debug,
},
&cli.BoolFlag{
Name: "quiet",
Usage: "silence stdout request logging output",
EnvVars: []string{"VGW_QUIET"},
Destination: &quiet,
Aliases: []string{"q"},
},
&cli.StringFlag{
Name: "access-log",
Usage: "enable server access logging to specified file",
@@ -209,171 +188,110 @@ func initFlags() []cli.Flag {
&cli.StringFlag{
Name: "event-kafka-url",
Usage: "kafka server url to send the bucket notifications.",
EnvVars: []string{"VGW_EVENT_KAFKA_URL"},
Destination: &kafkaURL,
Aliases: []string{"eku"},
},
&cli.StringFlag{
Name: "event-kafka-topic",
Usage: "kafka server pub-sub topic to send the bucket notifications to",
EnvVars: []string{"VGW_EVENT_KAFKA_TOPIC"},
Destination: &kafkaTopic,
Aliases: []string{"ekt"},
},
&cli.StringFlag{
Name: "event-kafka-key",
Usage: "kafka server put-sub topic key to send the bucket notifications to",
EnvVars: []string{"VGW_EVENT_KAFKA_KEY"},
Destination: &kafkaKey,
Aliases: []string{"ekk"},
},
&cli.StringFlag{
Name: "event-nats-url",
Usage: "nats server url to send the bucket notifications",
EnvVars: []string{"VGW_EVENT_NATS_URL"},
Destination: &natsURL,
Aliases: []string{"enu"},
},
&cli.StringFlag{
Name: "event-nats-topic",
Usage: "nats server pub-sub topic to send the bucket notifications to",
EnvVars: []string{"VGW_EVENT_NATS_TOPIC"},
Destination: &natsTopic,
Aliases: []string{"ent"},
},
&cli.StringFlag{
Name: "iam-dir",
Usage: "if defined, run internal iam service within this directory",
EnvVars: []string{"VGW_IAM_DIR"},
Destination: &iamDir,
},
&cli.StringFlag{
Name: "iam-ldap-url",
Usage: "ldap server url to store iam data",
EnvVars: []string{"VGW_IAM_LDAP_URL"},
Destination: &ldapURL,
},
&cli.StringFlag{
Name: "iam-ldap-bind-dn",
Usage: "ldap server binding dn, example: 'cn=admin,dc=example,dc=com'",
EnvVars: []string{"VGW_IAM_LDAP_BIND_DN"},
Destination: &ldapBindDN,
},
&cli.StringFlag{
Name: "iam-ldap-bind-pass",
Usage: "ldap server user password",
EnvVars: []string{"VGW_IAM_LDAP_BIND_PASS"},
Destination: &ldapPassword,
},
&cli.StringFlag{
Name: "iam-ldap-query-base",
Usage: "ldap server destination query, example: 'ou=iam,dc=example,dc=com'",
EnvVars: []string{"VGW_IAM_LDAP_QUERY_BASE"},
Destination: &ldapQueryBase,
},
&cli.StringFlag{
Name: "iam-ldap-object-classes",
Usage: "ldap server object classes used to store the data. provide it as comma separated string, example: 'top,person'",
EnvVars: []string{"VGW_IAM_LDAP_OBJECT_CLASSES"},
Destination: &ldapObjClasses,
},
&cli.StringFlag{
Name: "iam-ldap-access-atr",
Usage: "ldap server user access key id attribute name",
EnvVars: []string{"VGW_IAM_LDAP_ACCESS_ATR"},
Destination: &ldapAccessAtr,
},
&cli.StringFlag{
Name: "iam-ldap-secret-atr",
Usage: "ldap server user secret access key attribute name",
EnvVars: []string{"VGW_IAM_LDAP_SECRET_ATR"},
Destination: &ldapSecAtr,
},
&cli.StringFlag{
Name: "iam-ldap-role-atr",
Usage: "ldap server user role attribute name",
EnvVars: []string{"VGW_IAM_LDAP_ROLE_ATR"},
Destination: &ldapRoleAtr,
},
&cli.StringFlag{
Name: "s3-iam-access",
Usage: "s3 IAM access key",
EnvVars: []string{"VGW_S3_IAM_ACCESS_KEY"},
Destination: &s3IamAccess,
},
&cli.StringFlag{
Name: "s3-iam-secret",
Usage: "s3 IAM secret key",
EnvVars: []string{"VGW_S3_IAM_SECRET_KEY"},
Destination: &s3IamSecret,
},
&cli.StringFlag{
Name: "s3-iam-region",
Usage: "s3 IAM region",
EnvVars: []string{"VGW_S3_IAM_REGION"},
Destination: &s3IamRegion,
Value: "us-east-1",
},
&cli.StringFlag{
Name: "s3-iam-bucket",
Usage: "s3 IAM bucket",
EnvVars: []string{"VGW_S3_IAM_BUCKET"},
Destination: &s3IamBucket,
},
&cli.StringFlag{
Name: "s3-iam-endpoint",
Usage: "s3 IAM endpoint",
EnvVars: []string{"VGW_S3_IAM_ENDPOINT"},
Destination: &s3IamEndpoint,
},
&cli.BoolFlag{
Name: "s3-iam-noverify",
Usage: "s3 IAM disable ssl verification",
EnvVars: []string{"VGW_S3_IAM_NO_VERIFY"},
Destination: &s3IamSslNoVerify,
},
&cli.BoolFlag{
Name: "s3-iam-debug",
Usage: "s3 IAM debug output",
EnvVars: []string{"VGW_S3_IAM_DEBUG"},
Destination: &s3IamDebug,
},
&cli.BoolFlag{
Name: "iam-cache-disable",
Usage: "disable local iam cache",
EnvVars: []string{"VGW_IAM_CACHE_DISABLE"},
Destination: &iamCacheDisable,
},
&cli.IntFlag{
Name: "iam-cache-ttl",
Usage: "local iam cache entry ttl (seconds)",
EnvVars: []string{"VGW_IAM_CACHE_TTL"},
Value: 120,
Destination: &iamCacheTTL,
},
&cli.IntFlag{
Name: "iam-cache-prune",
Usage: "local iam cache cleanup interval (seconds)",
EnvVars: []string{"VGW_IAM_CACHE_PRUNE"},
Value: 3600,
Destination: &iamCachePrune,
},
&cli.StringFlag{
Name: "health",
Usage: `health check endpoint path. Health endpoint will be configured on GET http method: GET <health>
NOTICE: the path has to be specified with '/'. e.g /health`,
EnvVars: []string{"VGW_HEALTH"},
Destination: &healthPath,
},
}
}
func runGateway(ctx context.Context, be backend.Backend) error {
func runGateway(ctx *cli.Context, be backend.Backend) error {
// int32 max for 32 bit arch
blimit := int64(2*1024*1024*1024 - 1)
if strconv.IntSize > 32 {
// 5GB max for 64 bit arch
blimit = int64(5 * 1024 * 1024 * 1024)
}
app := fiber.New(fiber.Config{
AppName: "versitygw",
ServerHeader: "VERSITYGW",
StreamRequestBody: true,
DisableKeepalive: true,
AppName: "versitygw",
ServerHeader: "VERSITYGW",
BodyLimit: int(blimit),
})
var opts []s3api.Option
@@ -398,12 +316,6 @@ func runGateway(ctx context.Context, be backend.Backend) error {
if admPort == "" {
opts = append(opts, s3api.WithAdminServer())
}
if quiet {
opts = append(opts, s3api.WithQuiet())
}
if healthPath != "" {
opts = append(opts, s3api.WithHealth(healthPath))
}
admApp := fiber.New(fiber.Config{
AppName: "versitygw",
@@ -428,25 +340,18 @@ func runGateway(ctx context.Context, be backend.Backend) error {
}
iam, err := auth.New(&auth.Opts{
Dir: iamDir,
LDAPServerURL: ldapURL,
LDAPBindDN: ldapBindDN,
LDAPPassword: ldapPassword,
LDAPQueryBase: ldapQueryBase,
LDAPObjClasses: ldapObjClasses,
LDAPAccessAtr: ldapAccessAtr,
LDAPSecretAtr: ldapSecAtr,
LDAPRoleAtr: ldapRoleAtr,
S3Access: s3IamAccess,
S3Secret: s3IamSecret,
S3Region: s3IamRegion,
S3Bucket: s3IamBucket,
S3Endpoint: s3IamEndpoint,
S3DisableSSlVerfiy: s3IamSslNoVerify,
S3Debug: s3IamDebug,
CacheDisable: iamCacheDisable,
CacheTTL: iamCacheTTL,
CachePrune: iamCachePrune,
Dir: iamDir,
LDAPServerURL: ldapURL,
LDAPBindDN: ldapBindDN,
LDAPPassword: ldapPassword,
LDAPQueryBase: ldapQueryBase,
LDAPObjClasses: ldapObjClasses,
LDAPAccessAtr: ldapAccessAtr,
LDAPSecretAtr: ldapSecAtr,
LDAPRoleAtr: ldapRoleAtr,
CacheDisable: iamCacheDisable,
CacheTTL: iamCacheTTL,
CachePrune: iamCachePrune,
})
if err != nil {
return fmt.Errorf("setup iam: %w", err)

View File

@@ -49,5 +49,5 @@ func runPosix(ctx *cli.Context) error {
return fmt.Errorf("init posix: %v", err)
}
return runGateway(ctx.Context, be)
return runGateway(ctx, be)
}

View File

@@ -15,15 +15,11 @@
package main
import (
"fmt"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/s3proxy"
)
var (
s3proxyAccess string
s3proxySecret string
s3proxyEndpoint string
s3proxyRegion string
s3proxyDisableChecksum bool
@@ -39,22 +35,6 @@ func s3Command() *cli.Command {
to an s3 storage backend service.`,
Action: runS3,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "access",
Usage: "s3 proxy server access key id",
Value: "",
Required: true,
Destination: &s3proxyAccess,
Aliases: []string{"a"},
},
&cli.StringFlag{
Name: "secret",
Usage: "s3 proxy server secret access key",
Value: "",
Required: true,
Destination: &s3proxySecret,
Aliases: []string{"s"},
},
&cli.StringFlag{
Name: "endpoint",
Usage: "s3 service endpoint, default AWS if not specified",
@@ -90,10 +70,7 @@ to an s3 storage backend service.`,
}
func runS3(ctx *cli.Context) error {
be, err := s3proxy.New(s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
be := s3proxy.New(s3proxyEndpoint, s3proxyRegion,
s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
if err != nil {
return fmt.Errorf("init s3 backend: %w", err)
}
return runGateway(ctx.Context, be)
return runGateway(ctx, be)
}

View File

@@ -69,5 +69,5 @@ func runScoutfs(ctx *cli.Context) error {
return fmt.Errorf("init scoutfs: %v", err)
}
return runGateway(ctx.Context, be)
return runGateway(ctx, be)
}

View File

@@ -2,6 +2,9 @@ package main
import (
"fmt"
"math"
"os"
"text/tabwriter"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/integration"
@@ -13,6 +16,7 @@ var (
endpoint string
prefix string
dstBucket string
proxyURL string
partSize int64
objSize int64
concurrency int
@@ -67,7 +71,7 @@ func initTestFlags() []cli.Flag {
}
func initTestCommands() []*cli.Command {
return append([]*cli.Command{
return []*cli.Command{
{
Name: "full-flow",
Usage: "Tests the full flow of gateway.",
@@ -79,11 +83,6 @@ func initTestCommands() []*cli.Command {
Usage: "Tests posix specific features",
Action: getAction(integration.TestPosix),
},
{
Name: "iam",
Usage: "Tests iam service",
Action: getAction(integration.TestIAM),
},
{
Name: "bench",
Usage: "Runs download/upload performance test on the gateway",
@@ -123,6 +122,7 @@ func initTestCommands() []*cli.Command {
Name: "bucket",
Usage: "Destination bucket name to read/write data",
Destination: &dstBucket,
Required: true,
},
&cli.Int64Flag{
Name: "partSize",
@@ -148,6 +148,11 @@ func initTestCommands() []*cli.Command {
Value: false,
Destination: &checksumDisable,
},
&cli.StringFlag{
Name: "proxy-url",
Usage: "S3 proxy server url to compare",
Destination: &proxyURL,
},
},
Action: func(ctx *cli.Context) error {
if upload && download {
@@ -157,10 +162,6 @@ func initTestCommands() []*cli.Command {
return fmt.Errorf("must specify one of upload or download")
}
if dstBucket == "" {
return fmt.Errorf("must specify bucket")
}
opts := []integration.Option{
integration.WithAccess(awsID),
integration.WithSecret(awsSecret),
@@ -182,9 +183,47 @@ func initTestCommands() []*cli.Command {
s3conf := integration.NewS3Conf(opts...)
if upload {
return integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
if proxyURL == "" {
integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
return nil
} else {
size, elapsed, err := integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
opts = append(opts, integration.WithEndpoint(proxyURL))
proxyS3Conf := integration.NewS3Conf(opts...)
proxySize, proxyElapsed, proxyErr := integration.TestUpload(proxyS3Conf, files, objSize, dstBucket, prefix)
if err != nil || proxyErr != nil {
return nil
}
printProxyResultsTable([][4]string{
{" # ", "Total Size", "Time Taken", "Speed(MB/S)"},
{"---------", "----------", "----------", "-----------"},
{"S3 Server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
{"S3 Proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
})
return nil
}
} else {
return integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
if proxyURL == "" {
integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
return nil
} else {
size, elapsed, err := integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
opts = append(opts, integration.WithEndpoint(proxyURL))
proxyS3Conf := integration.NewS3Conf(opts...)
proxySize, proxyElapsed, proxyErr := integration.TestDownload(proxyS3Conf, files, objSize, dstBucket, prefix)
if err != nil || proxyErr != nil {
return nil
}
printProxyResultsTable([][4]string{
{" # ", "Total Size", "Time Taken", "Speed(MB/S)"},
{"---------", "----------", "----------", "-----------"},
{"S3 server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
{"S3 proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
})
return nil
}
}
},
},
@@ -216,12 +255,13 @@ func initTestCommands() []*cli.Command {
Value: false,
Destination: &checksumDisable,
},
&cli.StringFlag{
Name: "proxy-url",
Usage: "S3 proxy server url to compare",
Destination: &proxyURL,
},
},
Action: func(ctx *cli.Context) error {
if dstBucket == "" {
return fmt.Errorf("must specify the destination bucket")
}
opts := []integration.Option{
integration.WithAccess(awsID),
integration.WithSecret(awsSecret),
@@ -238,10 +278,30 @@ func initTestCommands() []*cli.Command {
s3conf := integration.NewS3Conf(opts...)
return integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
if proxyURL == "" {
_, _, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
return err
} else {
elapsed, rps, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
opts = append(opts, integration.WithEndpoint(proxyURL))
s3proxy := integration.NewS3Conf(opts...)
proxyElapsed, proxyRPS, proxyErr := integration.TestReqPerSec(s3proxy, totalReqs, dstBucket)
if err != nil || proxyErr != nil {
return nil
}
printProxyResultsTable([][4]string{
{" # ", "Total Requests", "Time Taken", "Requests Per Second(Req/Sec)"},
{"---------", "--------------", "----------", "----------------------------"},
{"S3 Server", fmt.Sprint(totalReqs), fmt.Sprintf("%v", elapsed), fmt.Sprint(rps)},
{"S3 Proxy", fmt.Sprint(totalReqs), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(proxyRPS)},
})
return nil
}
},
},
}, extractIntTests()...)
}
}
type testFunc func(*integration.S3Conf)
@@ -270,30 +330,12 @@ func getAction(tf testFunc) func(*cli.Context) error {
}
}
func extractIntTests() (commands []*cli.Command) {
tests := integration.GetIntTests()
for key, val := range tests {
k := key
testFunc := val
commands = append(commands, &cli.Command{
Name: k,
Usage: fmt.Sprintf("Runs %v integration test", key),
Action: func(ctx *cli.Context) error {
opts := []integration.Option{
integration.WithAccess(awsID),
integration.WithSecret(awsSecret),
integration.WithRegion(region),
integration.WithEndpoint(endpoint),
}
if debug {
opts = append(opts, integration.WithDebug())
}
s := integration.NewS3Conf(opts...)
err := testFunc(s)
return err
},
})
func printProxyResultsTable(stats [][4]string) {
w := new(tabwriter.Writer)
w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
for _, elem := range stats {
fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", elem[0], elem[1], elem[2], elem[3])
}
return
fmt.Fprintln(w)
w.Flush()
}

View File

@@ -1,43 +0,0 @@
version: "3"
services:
posix:
build:
context: .
dockerfile: ./Dockerfile.dev
args:
- IAM_DIR=${IAM_DIR}
- SETUP_DIR=${SETUP_DIR}
volumes:
- ./:/app
ports:
- "${POSIX_PORT}:${POSIX_PORT}"
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$POSIX_PORT -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR posix $SETUP_DIR"]
proxy:
build:
context: .
dockerfile: ./Dockerfile.dev
volumes:
- ./:/app
ports:
- "${PROXY_PORT}:${PROXY_PORT}"
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$PROXY_PORT s3 -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --endpoint http://posix:$POSIX_PORT"]
azurite:
image: mcr.microsoft.com/azure-storage/azurite
ports:
- "10000:10000"
- "10001:10001"
- "10002:10002"
restart: always
hostname: azurite
command: "azurite --oauth basic --cert /certs/azurite.pem --key /certs/azurite-key.pem --blobHost 0.0.0.0"
volumes:
- ./certs:/certs
azuritegw:
build:
context: .
dockerfile: ./Dockerfile.dev
volumes:
- ./:/app
ports:
- 7070:7070
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR azure -a $AZ_ACCOUNT_NAME -k $AZ_ACCOUNT_KEY --url https://azurite:10000/$AZ_ACCOUNT_NAME"]

76
go.mod
View File

@@ -1,66 +1,56 @@
module github.com/versity/versitygw
go 1.21
go 1.20
require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
github.com/aws/aws-sdk-go-v2 v1.25.2
github.com/aws/aws-sdk-go-v2/service/s3 v1.51.2
github.com/aws/smithy-go v1.20.1
github.com/aws/aws-sdk-go-v2 v1.22.2
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1
github.com/aws/smithy-go v1.16.0
github.com/go-ldap/ldap/v3 v3.4.6
github.com/gofiber/fiber/v2 v2.52.2
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
github.com/nats-io/nats.go v1.33.1
github.com/gofiber/fiber/v2 v2.50.0
github.com/google/uuid v1.4.0
github.com/nats-io/nats.go v1.31.0
github.com/pkg/xattr v0.4.9
github.com/segmentio/kafka-go v0.4.47
github.com/urfave/cli/v2 v2.27.1
github.com/valyala/fasthttp v1.52.0
github.com/segmentio/kafka-go v0.4.44
github.com/urfave/cli/v2 v2.25.7
github.com/valyala/fasthttp v1.50.0
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
golang.org/x/sys v0.18.0
golang.org/x/sys v0.14.0
)
require (
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.28.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/nats-io/nkeys v0.4.7 // indirect
github.com/nats-io/nkeys v0.4.6 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
golang.org/x/crypto v0.19.0 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/text v0.14.0 // indirect
github.com/stretchr/testify v1.8.1 // indirect
golang.org/x/crypto v0.14.0 // indirect
)
require (
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
github.com/aws/aws-sdk-go-v2/config v1.27.5
github.com/aws/aws-sdk-go-v2/credentials v1.17.5
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.7
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.2 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 // indirect
github.com/aws/aws-sdk-go-v2/config v1.24.0
github.com/aws/aws-sdk-go-v2/credentials v1.15.2
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/klauspost/compress v1.17.6 // indirect
github.com/klauspost/compress v1.17.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect

161
go.sum
View File

@@ -1,106 +1,85 @@
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 h1:n1DH8TPV4qqPTje2RcUBYwtrTWlabVp4n46+74X2pn4=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0/go.mod h1:HDcZnuGbiyppErN6lB+idp4CKhjbc8gwjto6OPpyggM=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/aws/aws-sdk-go-v2 v1.25.2 h1:/uiG1avJRgLGiQM9X3qJM8+Qa6KRGK5rRPuXE0HUM+w=
github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo=
github.com/aws/aws-sdk-go-v2/config v1.27.5 h1:brBPsyRFQn97M1ZhQ9tLXkO7Zytiar0NS06FGmEJBdg=
github.com/aws/aws-sdk-go-v2/config v1.27.5/go.mod h1:I53uvsfddRRTG5YcC4n5Z3aOD1BU8hYCoIG7iEJG4wM=
github.com/aws/aws-sdk-go-v2/credentials v1.17.5 h1:yn3zSvIKC2NZIs40cY3kckcy9Zma96PrRR07N54PCvY=
github.com/aws/aws-sdk-go-v2/credentials v1.17.5/go.mod h1:8JcKPAGZVnDWuR5lusAwmrSDtZnDIAnpQWaDC9RFt2g=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 h1:AK0J8iYBFeUk2Ax7O8YpLtFsfhdOByh2QIkHmigpRYk=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.7 h1:/r2O0R/JAD1Y1iCxxz7nClKntXqB9CLTrxu7csrAsSA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.7/go.mod h1:TbQoOduGh1PZbTNRqaEemgj/e+mmFC3hScHEQDTcUoQ=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 h1:bNo4LagzUKbjdxE0tIcR9pMzLR2U/Tgie1Hq1HQ3iH8=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2/go.mod h1:wRQv0nN6v9wDXuWThpovGQjqF1HFdcgWjporw14lS8k=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 h1:EtOU5jsPdIQNP+6Q2C5e3d65NKT1PeCiQk+9OdzO12Q=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2/go.mod h1:tyF5sKccmDz0Bv4NrstEr+/9YkSPJHrcO7UsUKf7pWM=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2 h1:en92G0Z7xlksoOylkUhuBSfJgijC7rHVLRdnIlHEs0E=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2/go.mod h1:HgtQ/wN5G+8QSlK62lbOtNwQ3wTSByJ4wH2rCkPt+AE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.3 h1:fpFzBoro/MetYBk+8kxoQGMeKSkXbymnbUh2gy6nVgk=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.3/go.mod h1:qmQPbMe5NQk/nEmpkl8iHyCSREJjEbRUrnqHpHabLlM=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.3 h1:x0N5ftQzgcfRpCpTiyZC40pvNUJYhzf4UgCsAyO6/P8=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.3/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.2 h1:1oY1AVEisRI4HNuFoLdRUB0hC63ylDAN6Me3MrfclEg=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.2/go.mod h1:KZ03VgvZwSjkT7fOetQ/wF3MZUvYFirlI1H5NklUNsY=
github.com/aws/aws-sdk-go-v2/service/s3 v1.51.2 h1:ukAaTX8n/pX0Essg9CxW8VCjACv75vnNo2GRONR1w1Q=
github.com/aws/aws-sdk-go-v2/service/s3 v1.51.2/go.mod h1:wt4wZz/CBlJJwY0L7X6vPQ9njh2aHi59knqpJ6B/2cM=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 h1:utEGkfdQ4L6YW/ietH7111ZYglLJvS+sLriHJ1NBJEQ=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 h1:9/GylMS45hGGFCcMrUZDVayQE1jYSIN6da9jo7RAYIw=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.2 h1:0YjXuWdYHvsm0HnT4vO8XpwG1D+i2roxSCBoN6deJ7M=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.2/go.mod h1:jI+FWmYkSMn+4APWmZiZTgt0oM0TrvymD51FMqCnWgA=
github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw=
github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/aws/aws-sdk-go-v2 v1.22.2 h1:lV0U8fnhAnPz8YcdmZVV60+tr6CakHzqA6P8T46ExJI=
github.com/aws/aws-sdk-go-v2 v1.22.2/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 h1:hHgLiIrTRtddC0AKcJr5s7i/hLgcpTt+q/FKxf1Zayk=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0/go.mod h1:w4I/v3NOWgD+qvs1NPEwhd++1h3XPHFaVxasfY6HlYQ=
github.com/aws/aws-sdk-go-v2/config v1.24.0 h1:4LEk29JO3w+y9dEo/5Tq5QTP7uIEw+KQrKiHOs4xlu4=
github.com/aws/aws-sdk-go-v2/config v1.24.0/go.mod h1:11nNDAuK86kOUHeuEQo8f3CkcV5xuUxvPwFjTZE/PnQ=
github.com/aws/aws-sdk-go-v2/credentials v1.15.2 h1:rKH7khRMxPdD0u3dHecd0Q7NOVw3EUe7AqdkUOkiOGI=
github.com/aws/aws-sdk-go-v2/credentials v1.15.2/go.mod h1:tXM8wmaeAhfC7nZoCxb0FzM/aRaB1m1WQ7x0qlBLq80=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 h1:G5KawTAkyHH6WyKQCdHiW4h3PmAXNJpOgwKg3H7sDRE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3/go.mod h1:hugKmSFnZB+HgNI1sYGT14BUPZkO6alC/e0AWu+0IAQ=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6 h1:IpQbitxCZeC64C1ALz9QZu6AHHWundnU2evQ9xbp5k8=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6/go.mod h1:27jIVQK+al9s0yTo3pkMdahRinbscqSC6zNGfNWXPZc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 h1:AaQsr5vvGR7rmeSWBtTCcw16tT9r51mWijuCQhzLnq8=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2/go.mod h1:o1IiRn7CWocIFTXJjGKJDOwxv1ibL53NpcvcqGWyRBA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 h1:UZx8SXZ0YtzRiALzYAWcjb9Y9hZUR7MBKaBQ5ouOjPs=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2/go.mod h1:ipuRpcSaklmxR6C39G187TpBAO132gUfleTGccUPs8c=
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 h1:usgqiJtamuGIBj+OvYmMq89+Z1hIKkMJToz1WpoeNUY=
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 h1:pyVrNAf7Hwz0u39dLKN5t+n0+K/3rMYKuiOoIum3AsU=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2/go.mod h1:mydrfOb9uiOYCxuCPR8YHQNQyGQwUQ7gPMZGBKbH8NY=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 h1:CJxo7ZBbaIzmXfV3hjcx36n9V87gJsIUPJflwqEHl3Q=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0/go.mod h1:yjVfjuY4nD1EW9i387Kau+I6V5cBA5YnC/mWNopjZrI=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 h1:f2LhPofnjcdOQKRtumKjMvIHkfSQ8aH/rwKUDEQ/SB4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2/go.mod h1:q+xX0H4OfuWDuBy7y/LDi4v8IBOWuF+vtp8Z6ex+lw4=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 h1:h7j73yuAVVjic8pqswh+L/7r2IHP43QwRyOu6zcCDDE=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2/go.mod h1:H07AHdK5LSy8F7EJUQhoxyiCNkePoHj2D8P2yGTWafo=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 h1:gbIaOzpXixUpoPK+js/bCBK1QBDXM22SigsnzGZio0U=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2/go.mod h1:p+S7RNbdGN8qgHDSg2SCQJ9FeMAmvcETQiVpeGhYnNM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 h1:o6MCcX1rJW8Y3g+hvg2xpjF6JR6DftuYhfl3Nc1WV9Q=
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1/go.mod h1:UDtxEWbREX6y4KREapT+jjtjoH0TiVSS6f5nfaY1UaM=
github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 h1:km+ZNjtLtpXYf42RdaDZnNHm9s7SYAuDGTafy6nd89A=
github.com/aws/aws-sdk-go-v2/service/sso v1.17.1/go.mod h1:aHBr3pvBSD5MbzOvQtYutyPLLRPbl/y9x86XyJJnUXQ=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 h1:iRFNqZH4a67IqPvK8xxtyQYnyrlsvwmpHOe9r55ggBA=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1/go.mod h1:pTy5WM+6sNv2tB24JNKFtn6EvciQ5k40ZJ0pq/Iaxj0=
github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 h1:txgVXIXWPXyqdiVn92BV6a/rgtpX31HYdsOYj0sVQQQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.25.1/go.mod h1:VAiJiNaoP1L89STFlEMgmHX1bKixY+FaP+TpRFrmyZ4=
github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik=
github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
github.com/gofiber/fiber/v2 v2.52.2 h1:b0rYH6b06Df+4NyrbdptQL8ifuxw/Tf2DgfkZkDaxEo=
github.com/gofiber/fiber/v2 v2.52.2/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70=
github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -110,20 +89,21 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
github.com/segmentio/kafka-go v0.4.44 h1:Vjjksniy0WSTZ7CuVJrz1k04UoZeTc77UV6Yyk6tLY4=
github.com/segmentio/kafka-go v0.4.44/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
@@ -140,9 +120,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -150,9 +129,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -163,14 +141,13 @@ golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -183,18 +160,16 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -17,16 +17,16 @@ type prefResult struct {
err error
}
func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
var sg sync.WaitGroup
results := make([]prefResult, files)
start := time.Now()
if objSize == 0 {
return fmt.Errorf("must specify object size for upload")
return 0, time.Since(start), fmt.Errorf("must specify object size for upload")
}
if objSize > (int64(10000) * s.PartSize) {
return fmt.Errorf("object size can not exceed 10000 * chunksize")
return 0, time.Since(start), fmt.Errorf("object size can not exceed 10000 * chunksize")
}
runF("performance test: upload objects")
@@ -45,13 +45,13 @@ func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) erro
}(i)
}
sg.Wait()
elapsed := time.Since(start)
elapsed = time.Since(start)
var tot int64
for i, res := range results {
if res.err != nil {
failF("%v: %v\n", i, res.err)
break
return 0, time.Since(start), res.err
}
tot += res.size
fmt.Printf("%v: %v in %v (%v MB/s)\n",
@@ -63,10 +63,10 @@ func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) erro
passF("run upload: %v in %v (%v MB/s)\n",
tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
return nil
return tot, time.Since(start), nil
}
func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
var sg sync.WaitGroup
results := make([]prefResult, files)
start := time.Now()
@@ -86,13 +86,13 @@ func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) er
}(i)
}
sg.Wait()
elapsed := time.Since(start)
elapsed = time.Since(start)
var tot int64
for i, res := range results {
if res.err != nil {
failF("%v: %v\n", i, res.err)
break
return 0, elapsed, err
}
tot += res.size
fmt.Printf("%v: %v in %v (%v MB/s)\n",
@@ -104,10 +104,10 @@ func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) er
passF("run download: %v in %v (%v MB/s)\n",
tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
return nil
return tot, elapsed, nil
}
func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) error {
func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) (time.Duration, int, error) {
client := s3.NewFromConfig(s.Config())
var wg sync.WaitGroup
var resErr error
@@ -132,11 +132,11 @@ func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) error {
wg.Wait()
if resErr != nil {
failF("performance test failed with error: %w", resErr)
return nil
return time.Since(startTime), 0, resErr
}
elapsedTime := time.Since(startTime)
rps := int(float64(totalReqs) / elapsedTime.Seconds())
passF("Success\nTotal Requests: %d,\nConcurrency Level: %d,\nTime Taken: %s,\nRequests Per Second: %dreq/sec", totalReqs, s.Concurrency, elapsedTime, rps)
return nil
return elapsedTime, rps, nil
}

View File

@@ -22,36 +22,10 @@ func TestAuthentication(s *S3Conf) {
Authentication_signature_error_incorrect_secret_key(s)
}
func TestPresignedAuthentication(s *S3Conf) {
PresignedAuth_missing_algo_query_param(s)
PresignedAuth_unsupported_algorithm(s)
PresignedAuth_missing_credentials_query_param(s)
PresignedAuth_malformed_creds_invalid_parts(s)
PresignedAuth_malformed_creds_invalid_parts(s)
PresignedAuth_creds_incorrect_service(s)
PresignedAuth_creds_incorrect_region(s)
PresignedAuth_creds_invalid_date(s)
PresignedAuth_missing_date_query(s)
PresignedAuth_dates_mismatch(s)
PresignedAuth_non_existing_access_key_id(s)
PresignedAuth_missing_signed_headers_query_param(s)
PresignedAuth_missing_expiration_query_param(s)
PresignedAuth_invalid_expiration_query_param(s)
PresignedAuth_negative_expiration_query_param(s)
PresignedAuth_exceeding_expiration_query_param(s)
PresignedAuth_expired_request(s)
PresignedAuth_incorrect_secret_key(s)
PresignedAuth_PutObject_success(s)
PresignedAuth_Put_GetObject_with_data(s)
PresignedAuth_UploadPart(s)
}
func TestCreateBucket(s *S3Conf) {
CreateBucket_invalid_bucket_name(s)
CreateBucket_existing_bucket(s)
CreateBucket_as_user(s)
CreateBucket_default_acl(s)
CreateBucket_non_default_acl(s)
CreateDeleteBucket_success(s)
}
@@ -72,29 +46,11 @@ func TestDeleteBucket(s *S3Conf) {
DeleteBucket_success_status_code(s)
}
func TestPutBucketTagging(s *S3Conf) {
PutBucketTagging_non_existing_bucket(s)
PutBucketTagging_long_tags(s)
PutBucketTagging_success(s)
}
func TestGetBucketTagging(s *S3Conf) {
GetBucketTagging_non_existing_bucket(s)
GetBucketTagging_success(s)
}
func TestDeleteBucketTagging(s *S3Conf) {
DeleteBucketTagging_non_existing_object(s)
DeleteBucketTagging_success_status(s)
DeleteBucketTagging_success(s)
}
func TestPutObject(s *S3Conf) {
PutObject_non_existing_bucket(s)
PutObject_special_chars(s)
PutObject_invalid_long_tags(s)
PutObject_success(s)
PutObject_invalid_credentials(s)
}
func TestHeadObject(s *S3Conf) {
@@ -121,13 +77,6 @@ func TestListObjects(s *S3Conf) {
ListObjects_marker_not_from_obj_list(s)
}
func TestListObjectsV2(s *S3Conf) {
ListObjectsV2_start_after(s)
ListObjectsV2_both_start_after_and_continuation_token(s)
ListObjectsV2_start_after_not_in_list(s)
ListObjectsV2_start_after_empty_result(s)
}
func TestDeleteObject(s *S3Conf) {
DeleteObject_non_existing_object(s)
DeleteObject_success(s)
@@ -243,19 +192,14 @@ func TestGetBucketAcl(s *S3Conf) {
func TestFullFlow(s *S3Conf) {
TestAuthentication(s)
TestPresignedAuthentication(s)
TestCreateBucket(s)
TestHeadBucket(s)
TestListBuckets(s)
TestDeleteBucket(s)
TestPutBucketTagging(s)
TestGetBucketTagging(s)
TestDeleteBucketTagging(s)
TestPutObject(s)
TestHeadObject(s)
TestGetObject(s)
TestListObjects(s)
TestListObjectsV2(s)
TestDeleteObject(s)
TestDeleteObjects(s)
TestCopyObject(s)
@@ -278,178 +222,3 @@ func TestPosix(s *S3Conf) {
PutObject_dir_obj_with_data(s)
CreateMultipartUpload_dir_obj(s)
}
func TestIAM(s *S3Conf) {
IAM_user_access_denied(s)
IAM_userplus_access_denied(s)
IAM_userplus_CreateBucket(s)
IAM_admin_ChangeBucketOwner(s)
}
type IntTests map[string]func(s *S3Conf) error
func GetIntTests() IntTests {
return IntTests{
"Authentication_empty_auth_header": Authentication_empty_auth_header,
"Authentication_invalid_auth_header": Authentication_invalid_auth_header,
"Authentication_unsupported_signature_version": Authentication_unsupported_signature_version,
"Authentication_malformed_credentials": Authentication_malformed_credentials,
"Authentication_malformed_credentials_invalid_parts": Authentication_malformed_credentials_invalid_parts,
"Authentication_credentials_terminated_string": Authentication_credentials_terminated_string,
"Authentication_credentials_incorrect_service": Authentication_credentials_incorrect_service,
"Authentication_credentials_incorrect_region": Authentication_credentials_incorrect_region,
"Authentication_credentials_invalid_date": Authentication_credentials_invalid_date,
"Authentication_credentials_future_date": Authentication_credentials_future_date,
"Authentication_credentials_past_date": Authentication_credentials_past_date,
"Authentication_credentials_non_existing_access_key": Authentication_credentials_non_existing_access_key,
"Authentication_invalid_signed_headers": Authentication_invalid_signed_headers,
"Authentication_missing_date_header": Authentication_missing_date_header,
"Authentication_invalid_date_header": Authentication_invalid_date_header,
"Authentication_date_mismatch": Authentication_date_mismatch,
"Authentication_incorrect_payload_hash": Authentication_incorrect_payload_hash,
"Authentication_incorrect_md5": Authentication_incorrect_md5,
"Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key,
"PresignedAuth_missing_algo_query_param": PresignedAuth_missing_algo_query_param,
"PresignedAuth_unsupported_algorithm": PresignedAuth_unsupported_algorithm,
"PresignedAuth_missing_credentials_query_param": PresignedAuth_missing_credentials_query_param,
"PresignedAuth_malformed_creds_invalid_parts": PresignedAuth_malformed_creds_invalid_parts,
"PresignedAuth_creds_invalid_terminator": PresignedAuth_creds_invalid_terminator,
"PresignedAuth_creds_incorrect_service": PresignedAuth_creds_incorrect_service,
"PresignedAuth_creds_incorrect_region": PresignedAuth_creds_incorrect_region,
"PresignedAuth_creds_invalid_date": PresignedAuth_creds_invalid_date,
"PresignedAuth_missing_date_query": PresignedAuth_missing_date_query,
"PresignedAuth_dates_mismatch": PresignedAuth_dates_mismatch,
"PresignedAuth_non_existing_access_key_id": PresignedAuth_non_existing_access_key_id,
"PresignedAuth_missing_signed_headers_query_param": PresignedAuth_missing_signed_headers_query_param,
"PresignedAuth_missing_expiration_query_param": PresignedAuth_missing_expiration_query_param,
"PresignedAuth_invalid_expiration_query_param": PresignedAuth_invalid_expiration_query_param,
"PresignedAuth_negative_expiration_query_param": PresignedAuth_negative_expiration_query_param,
"PresignedAuth_exceeding_expiration_query_param": PresignedAuth_exceeding_expiration_query_param,
"PresignedAuth_expired_request": PresignedAuth_expired_request,
"PresignedAuth_incorrect_secret_key": PresignedAuth_incorrect_secret_key,
"PresignedAuth_PutObject_success": PresignedAuth_PutObject_success,
"PresignedAuth_Put_GetObject_with_data": PresignedAuth_Put_GetObject_with_data,
"PresignedAuth_UploadPart": PresignedAuth_UploadPart,
"CreateBucket_invalid_bucket_name": CreateBucket_invalid_bucket_name,
"CreateBucket_existing_bucket": CreateBucket_existing_bucket,
"CreateBucket_as_user": CreateBucket_as_user,
"CreateDeleteBucket_success": CreateDeleteBucket_success,
"CreateBucket_default_acl": CreateBucket_default_acl,
"CreateBucket_non_default_acl": CreateBucket_non_default_acl,
"HeadBucket_non_existing_bucket": HeadBucket_non_existing_bucket,
"HeadBucket_success": HeadBucket_success,
"ListBuckets_as_user": ListBuckets_as_user,
"ListBuckets_as_admin": ListBuckets_as_admin,
"ListBuckets_success": ListBuckets_success,
"DeleteBucket_non_existing_bucket": DeleteBucket_non_existing_bucket,
"DeleteBucket_non_empty_bucket": DeleteBucket_non_empty_bucket,
"DeleteBucket_success_status_code": DeleteBucket_success_status_code,
"PutBucketTagging_non_existing_bucket": PutBucketTagging_non_existing_bucket,
"PutBucketTagging_long_tags": PutBucketTagging_long_tags,
"PutBucketTagging_success": PutBucketTagging_success,
"GetBucketTagging_non_existing_bucket": GetBucketTagging_non_existing_bucket,
"GetBucketTagging_success": GetBucketTagging_success,
"DeleteBucketTagging_non_existing_object": DeleteBucketTagging_non_existing_object,
"DeleteBucketTagging_success_status": DeleteBucketTagging_success_status,
"DeleteBucketTagging_success": DeleteBucketTagging_success,
"PutObject_non_existing_bucket": PutObject_non_existing_bucket,
"PutObject_special_chars": PutObject_special_chars,
"PutObject_invalid_long_tags": PutObject_invalid_long_tags,
"PutObject_success": PutObject_success,
"HeadObject_non_existing_object": HeadObject_non_existing_object,
"HeadObject_success": HeadObject_success,
"GetObject_non_existing_key": GetObject_non_existing_key,
"GetObject_invalid_ranges": GetObject_invalid_ranges,
"GetObject_with_meta": GetObject_with_meta,
"GetObject_success": GetObject_success,
"GetObject_by_range_success": GetObject_by_range_success,
"ListObjects_non_existing_bucket": ListObjects_non_existing_bucket,
"ListObjects_with_prefix": ListObjects_with_prefix,
"ListObject_truncated": ListObject_truncated,
"ListObjects_invalid_max_keys": ListObjects_invalid_max_keys,
"ListObjects_max_keys_0": ListObjects_max_keys_0,
"ListObjects_delimiter": ListObjects_delimiter,
"ListObjects_max_keys_none": ListObjects_max_keys_none,
"ListObjects_marker_not_from_obj_list": ListObjects_marker_not_from_obj_list,
"ListObjectsV2_start_after": ListObjectsV2_start_after,
"ListObjectsV2_both_start_after_and_continuation_token": ListObjectsV2_both_start_after_and_continuation_token,
"ListObjectsV2_start_after_not_in_list": ListObjectsV2_start_after_not_in_list,
"ListObjectsV2_start_after_empty_result": ListObjectsV2_start_after_empty_result,
"DeleteObject_non_existing_object": DeleteObject_non_existing_object,
"DeleteObject_success": DeleteObject_success,
"DeleteObject_success_status_code": DeleteObject_success_status_code,
"DeleteObjects_empty_input": DeleteObjects_empty_input,
"DeleteObjects_non_existing_objects": DeleteObjects_non_existing_objects,
"DeleteObjects_success": DeleteObjects_success,
"CopyObject_non_existing_dst_bucket": CopyObject_non_existing_dst_bucket,
"CopyObject_not_owned_source_bucket": CopyObject_not_owned_source_bucket,
"CopyObject_copy_to_itself": CopyObject_copy_to_itself,
"CopyObject_to_itself_with_new_metadata": CopyObject_to_itself_with_new_metadata,
"CopyObject_success": CopyObject_success,
"PutObjectTagging_non_existing_object": PutObjectTagging_non_existing_object,
"PutObjectTagging_long_tags": PutObjectTagging_long_tags,
"PutObjectTagging_success": PutObjectTagging_success,
"GetObjectTagging_non_existing_object": GetObjectTagging_non_existing_object,
"GetObjectTagging_success": GetObjectTagging_success,
"DeleteObjectTagging_non_existing_object": DeleteObjectTagging_non_existing_object,
"DeleteObjectTagging_success_status": DeleteObjectTagging_success_status,
"DeleteObjectTagging_success": DeleteObjectTagging_success,
"CreateMultipartUpload_non_existing_bucket": CreateMultipartUpload_non_existing_bucket,
"CreateMultipartUpload_success": CreateMultipartUpload_success,
"UploadPart_non_existing_bucket": UploadPart_non_existing_bucket,
"UploadPart_invalid_part_number": UploadPart_invalid_part_number,
"UploadPart_non_existing_key": UploadPart_non_existing_key,
"UploadPart_non_existing_mp_upload": UploadPart_non_existing_mp_upload,
"UploadPart_success": UploadPart_success,
"UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket,
"UploadPartCopy_incorrect_uploadId": UploadPartCopy_incorrect_uploadId,
"UploadPartCopy_incorrect_object_key": UploadPartCopy_incorrect_object_key,
"UploadPartCopy_invalid_part_number": UploadPartCopy_invalid_part_number,
"UploadPartCopy_invalid_copy_source": UploadPartCopy_invalid_copy_source,
"UploadPartCopy_non_existing_source_bucket": UploadPartCopy_non_existing_source_bucket,
"UploadPartCopy_non_existing_source_object_key": UploadPartCopy_non_existing_source_object_key,
"UploadPartCopy_success": UploadPartCopy_success,
"UploadPartCopy_by_range_invalid_range": UploadPartCopy_by_range_invalid_range,
"UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size,
"UploadPartCopy_by_range_success": UploadPartCopy_by_range_success,
"ListParts_incorrect_uploadId": ListParts_incorrect_uploadId,
"ListParts_incorrect_object_key": ListParts_incorrect_object_key,
"ListParts_success": ListParts_success,
"ListMultipartUploads_non_existing_bucket": ListMultipartUploads_non_existing_bucket,
"ListMultipartUploads_empty_result": ListMultipartUploads_empty_result,
"ListMultipartUploads_invalid_max_uploads": ListMultipartUploads_invalid_max_uploads,
"ListMultipartUploads_max_uploads": ListMultipartUploads_max_uploads,
"ListMultipartUploads_incorrect_next_key_marker": ListMultipartUploads_incorrect_next_key_marker,
"ListMultipartUploads_ignore_upload_id_marker": ListMultipartUploads_ignore_upload_id_marker,
"ListMultipartUploads_success": ListMultipartUploads_success,
"AbortMultipartUpload_non_existing_bucket": AbortMultipartUpload_non_existing_bucket,
"AbortMultipartUpload_incorrect_uploadId": AbortMultipartUpload_incorrect_uploadId,
"AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key,
"AbortMultipartUpload_success": AbortMultipartUpload_success,
"AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code,
"CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket,
"CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number,
"CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag,
"CompleteMultipartUpload_success": CompleteMultipartUpload_success,
"PutBucketAcl_non_existing_bucket": PutBucketAcl_non_existing_bucket,
"PutBucketAcl_invalid_acl_canned_and_acp": PutBucketAcl_invalid_acl_canned_and_acp,
"PutBucketAcl_invalid_acl_canned_and_grants": PutBucketAcl_invalid_acl_canned_and_grants,
"PutBucketAcl_invalid_acl_acp_and_grants": PutBucketAcl_invalid_acl_acp_and_grants,
"PutBucketAcl_invalid_owner": PutBucketAcl_invalid_owner,
"PutBucketAcl_success_access_denied": PutBucketAcl_success_access_denied,
"PutBucketAcl_success_grants": PutBucketAcl_success_grants,
"PutBucketAcl_success_canned_acl": PutBucketAcl_success_canned_acl,
"PutBucketAcl_success_acp": PutBucketAcl_success_acp,
"GetBucketAcl_non_existing_bucket": GetBucketAcl_non_existing_bucket,
"GetBucketAcl_access_denied": GetBucketAcl_access_denied,
"GetBucketAcl_success": GetBucketAcl_success,
"PutObject_overwrite_dir_obj": PutObject_overwrite_dir_obj,
"PutObject_overwrite_file_obj": PutObject_overwrite_file_obj,
"PutObject_dir_obj_with_data": PutObject_dir_obj_with_data,
"CreateMultipartUpload_dir_obj": CreateMultipartUpload_dir_obj,
"IAM_user_access_denied": IAM_user_access_denied,
"IAM_userplus_access_denied": IAM_userplus_access_denied,
"IAM_userplus_CreateBucket": IAM_userplus_CreateBucket,
"IAM_admin_ChangeBucketOwner": IAM_admin_ChangeBucketOwner,
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -12,7 +12,6 @@ import (
"io"
rnd "math/rand"
"net/http"
"net/url"
"os"
"os/exec"
"strings"
@@ -28,11 +27,9 @@ import (
)
var (
bcktCount = 0
succUsrCrt = "The user has been created successfully"
failUsrCrt = "failed to create a user: update iam data: account already exists"
adminAccessDeniedMsg = "access denied: only admin users have access to this resource"
succDeleteUserMsg = "The user has been deleted successfully"
bcktCount = 0
succUsrCrt = "The user has been created successfully"
failUsrCrt = "failed to create a user: update iam data: account already exists"
)
func getBucketName() string {
@@ -63,7 +60,7 @@ func teardown(s *S3Conf, bucket string) error {
})
cancel()
if err != nil {
return fmt.Errorf("failed to delete object %v: %w", *key, err)
return fmt.Errorf("failed to delete object %v: %v", *key, err)
}
return nil
}
@@ -74,7 +71,7 @@ func teardown(s *S3Conf, bucket string) error {
out, err := s3client.ListObjectsV2(ctx, in)
cancel()
if err != nil {
return fmt.Errorf("failed to list objects: %w", err)
return fmt.Errorf("failed to list objects: %v", err)
}
for _, item := range out.Contents {
@@ -84,7 +81,7 @@ func teardown(s *S3Conf, bucket string) error {
}
}
if out.IsTruncated != nil && *out.IsTruncated {
if out.IsTruncated {
in.ContinuationToken = out.ContinuationToken
} else {
break
@@ -99,32 +96,31 @@ func teardown(s *S3Conf, bucket string) error {
return err
}
func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) error {
func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) {
runF(testName)
bucketName := getBucketName()
err := setup(s, bucketName)
if err != nil {
failF("%v: failed to create a bucket: %v", testName, err)
return fmt.Errorf("%v: failed to create a bucket: %w", testName, err)
failF("%v: failed to create a bucket: %v", testName, err.Error())
return
}
client := s3.NewFromConfig(s.Config())
handlerErr := handler(client, bucketName)
if handlerErr != nil {
failF("%v: %v", testName, handlerErr)
failF("%v: %v", testName, handlerErr.Error())
}
err = teardown(s, bucketName)
if err != nil {
fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err)
if handlerErr == nil {
return fmt.Errorf("%v: failed to delete the bucket: %w", testName, err)
failF("%v: failed to delete the bucket: %v", testName, err.Error())
} else {
fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err.Error())
}
}
if handlerErr == nil {
passF(testName)
}
return handlerErr
}
type authConfig struct {
@@ -136,35 +132,20 @@ type authConfig struct {
date time.Time
}
func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) error {
func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) {
runF(cfg.testName)
req, err := createSignedReq(cfg.method, s.endpoint, cfg.path, s.awsID, s.awsSecret, cfg.service, s.awsRegion, cfg.body, cfg.date)
if err != nil {
failF("%v: %v", cfg.testName, err)
return fmt.Errorf("%v: %w", cfg.testName, err)
failF("%v: %v", cfg.testName, err.Error())
return
}
err = handler(req)
if err != nil {
failF("%v: %v", cfg.testName, err)
return fmt.Errorf("%v: %w", cfg.testName, err)
failF("%v: %v", cfg.testName, err.Error())
return
}
passF(cfg.testName)
return nil
}
func presignedAuthHandler(s *S3Conf, testName string, handler func(client *s3.PresignClient) error) error {
runF(testName)
clt := s3.NewPresignClient(s3.NewFromConfig(s.Config()))
err := handler(clt)
if err != nil {
failF("%v: %v", testName, err)
return fmt.Errorf("%v: %w", testName, err)
}
passF(testName)
return nil
}
func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
@@ -224,15 +205,16 @@ func checkApiErr(err error, apiErr s3err.APIError) error {
}
return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
} else {
return fmt.Errorf("expected aws api error, instead got: %v", err.Error())
}
return fmt.Errorf("expected aws api error, instead got: %w", err)
}
func checkSdkApiErr(err error, code string) error {
var ae smithy.APIError
if errors.As(err, &ae) {
if ae.ErrorCode() != code {
return fmt.Errorf("expected %v, instead got %v", code, ae.ErrorCode())
return fmt.Errorf("expected %v, instead got %v", ae.ErrorCode(), code)
}
return nil
}
@@ -311,7 +293,7 @@ func compareParts(parts1, parts2 []types.Part) bool {
}
for i, prt := range parts1 {
if *prt.PartNumber != *parts2[i].PartNumber {
if prt.PartNumber != parts2[i].PartNumber {
return false
}
if *prt.ETag != *parts2[i].ETag {
@@ -502,23 +484,20 @@ func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId s
return parts, err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
pn := int32(partNumber)
out, err := client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
Body: bytes.NewReader(partBuffer),
PartNumber: &pn,
PartNumber: int32(partNumber),
})
cancel()
if err != nil {
return parts, err
} else {
parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)})
offset += partSize
}
parts = append(parts, types.Part{
ETag: out.ETag,
PartNumber: &pn,
})
offset += partSize
}
return parts, err
@@ -543,18 +522,6 @@ func createUsers(s *S3Conf, users []user) error {
return nil
}
func deleteUser(s *S3Conf, access string) error {
out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "delete-user", "-a", access)
if err != nil {
return err
}
if !strings.Contains(string(out), succDeleteUserMsg) {
return fmt.Errorf("failed to delete the user account")
}
return nil
}
func changeBucketsOwner(s *S3Conf, buckets []string, owner string) error {
for _, bucket := range buckets {
out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "-er", s.endpoint, "change-bucket-owner", "-b", bucket, "-o", owner)
@@ -580,26 +547,3 @@ func genRandString(length int) string {
}
return string(result)
}
const (
credAccess int = iota
credDate
credRegion
credService
credTerminator
)
func changeAuthCred(uri, newVal string, index int) (string, error) {
urlParsed, err := url.Parse(uri)
if err != nil {
return "", err
}
queries := urlParsed.Query()
creds := strings.Split(queries.Get("X-Amz-Credential"), "/")
creds[index] = newVal
queries.Set("X-Amz-Credential", strings.Join(creds, "/"))
urlParsed.RawQuery = queries.Encode()
return urlParsed.String(), nil
}

View File

@@ -1,7 +1,6 @@
#!/bin/bash
# make temp dirs
rm -rf /tmp/gw
mkdir /tmp/gw
rm -rf /tmp/covdata
mkdir /tmp/covdata
@@ -20,21 +19,8 @@ if ! kill -0 $GW_PID; then
fi
# run tests
# full flow tests
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 full-flow; then
echo "full flow tests failed"
kill $GW_PID
exit 1
fi
# posix tests
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 posix; then
echo "posix tests failed"
kill $GW_PID
exit 1
fi
# iam tests
if ! ./versitygw test -a user -s pass -e http://127.0.0.1:7070 iam; then
echo "iam tests failed"
echo "tests failed"
kill $GW_PID
exit 1
fi

View File

@@ -43,8 +43,8 @@ func (c AdminController) CreateUser(ctx *fiber.Ctx) error {
return fmt.Errorf("failed to parse request body: %w", err)
}
if usr.Role != auth.RoleAdmin && usr.Role != auth.RoleUser && usr.Role != auth.RoleUserPlus {
return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin', 'userplus'")
if usr.Role != "user" && usr.Role != "admin" {
return fmt.Errorf("invalid parameters: user role have to be one of the following: 'user', 'admin'")
}
err = c.iam.CreateAccount(usr)

View File

@@ -4,7 +4,6 @@
package controllers
import (
"bufio"
"context"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/versity/versitygw/backend"
@@ -35,7 +34,7 @@ var _ backend.Backend = &BackendMock{}
// CopyObjectFunc: func(contextMoqParam context.Context, copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
// panic("mock out the CopyObject method")
// },
// CreateBucketFunc: func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error {
// CreateBucketFunc: func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput) error {
// panic("mock out the CreateBucket method")
// },
// CreateMultipartUploadFunc: func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
@@ -44,9 +43,6 @@ var _ backend.Backend = &BackendMock{}
// DeleteBucketFunc: func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error {
// panic("mock out the DeleteBucket method")
// },
// DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
// panic("mock out the DeleteBucketTagging method")
// },
// DeleteObjectFunc: func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error {
// panic("mock out the DeleteObject method")
// },
@@ -59,12 +55,6 @@ var _ backend.Backend = &BackendMock{}
// GetBucketAclFunc: func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) {
// panic("mock out the GetBucketAcl method")
// },
// GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
// panic("mock out the GetBucketTagging method")
// },
// GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
// panic("mock out the GetBucketVersioning method")
// },
// GetObjectFunc: func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
// panic("mock out the GetObject method")
// },
@@ -92,9 +82,6 @@ var _ backend.Backend = &BackendMock{}
// ListMultipartUploadsFunc: func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
// panic("mock out the ListMultipartUploads method")
// },
// ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
// panic("mock out the ListObjectVersions method")
// },
// ListObjectsFunc: func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
// panic("mock out the ListObjects method")
// },
@@ -107,12 +94,6 @@ var _ backend.Backend = &BackendMock{}
// PutBucketAclFunc: func(contextMoqParam context.Context, bucket string, data []byte) error {
// panic("mock out the PutBucketAcl method")
// },
// PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
// panic("mock out the PutBucketTagging method")
// },
// PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
// panic("mock out the PutBucketVersioning method")
// },
// PutObjectFunc: func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) {
// panic("mock out the PutObject method")
// },
@@ -125,7 +106,7 @@ var _ backend.Backend = &BackendMock{}
// RestoreObjectFunc: func(contextMoqParam context.Context, restoreObjectInput *s3.RestoreObjectInput) error {
// panic("mock out the RestoreObject method")
// },
// SelectObjectContentFunc: func(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
// SelectObjectContentFunc: func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
// panic("mock out the SelectObjectContent method")
// },
// ShutdownFunc: func() {
@@ -160,7 +141,7 @@ type BackendMock struct {
CopyObjectFunc func(contextMoqParam context.Context, copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error)
// CreateBucketFunc mocks the CreateBucket method.
CreateBucketFunc func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error
CreateBucketFunc func(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput) error
// CreateMultipartUploadFunc mocks the CreateMultipartUpload method.
CreateMultipartUploadFunc func(contextMoqParam context.Context, createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error)
@@ -168,9 +149,6 @@ type BackendMock struct {
// DeleteBucketFunc mocks the DeleteBucket method.
DeleteBucketFunc func(contextMoqParam context.Context, deleteBucketInput *s3.DeleteBucketInput) error
// DeleteBucketTaggingFunc mocks the DeleteBucketTagging method.
DeleteBucketTaggingFunc func(contextMoqParam context.Context, bucket string) error
// DeleteObjectFunc mocks the DeleteObject method.
DeleteObjectFunc func(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error
@@ -183,12 +161,6 @@ type BackendMock struct {
// GetBucketAclFunc mocks the GetBucketAcl method.
GetBucketAclFunc func(contextMoqParam context.Context, getBucketAclInput *s3.GetBucketAclInput) ([]byte, error)
// GetBucketTaggingFunc mocks the GetBucketTagging method.
GetBucketTaggingFunc func(contextMoqParam context.Context, bucket string) (map[string]string, error)
// GetBucketVersioningFunc mocks the GetBucketVersioning method.
GetBucketVersioningFunc func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error)
// GetObjectFunc mocks the GetObject method.
GetObjectFunc func(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error)
@@ -216,9 +188,6 @@ type BackendMock struct {
// ListMultipartUploadsFunc mocks the ListMultipartUploads method.
ListMultipartUploadsFunc func(contextMoqParam context.Context, listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error)
// ListObjectVersionsFunc mocks the ListObjectVersions method.
ListObjectVersionsFunc func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
// ListObjectsFunc mocks the ListObjects method.
ListObjectsFunc func(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error)
@@ -231,12 +200,6 @@ type BackendMock struct {
// PutBucketAclFunc mocks the PutBucketAcl method.
PutBucketAclFunc func(contextMoqParam context.Context, bucket string, data []byte) error
// PutBucketTaggingFunc mocks the PutBucketTagging method.
PutBucketTaggingFunc func(contextMoqParam context.Context, bucket string, tags map[string]string) error
// PutBucketVersioningFunc mocks the PutBucketVersioning method.
PutBucketVersioningFunc func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error
// PutObjectFunc mocks the PutObject method.
PutObjectFunc func(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error)
@@ -250,7 +213,7 @@ type BackendMock struct {
RestoreObjectFunc func(contextMoqParam context.Context, restoreObjectInput *s3.RestoreObjectInput) error
// SelectObjectContentFunc mocks the SelectObjectContent method.
SelectObjectContentFunc func(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
SelectObjectContentFunc func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
// ShutdownFunc mocks the Shutdown method.
ShutdownFunc func()
@@ -302,8 +265,6 @@ type BackendMock struct {
ContextMoqParam context.Context
// CreateBucketInput is the createBucketInput argument value.
CreateBucketInput *s3.CreateBucketInput
// DefaultACL is the defaultACL argument value.
DefaultACL []byte
}
// CreateMultipartUpload holds details about calls to the CreateMultipartUpload method.
CreateMultipartUpload []struct {
@@ -319,13 +280,6 @@ type BackendMock struct {
// DeleteBucketInput is the deleteBucketInput argument value.
DeleteBucketInput *s3.DeleteBucketInput
}
// DeleteBucketTagging holds details about calls to the DeleteBucketTagging method.
DeleteBucketTagging []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
}
// DeleteObject holds details about calls to the DeleteObject method.
DeleteObject []struct {
// ContextMoqParam is the contextMoqParam argument value.
@@ -356,20 +310,6 @@ type BackendMock struct {
// GetBucketAclInput is the getBucketAclInput argument value.
GetBucketAclInput *s3.GetBucketAclInput
}
// GetBucketTagging holds details about calls to the GetBucketTagging method.
GetBucketTagging []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
}
// GetBucketVersioning holds details about calls to the GetBucketVersioning method.
GetBucketVersioning []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
}
// GetObject holds details about calls to the GetObject method.
GetObject []struct {
// ContextMoqParam is the contextMoqParam argument value.
@@ -437,13 +377,6 @@ type BackendMock struct {
// ListMultipartUploadsInput is the listMultipartUploadsInput argument value.
ListMultipartUploadsInput *s3.ListMultipartUploadsInput
}
// ListObjectVersions holds details about calls to the ListObjectVersions method.
ListObjectVersions []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// ListObjectVersionsInput is the listObjectVersionsInput argument value.
ListObjectVersionsInput *s3.ListObjectVersionsInput
}
// ListObjects holds details about calls to the ListObjects method.
ListObjects []struct {
// ContextMoqParam is the contextMoqParam argument value.
@@ -474,22 +407,6 @@ type BackendMock struct {
// Data is the data argument value.
Data []byte
}
// PutBucketTagging holds details about calls to the PutBucketTagging method.
PutBucketTagging []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// Bucket is the bucket argument value.
Bucket string
// Tags is the tags argument value.
Tags map[string]string
}
// PutBucketVersioning holds details about calls to the PutBucketVersioning method.
PutBucketVersioning []struct {
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// PutBucketVersioningInput is the putBucketVersioningInput argument value.
PutBucketVersioningInput *s3.PutBucketVersioningInput
}
// PutObject holds details about calls to the PutObject method.
PutObject []struct {
// ContextMoqParam is the contextMoqParam argument value.
@@ -524,10 +441,10 @@ type BackendMock struct {
}
// SelectObjectContent holds details about calls to the SelectObjectContent method.
SelectObjectContent []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// Input is the input argument value.
Input *s3.SelectObjectContentInput
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// SelectObjectContentInput is the selectObjectContentInput argument value.
SelectObjectContentInput *s3.SelectObjectContentInput
}
// Shutdown holds details about calls to the Shutdown method.
Shutdown []struct {
@@ -557,13 +474,10 @@ type BackendMock struct {
lockCreateBucket sync.RWMutex
lockCreateMultipartUpload sync.RWMutex
lockDeleteBucket sync.RWMutex
lockDeleteBucketTagging sync.RWMutex
lockDeleteObject sync.RWMutex
lockDeleteObjectTagging sync.RWMutex
lockDeleteObjects sync.RWMutex
lockGetBucketAcl sync.RWMutex
lockGetBucketTagging sync.RWMutex
lockGetBucketVersioning sync.RWMutex
lockGetObject sync.RWMutex
lockGetObjectAcl sync.RWMutex
lockGetObjectAttributes sync.RWMutex
@@ -573,13 +487,10 @@ type BackendMock struct {
lockListBuckets sync.RWMutex
lockListBucketsAndOwners sync.RWMutex
lockListMultipartUploads sync.RWMutex
lockListObjectVersions sync.RWMutex
lockListObjects sync.RWMutex
lockListObjectsV2 sync.RWMutex
lockListParts sync.RWMutex
lockPutBucketAcl sync.RWMutex
lockPutBucketTagging sync.RWMutex
lockPutBucketVersioning sync.RWMutex
lockPutObject sync.RWMutex
lockPutObjectAcl sync.RWMutex
lockPutObjectTagging sync.RWMutex
@@ -740,23 +651,21 @@ func (mock *BackendMock) CopyObjectCalls() []struct {
}
// CreateBucket calls CreateBucketFunc.
func (mock *BackendMock) CreateBucket(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput, defaultACL []byte) error {
func (mock *BackendMock) CreateBucket(contextMoqParam context.Context, createBucketInput *s3.CreateBucketInput) error {
if mock.CreateBucketFunc == nil {
panic("BackendMock.CreateBucketFunc: method is nil but Backend.CreateBucket was just called")
}
callInfo := struct {
ContextMoqParam context.Context
CreateBucketInput *s3.CreateBucketInput
DefaultACL []byte
}{
ContextMoqParam: contextMoqParam,
CreateBucketInput: createBucketInput,
DefaultACL: defaultACL,
}
mock.lockCreateBucket.Lock()
mock.calls.CreateBucket = append(mock.calls.CreateBucket, callInfo)
mock.lockCreateBucket.Unlock()
return mock.CreateBucketFunc(contextMoqParam, createBucketInput, defaultACL)
return mock.CreateBucketFunc(contextMoqParam, createBucketInput)
}
// CreateBucketCalls gets all the calls that were made to CreateBucket.
@@ -766,12 +675,10 @@ func (mock *BackendMock) CreateBucket(contextMoqParam context.Context, createBuc
func (mock *BackendMock) CreateBucketCalls() []struct {
ContextMoqParam context.Context
CreateBucketInput *s3.CreateBucketInput
DefaultACL []byte
} {
var calls []struct {
ContextMoqParam context.Context
CreateBucketInput *s3.CreateBucketInput
DefaultACL []byte
}
mock.lockCreateBucket.RLock()
calls = mock.calls.CreateBucket
@@ -851,42 +758,6 @@ func (mock *BackendMock) DeleteBucketCalls() []struct {
return calls
}
// DeleteBucketTagging calls DeleteBucketTaggingFunc.
func (mock *BackendMock) DeleteBucketTagging(contextMoqParam context.Context, bucket string) error {
if mock.DeleteBucketTaggingFunc == nil {
panic("BackendMock.DeleteBucketTaggingFunc: method is nil but Backend.DeleteBucketTagging was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
}
mock.lockDeleteBucketTagging.Lock()
mock.calls.DeleteBucketTagging = append(mock.calls.DeleteBucketTagging, callInfo)
mock.lockDeleteBucketTagging.Unlock()
return mock.DeleteBucketTaggingFunc(contextMoqParam, bucket)
}
// DeleteBucketTaggingCalls gets all the calls that were made to DeleteBucketTagging.
// Check the length with:
//
// len(mockedBackend.DeleteBucketTaggingCalls())
func (mock *BackendMock) DeleteBucketTaggingCalls() []struct {
ContextMoqParam context.Context
Bucket string
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
}
mock.lockDeleteBucketTagging.RLock()
calls = mock.calls.DeleteBucketTagging
mock.lockDeleteBucketTagging.RUnlock()
return calls
}
// DeleteObject calls DeleteObjectFunc.
func (mock *BackendMock) DeleteObject(contextMoqParam context.Context, deleteObjectInput *s3.DeleteObjectInput) error {
if mock.DeleteObjectFunc == nil {
@@ -1035,78 +906,6 @@ func (mock *BackendMock) GetBucketAclCalls() []struct {
return calls
}
// GetBucketTagging calls GetBucketTaggingFunc.
func (mock *BackendMock) GetBucketTagging(contextMoqParam context.Context, bucket string) (map[string]string, error) {
if mock.GetBucketTaggingFunc == nil {
panic("BackendMock.GetBucketTaggingFunc: method is nil but Backend.GetBucketTagging was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
}
mock.lockGetBucketTagging.Lock()
mock.calls.GetBucketTagging = append(mock.calls.GetBucketTagging, callInfo)
mock.lockGetBucketTagging.Unlock()
return mock.GetBucketTaggingFunc(contextMoqParam, bucket)
}
// GetBucketTaggingCalls gets all the calls that were made to GetBucketTagging.
// Check the length with:
//
// len(mockedBackend.GetBucketTaggingCalls())
func (mock *BackendMock) GetBucketTaggingCalls() []struct {
ContextMoqParam context.Context
Bucket string
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
}
mock.lockGetBucketTagging.RLock()
calls = mock.calls.GetBucketTagging
mock.lockGetBucketTagging.RUnlock()
return calls
}
// GetBucketVersioning calls GetBucketVersioningFunc.
func (mock *BackendMock) GetBucketVersioning(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
if mock.GetBucketVersioningFunc == nil {
panic("BackendMock.GetBucketVersioningFunc: method is nil but Backend.GetBucketVersioning was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
}
mock.lockGetBucketVersioning.Lock()
mock.calls.GetBucketVersioning = append(mock.calls.GetBucketVersioning, callInfo)
mock.lockGetBucketVersioning.Unlock()
return mock.GetBucketVersioningFunc(contextMoqParam, bucket)
}
// GetBucketVersioningCalls gets all the calls that were made to GetBucketVersioning.
// Check the length with:
//
// len(mockedBackend.GetBucketVersioningCalls())
func (mock *BackendMock) GetBucketVersioningCalls() []struct {
ContextMoqParam context.Context
Bucket string
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
}
mock.lockGetBucketVersioning.RLock()
calls = mock.calls.GetBucketVersioning
mock.lockGetBucketVersioning.RUnlock()
return calls
}
// GetObject calls GetObjectFunc.
func (mock *BackendMock) GetObject(contextMoqParam context.Context, getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
if mock.GetObjectFunc == nil {
@@ -1439,42 +1238,6 @@ func (mock *BackendMock) ListMultipartUploadsCalls() []struct {
return calls
}
// ListObjectVersions calls ListObjectVersionsFunc.
func (mock *BackendMock) ListObjectVersions(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
if mock.ListObjectVersionsFunc == nil {
panic("BackendMock.ListObjectVersionsFunc: method is nil but Backend.ListObjectVersions was just called")
}
callInfo := struct {
ContextMoqParam context.Context
ListObjectVersionsInput *s3.ListObjectVersionsInput
}{
ContextMoqParam: contextMoqParam,
ListObjectVersionsInput: listObjectVersionsInput,
}
mock.lockListObjectVersions.Lock()
mock.calls.ListObjectVersions = append(mock.calls.ListObjectVersions, callInfo)
mock.lockListObjectVersions.Unlock()
return mock.ListObjectVersionsFunc(contextMoqParam, listObjectVersionsInput)
}
// ListObjectVersionsCalls gets all the calls that were made to ListObjectVersions.
// Check the length with:
//
// len(mockedBackend.ListObjectVersionsCalls())
func (mock *BackendMock) ListObjectVersionsCalls() []struct {
ContextMoqParam context.Context
ListObjectVersionsInput *s3.ListObjectVersionsInput
} {
var calls []struct {
ContextMoqParam context.Context
ListObjectVersionsInput *s3.ListObjectVersionsInput
}
mock.lockListObjectVersions.RLock()
calls = mock.calls.ListObjectVersions
mock.lockListObjectVersions.RUnlock()
return calls
}
// ListObjects calls ListObjectsFunc.
func (mock *BackendMock) ListObjects(contextMoqParam context.Context, listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
if mock.ListObjectsFunc == nil {
@@ -1623,82 +1386,6 @@ func (mock *BackendMock) PutBucketAclCalls() []struct {
return calls
}
// PutBucketTagging calls PutBucketTaggingFunc.
func (mock *BackendMock) PutBucketTagging(contextMoqParam context.Context, bucket string, tags map[string]string) error {
if mock.PutBucketTaggingFunc == nil {
panic("BackendMock.PutBucketTaggingFunc: method is nil but Backend.PutBucketTagging was just called")
}
callInfo := struct {
ContextMoqParam context.Context
Bucket string
Tags map[string]string
}{
ContextMoqParam: contextMoqParam,
Bucket: bucket,
Tags: tags,
}
mock.lockPutBucketTagging.Lock()
mock.calls.PutBucketTagging = append(mock.calls.PutBucketTagging, callInfo)
mock.lockPutBucketTagging.Unlock()
return mock.PutBucketTaggingFunc(contextMoqParam, bucket, tags)
}
// PutBucketTaggingCalls gets all the calls that were made to PutBucketTagging.
// Check the length with:
//
// len(mockedBackend.PutBucketTaggingCalls())
func (mock *BackendMock) PutBucketTaggingCalls() []struct {
ContextMoqParam context.Context
Bucket string
Tags map[string]string
} {
var calls []struct {
ContextMoqParam context.Context
Bucket string
Tags map[string]string
}
mock.lockPutBucketTagging.RLock()
calls = mock.calls.PutBucketTagging
mock.lockPutBucketTagging.RUnlock()
return calls
}
// PutBucketVersioning calls PutBucketVersioningFunc.
func (mock *BackendMock) PutBucketVersioning(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
if mock.PutBucketVersioningFunc == nil {
panic("BackendMock.PutBucketVersioningFunc: method is nil but Backend.PutBucketVersioning was just called")
}
callInfo := struct {
ContextMoqParam context.Context
PutBucketVersioningInput *s3.PutBucketVersioningInput
}{
ContextMoqParam: contextMoqParam,
PutBucketVersioningInput: putBucketVersioningInput,
}
mock.lockPutBucketVersioning.Lock()
mock.calls.PutBucketVersioning = append(mock.calls.PutBucketVersioning, callInfo)
mock.lockPutBucketVersioning.Unlock()
return mock.PutBucketVersioningFunc(contextMoqParam, putBucketVersioningInput)
}
// PutBucketVersioningCalls gets all the calls that were made to PutBucketVersioning.
// Check the length with:
//
// len(mockedBackend.PutBucketVersioningCalls())
func (mock *BackendMock) PutBucketVersioningCalls() []struct {
ContextMoqParam context.Context
PutBucketVersioningInput *s3.PutBucketVersioningInput
} {
var calls []struct {
ContextMoqParam context.Context
PutBucketVersioningInput *s3.PutBucketVersioningInput
}
mock.lockPutBucketVersioning.RLock()
calls = mock.calls.PutBucketVersioning
mock.lockPutBucketVersioning.RUnlock()
return calls
}
// PutObject calls PutObjectFunc.
func (mock *BackendMock) PutObject(contextMoqParam context.Context, putObjectInput *s3.PutObjectInput) (string, error) {
if mock.PutObjectFunc == nil {
@@ -1852,21 +1539,21 @@ func (mock *BackendMock) RestoreObjectCalls() []struct {
}
// SelectObjectContent calls SelectObjectContentFunc.
func (mock *BackendMock) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
func (mock *BackendMock) SelectObjectContent(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
if mock.SelectObjectContentFunc == nil {
panic("BackendMock.SelectObjectContentFunc: method is nil but Backend.SelectObjectContent was just called")
}
callInfo := struct {
Ctx context.Context
Input *s3.SelectObjectContentInput
ContextMoqParam context.Context
SelectObjectContentInput *s3.SelectObjectContentInput
}{
Ctx: ctx,
Input: input,
ContextMoqParam: contextMoqParam,
SelectObjectContentInput: selectObjectContentInput,
}
mock.lockSelectObjectContent.Lock()
mock.calls.SelectObjectContent = append(mock.calls.SelectObjectContent, callInfo)
mock.lockSelectObjectContent.Unlock()
return mock.SelectObjectContentFunc(ctx, input)
return mock.SelectObjectContentFunc(contextMoqParam, selectObjectContentInput)
}
// SelectObjectContentCalls gets all the calls that were made to SelectObjectContent.
@@ -1874,12 +1561,12 @@ func (mock *BackendMock) SelectObjectContent(ctx context.Context, input *s3.Sele
//
// len(mockedBackend.SelectObjectContentCalls())
func (mock *BackendMock) SelectObjectContentCalls() []struct {
Ctx context.Context
Input *s3.SelectObjectContentInput
ContextMoqParam context.Context
SelectObjectContentInput *s3.SelectObjectContentInput
} {
var calls []struct {
Ctx context.Context
Input *s3.SelectObjectContentInput
ContextMoqParam context.Context
SelectObjectContentInput *s3.SelectObjectContentInput
}
mock.lockSelectObjectContent.RLock()
calls = mock.calls.SelectObjectContent

File diff suppressed because it is too large Load Diff

View File

@@ -15,7 +15,6 @@
package controllers
import (
"bufio"
"context"
"encoding/json"
"fmt"
@@ -175,7 +174,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
now := time.Now()
app := fiber.New()
contentLength := int64(1000)
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -196,7 +194,7 @@ func TestS3ApiController_GetActions(t *testing.T) {
ContentType: getPtr("application/xml"),
ContentEncoding: getPtr("gzip"),
ETag: getPtr("98sda7f97sa9df798sd79f8as9df"),
ContentLength: &contentLength,
ContentLength: 1000,
LastModified: &now,
StorageClass: "storage class",
}, nil
@@ -343,15 +341,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return &s3.ListObjectsOutput{}, nil
},
GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
return map[string]string{}, nil
},
GetBucketVersioningFunc: func(contextMoqParam context.Context, bucket string) (*s3.GetBucketVersioningOutput, error) {
return &s3.GetBucketVersioningOutput{}, nil
},
ListObjectVersionsFunc: func(contextMoqParam context.Context, listObjectVersionsInput *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) {
return &s3.ListObjectVersionsOutput{}, nil
},
},
}
@@ -374,9 +363,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
ListObjectsFunc: func(context.Context, *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
return nil, s3err.GetAPIError(s3err.ErrNotImplemented)
},
GetBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) (map[string]string, error) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
},
},
}
appError := fiber.New()
@@ -396,24 +382,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
wantErr bool
statusCode int
}{
{
name: "Get-bucket-tagging-non-existing-bucket",
app: appError,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket?tagging", nil),
},
wantErr: false,
statusCode: 404,
},
{
name: "Get-bucket-tagging-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket?tagging", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "Get-bucket-acl-success",
app: app,
@@ -459,24 +427,6 @@ func TestS3ApiController_ListActions(t *testing.T) {
wantErr: false,
statusCode: 501,
},
{
name: "List-actions-get-bucket-versioning-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket?versioning", nil),
},
wantErr: false,
statusCode: 200,
},
{
name: "List-actions-list-object-versions-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodGet, "/my-bucket?versions", nil),
},
wantErr: false,
statusCode: 200,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -540,24 +490,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
</AccessControlPolicy>
`
tagBody := `
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>organization</Key>
<Value>marketing</Value>
</Tag>
</TagSet>
</Tagging>
`
versioningBody := `
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Status>Enabled</Status>
<MfaDelete>Enabled</MfaDelete>
</VersioningConfiguration>
`
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -566,13 +498,7 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
PutBucketAclFunc: func(context.Context, string, []byte) error {
return nil
},
CreateBucketFunc: func(context.Context, *s3.CreateBucketInput, []byte) error {
return nil
},
PutBucketTaggingFunc: func(contextMoqParam context.Context, bucket string, tags map[string]string) error {
return nil
},
PutBucketVersioningFunc: func(contextMoqParam context.Context, putBucketVersioningInput *s3.PutBucketVersioningInput) error {
CreateBucketFunc: func(context.Context, *s3.CreateBucketInput) error {
return nil
},
},
@@ -615,42 +541,6 @@ func TestS3ApiController_PutBucketActions(t *testing.T) {
wantErr bool
statusCode int
}{
{
name: "Put-bucket-tagging-invalid-body",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Put-bucket-tagging-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket?tagging", strings.NewReader(tagBody)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Put-bucket-versioning-invalid-body",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket?versioning", nil),
},
wantErr: false,
statusCode: 400,
},
{
name: "Put-bucket-versioning-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodPut, "/my-bucket?versioning", strings.NewReader(versioningBody)),
},
wantErr: false,
statusCode: 200,
},
{
name: "Put-bucket-acl-invalid-acl",
app: app,
@@ -958,12 +848,12 @@ func TestS3ApiController_PutActions(t *testing.T) {
resp, err := tt.app.Test(tt.args.req)
if (err != nil) != tt.wantErr {
t.Errorf("S3ApiController.PutActions() %v error = %v, wantErr %v",
t.Errorf("S3ApiController.GetActions() %v error = %v, wantErr %v",
tt.name, err, tt.wantErr)
}
if resp.StatusCode != tt.statusCode {
t.Errorf("S3ApiController.PutActions() %v statusCode = %v, wantStatusCode = %v",
t.Errorf("S3ApiController.GetActions() %v statusCode = %v, wantStatusCode = %v",
tt.name, resp.StatusCode, tt.statusCode)
}
}
@@ -977,10 +867,10 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
app := fiber.New()
s3ApiController := S3ApiController{
be: &BackendMock{
DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
return nil
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
return acldata, nil
},
DeleteBucketTaggingFunc: func(contextMoqParam context.Context, bucket string) error {
DeleteBucketFunc: func(context.Context, *s3.DeleteBucketInput) error {
return nil
},
},
@@ -1012,15 +902,6 @@ func TestS3ApiController_DeleteBucket(t *testing.T) {
wantErr: false,
statusCode: 204,
},
{
name: "Delete-bucket-tagging-success",
app: app,
args: args{
req: httptest.NewRequest(http.MethodDelete, "/my-bucket?tagging", nil),
},
wantErr: false,
statusCode: 204,
},
}
for _, tt := range tests {
resp, err := tt.app.Test(tt.args.req)
@@ -1317,7 +1198,6 @@ func TestS3ApiController_HeadObject(t *testing.T) {
contentType := "application/xml"
eTag := "Valid etag"
lastModifie := time.Now()
contentLength := int64(64)
s3ApiController := S3ApiController{
be: &BackendMock{
@@ -1327,7 +1207,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return &s3.HeadObjectOutput{
ContentEncoding: &contentEncoding,
ContentLength: &contentLength,
ContentLength: 64,
ContentType: &contentType,
LastModified: &lastModifie,
ETag: &eTag,
@@ -1426,8 +1306,8 @@ func TestS3ApiController_CreateActions(t *testing.T) {
CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
return &s3.CreateMultipartUploadOutput{}, nil
},
SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) {
return func(w *bufio.Writer) {}
SelectObjectContentFunc: func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
return s3response.SelectObjectContentResult{}, nil
},
},
}

View File

@@ -16,7 +16,6 @@ package middlewares
import (
"net/http"
"regexp"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -27,10 +26,6 @@ import (
"github.com/versity/versitygw/s3log"
)
var (
singlePath = regexp.MustCompile(`^/[^/]+/?$`)
)
func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
return func(ctx *fiber.Ctx) error {
isRoot, acct := ctx.Locals("isRoot").(bool), ctx.Locals("account").(auth.Account)
@@ -43,12 +38,8 @@ func AclParser(be backend.Backend, logger s3log.AuditLogger) fiber.Handler {
if ctx.Method() == http.MethodPatch {
return ctx.Next()
}
if singlePath.MatchString(path) &&
ctx.Method() == http.MethodPut &&
!ctx.Request().URI().QueryArgs().Has("acl") &&
!ctx.Request().URI().QueryArgs().Has("tagging") &&
!ctx.Request().URI().QueryArgs().Has("versioning") {
if err := auth.MayCreateBucket(acct, isRoot); err != nil {
if len(pathParts) == 2 && pathParts[1] != "" && ctx.Method() == http.MethodPut && !ctx.Request().URI().QueryArgs().Has("acl") {
if err := auth.IsAdmin(acct, isRoot); err != nil {
return controllers.SendXMLResponse(ctx, nil, err, &controllers.MetaOpts{Logger: logger, Action: "CreateBucket"})
}
return ctx.Next()

View File

@@ -18,11 +18,15 @@ import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"math"
"net/http"
"strconv"
"os"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/smithy-go/logging"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/controllers"
@@ -33,6 +37,7 @@ import (
const (
iso8601Format = "20060102T150405Z"
YYYYMMDD = "20060102"
)
type RootUserConfig struct {
@@ -44,103 +49,141 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
// If account is set in context locals, it means it was presigned url case
_, ok := ctx.Locals("account").(auth.Account)
if ok {
return ctx.Next()
}
ctx.Locals("region", region)
ctx.Locals("startTime", time.Now())
authorization := ctx.Get("Authorization")
if authorization == "" {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger)
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), &controllers.MetaOpts{Logger: logger})
}
authData, err := utils.ParseAuthorization(authorization)
if err != nil {
return sendResponse(ctx, err, logger)
// Check the signature version
authParts := strings.Split(authorization, ",")
for i, el := range authParts {
authParts[i] = strings.TrimSpace(el)
}
if authData.Algorithm != "AWS4-HMAC-SHA256" {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger)
if len(authParts) != 3 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
}
if authData.Region != region {
return sendResponse(ctx, s3err.APIError{
startParts := strings.Split(authParts[0], " ")
if startParts[0] != "AWS4-HMAC-SHA256" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), &controllers.MetaOpts{Logger: logger})
}
credKv := strings.Split(startParts[1], "=")
if len(credKv) != 2 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
}
// Credential variables validation
creds := strings.Split(credKv[1], "/")
if len(creds) != 5 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
}
if creds[4] != "aws4_request" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureTerminationStr), &controllers.MetaOpts{Logger: logger})
}
if creds[3] != "s3" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureIncorrService), &controllers.MetaOpts{Logger: logger})
}
if creds[2] != region {
return controllers.SendResponse(ctx, s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region),
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),
HTTPStatusCode: http.StatusForbidden,
}, logger)
}, &controllers.MetaOpts{Logger: logger})
}
ctx.Locals("isRoot", authData.Access == root.Access)
ctx.Locals("isRoot", creds[0] == root.Access)
account, err := acct.getAccount(authData.Access)
_, err := time.Parse(YYYYMMDD, creds[1])
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
}
signHdrKv := strings.Split(authParts[1], "=")
if len(signHdrKv) != 2 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
}
signedHdrs := strings.Split(signHdrKv[1], ";")
account, err := acct.getAccount(creds[0])
if err == auth.ErrNoSuchUser {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), &controllers.MetaOpts{Logger: logger})
}
if err != nil {
return sendResponse(ctx, err, logger)
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}
ctx.Locals("account", account)
// Check X-Amz-Date header
date := ctx.Get("X-Amz-Date")
if date == "" {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger)
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), &controllers.MetaOpts{Logger: logger})
}
// Parse the date and check the date validity
tdate, err := time.Parse(iso8601Format, date)
if err != nil {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger)
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), &controllers.MetaOpts{Logger: logger})
}
if date[:8] != authData.Date {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger)
if date[:8] != creds[1] {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
}
// Validate the dates difference
err = utils.ValidateDate(tdate)
err = validateDate(tdate)
if err != nil {
return sendResponse(ctx, err, logger)
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}
if utils.IsBigDataAction(ctx) {
// for streaming PUT actions, authorization is deferred
// until end of stream due to need to get length and
// checksum of the stream to validate authorization
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
return utils.NewAuthReader(ctx, r, authData, account.Secret, debug)
})
return ctx.Next()
}
hashPayloadHeader := ctx.Get("X-Amz-Content-Sha256")
ok := isSpecialPayload(hashPayloadHeader)
hashPayload := ctx.Get("X-Amz-Content-Sha256")
if !utils.IsSpecialPayload(hashPayload) {
if !ok {
// Calculate the hash of the request payload
hashedPayload := sha256.Sum256(ctx.Body())
hexPayload := hex.EncodeToString(hashedPayload[:])
// Compare the calculated hash with the hash provided
if hashPayload != hexPayload {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger)
if hashPayloadHeader != hexPayload {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), &controllers.MetaOpts{Logger: logger})
}
}
var contentLength int64
contentLengthStr := ctx.Get("Content-Length")
if contentLengthStr != "" {
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger)
}
}
err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
// Create a new http request instance from fasthttp request
req, err := utils.CreateHttpRequestFromCtx(ctx, signedHdrs)
if err != nil {
return sendResponse(ctx, err, logger)
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
}
signer := v4.NewSigner()
signErr := signer.SignHTTP(req.Context(), aws.Credentials{
AccessKeyID: creds[0],
SecretAccessKey: account.Secret,
}, req, hashPayloadHeader, creds[3], region, tdate, func(options *v4.SignerOptions) {
options.DisableURIPathEscaping = true
if debug {
options.LogSigning = true
options.Logger = logging.NewStandardLogger(os.Stderr)
}
})
if signErr != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
}
parts := strings.Split(req.Header.Get("Authorization"), " ")
if len(parts) < 4 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
}
calculatedSign := strings.Split(parts[3], "=")[1]
expectedSign := strings.Split(authParts[2], "=")[1]
if expectedSign != calculatedSign {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch), &controllers.MetaOpts{Logger: logger})
}
return ctx.Next()
@@ -164,6 +207,39 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
return a.iam.GetUserAccount(access)
}
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
func isSpecialPayload(str string) bool {
specialValues := map[string]bool{
"UNSIGNED-PAYLOAD": true,
"STREAMING-UNSIGNED-PAYLOAD-TRAILER": true,
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD": true,
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER": true,
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD": true,
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
}
return specialValues[str]
}
func validateDate(date time.Time) error {
now := time.Now().UTC()
diff := date.Unix() - now.Unix()
// Checks the dates difference to be less than a minute
if math.Abs(float64(diff)) > 60 {
if diff > 0 {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
}
} else {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
}
}
}
return nil
}

View File

@@ -1,31 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"io"
"github.com/gofiber/fiber/v2"
)
func wrapBodyReader(ctx *fiber.Ctx, wr func(io.Reader) io.Reader) {
r, ok := ctx.Locals("body-reader").(io.Reader)
if !ok {
r = ctx.Request().BodyStream()
}
r = wr(r)
ctx.Locals("body-reader", r)
}

View File

@@ -1,61 +0,0 @@
// Copyright 2024 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"io"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3log"
)
// ProcessChunkedBody initializes the chunked upload stream if the
// request appears to be a chunked upload
func ProcessChunkedBody(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string) fiber.Handler {
return func(ctx *fiber.Ctx) error {
decodedLength := ctx.Get("X-Amz-Decoded-Content-Length")
if decodedLength == "" {
return ctx.Next()
}
// TODO: validate content length
authData, err := utils.ParseAuthorization(ctx.Get("Authorization"))
if err != nil {
return sendResponse(ctx, err, logger)
}
acct := ctx.Locals("account").(auth.Account)
amzdate := ctx.Get("X-Amz-Date")
date, _ := time.Parse(iso8601Format, amzdate)
if utils.IsBigDataAction(ctx) {
var err error
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
var cr *utils.ChunkReader
cr, err = utils.NewChunkReader(ctx, r, authData, region, acct.Secret, date)
return cr
})
if err != nil {
return sendResponse(ctx, err, logger)
}
return ctx.Next()
}
return ctx.Next()
}
}

View File

@@ -16,11 +16,10 @@ package middlewares
import (
"crypto/md5"
"io"
"encoding/base64"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
@@ -32,20 +31,8 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
return ctx.Next()
}
if utils.IsBigDataAction(ctx) {
var err error
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
r, err = utils.NewHashReader(r, incomingSum, utils.HashTypeMd5)
return r
})
if err != nil {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}
return ctx.Next()
}
sum := md5.Sum(ctx.Body())
calculatedSum := utils.Md5SumString(sum[:])
calculatedSum := base64.StdEncoding.EncodeToString(sum[:])
if incomingSum != calculatedSum {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})

View File

@@ -1,69 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"io"
"time"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.AuditLogger, region string, debug bool) fiber.Handler {
acct := accounts{root: root, iam: iam}
return func(ctx *fiber.Ctx) error {
if ctx.Query("X-Amz-Signature") == "" {
return ctx.Next()
}
ctx.Locals("region", region)
ctx.Locals("startTime", time.Now())
authData, err := utils.ParsePresignedURIParts(ctx)
if err != nil {
return sendResponse(ctx, err, logger)
}
ctx.Locals("isRoot", authData.Access == root.Access)
account, err := acct.getAccount(authData.Access)
if err == auth.ErrNoSuchUser {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
}
if err != nil {
return sendResponse(ctx, err, logger)
}
ctx.Locals("account", account)
if utils.IsBigDataAction(ctx) {
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
return utils.NewPresignedAuthReader(ctx, r, authData, account.Secret, debug)
})
return ctx.Next()
}
err = utils.CheckPresignedSignature(ctx, authData, account.Secret, debug)
if err != nil {
return sendResponse(ctx, err, logger)
}
return ctx.Next()
}
}

Some files were not shown because too many files have changed in this diff Show More