Compare commits

..

1 Commits

Author SHA1 Message Date
jonaustin09
27a8aa66d9 feat: Added s3 proxy comparison to upload, download and throughput benchmark tests 2023-11-20 14:28:02 -05:00
41 changed files with 1013 additions and 2801 deletions

View File

@@ -1,46 +0,0 @@
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
cmd/versitygw/versitygw
/versitygw
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Go workspace file
go.work
# ignore IntelliJ directories
.idea
# auto generated VERSION file
VERSION
# build output
/versitygw.spec
/versitygw.spec.in
*.tar
*.tar.gz
**/rand.data
/profile.txt
dist/
# Release config files
/.github
# Docker configuration files
*Dockerfile
/docker-compose.yml
# read files
/LICENSE
/NOTICE
/CODE_OF_CONDUCT.md
/README.md

View File

@@ -1,6 +0,0 @@
POSIX_PORT=
PROXY_PORT=
ACCESS_KEY_ID=
SECRET_ACCESS_KEY=
IAM_DIR=
SETUP_DIR=

3
.gitignore vendored
View File

@@ -25,9 +25,6 @@ go.work
# ignore IntelliJ directories
.idea
# ignore VS code directories
.vscode
# auto generated VERSION file
VERSION

View File

@@ -9,10 +9,6 @@ builds:
# windows is untested, we can start doing windows releases
# if someone is interested in taking on testing
# - windows
env:
# disable cgo to fix glibc issues: https://github.com/golang/go/issues/58550
# once we need to enable this, we will need to do per distro releases
- CGO_ENABLED=0
main: ./cmd/versitygw
binary: ./cmd/versitygw
id: versitygw

View File

@@ -1,24 +0,0 @@
FROM golang:1.20-alpine
WORKDIR /app
COPY go.mod ./
RUN go mod download
COPY ./ ./
WORKDIR /app/cmd/versitygw
RUN go build -o versitygw
FROM alpine:latest
# These arguments can be overriden when building the image
ARG IAM_DIR=/tmp/vgw
ARG SETUP_DIR=/tmp/vgw
RUN mkdir -p $IAM_DIR
RUN mkdir -p $SETUP_DIR
COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
ENTRYPOINT [ "/app/versitygw" ]

View File

@@ -1,17 +0,0 @@
FROM golang:1.20
WORKDIR /app
COPY go.mod ./
RUN go mod download
COPY ./ ./
ARG IAM_DIR=/tmp/vgw
ARG SETUP_DIR=/tmp/vgw
RUN mkdir -p $IAM_DIR
RUN mkdir -p $SETUP_DIR
RUN go get github.com/githubnemo/CompileDaemon
RUN go install github.com/githubnemo/CompileDaemon

View File

@@ -74,18 +74,3 @@ dist: $(BIN).spec
rm -f VERSION
rm -f $(BIN).spec
gzip -f $(TARFILE)
# Creates and runs S3 gateway instance in a docker container
.PHONY: up-posix
up-posix:
docker compose --env-file .env.dev up posix
# Creates and runs S3 gateway proxy instance in a docker container
.PHONY: up-proxy
up-proxy:
docker compose --env-file .env.dev up proxy
# Creates and runs both S3 gateway and proxy server instances in docker containers
.PHONY: up-app
up-app:
docker compose --env-file .env.dev up

View File

@@ -16,7 +16,6 @@ package auth
import (
"errors"
"fmt"
"time"
)
@@ -44,25 +43,18 @@ type IAMService interface {
var ErrNoSuchUser = errors.New("user not found")
type Opts struct {
Dir string
LDAPServerURL string
LDAPBindDN string
LDAPPassword string
LDAPQueryBase string
LDAPObjClasses string
LDAPAccessAtr string
LDAPSecretAtr string
LDAPRoleAtr string
S3Access string
S3Secret string
S3Region string
S3Bucket string
S3Endpoint string
S3DisableSSlVerfiy bool
S3Debug bool
CacheDisable bool
CacheTTL int
CachePrune int
Dir string
LDAPServerURL string
LDAPBindDN string
LDAPPassword string
LDAPQueryBase string
LDAPObjClasses string
LDAPAccessAtr string
LDAPSecretAtr string
LDAPRoleAtr string
CacheDisable bool
CacheTTL int
CachePrune int
}
func New(o *Opts) (IAMService, error) {
@@ -72,20 +64,12 @@ func New(o *Opts) (IAMService, error) {
switch {
case o.Dir != "":
svc, err = NewInternal(o.Dir)
fmt.Printf("initializing internal IAM with %q\n", o.Dir)
case o.LDAPServerURL != "":
svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
o.LDAPObjClasses)
fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
case o.S3Endpoint != "":
svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
fmt.Printf("initializing S3 IAM with '%v/%v'\n",
o.S3Endpoint, o.S3Bucket)
default:
// if no iam options selected, default to the single user mode
fmt.Println("No IAM service configured, enabling single account mode")
return IAMServiceSingle{}, nil
}

View File

@@ -1,263 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package auth
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sort"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
)
// IAMServiceS3 stores user accounts in an S3 object
// The endpoint, credentials, bucket, and region are provided
// from cli configuration.
// The object format and name is the same as the internal IAM service:
// coming from iAMConfig and iamFile in iam_internal.
type IAMServiceS3 struct {
access string
secret string
region string
bucket string
endpoint string
sslSkipVerify bool
debug bool
client *s3.Client
}
var _ IAMService = &IAMServiceS3{}
func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
if access == "" {
return nil, fmt.Errorf("must provide s3 IAM service access key")
}
if secret == "" {
return nil, fmt.Errorf("must provide s3 IAM service secret key")
}
if region == "" {
return nil, fmt.Errorf("must provide s3 IAM service region")
}
if bucket == "" {
return nil, fmt.Errorf("must provide s3 IAM service bucket")
}
if endpoint == "" {
return nil, fmt.Errorf("must provide s3 IAM service endpoint")
}
i := &IAMServiceS3{
access: access,
secret: secret,
region: region,
bucket: bucket,
endpoint: endpoint,
sslSkipVerify: sslSkipVerify,
debug: debug,
}
cfg, err := i.getConfig()
if err != nil {
return nil, fmt.Errorf("init s3 IAM: %v", err)
}
i.client = s3.NewFromConfig(cfg)
return i, nil
}
func (s *IAMServiceS3) CreateAccount(account Account) error {
conf, err := s.getAccounts()
if err != nil {
return err
}
_, ok := conf.AccessAccounts[account.Access]
if ok {
return fmt.Errorf("account already exists")
}
conf.AccessAccounts[account.Access] = account
return s.storeAccts(conf)
}
func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
conf, err := s.getAccounts()
if err != nil {
return Account{}, err
}
acct, ok := conf.AccessAccounts[access]
if !ok {
return Account{}, ErrNoSuchUser
}
return acct, nil
}
func (s *IAMServiceS3) DeleteUserAccount(access string) error {
conf, err := s.getAccounts()
if err != nil {
return err
}
_, ok := conf.AccessAccounts[access]
if !ok {
return fmt.Errorf("account does not exist")
}
delete(conf.AccessAccounts, access)
return s.storeAccts(conf)
}
func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
conf, err := s.getAccounts()
if err != nil {
return nil, err
}
keys := make([]string, 0, len(conf.AccessAccounts))
for k := range conf.AccessAccounts {
keys = append(keys, k)
}
sort.Strings(keys)
var accs []Account
for _, k := range keys {
accs = append(accs, Account{
Access: k,
Secret: conf.AccessAccounts[k].Secret,
Role: conf.AccessAccounts[k].Role,
UserID: conf.AccessAccounts[k].UserID,
GroupID: conf.AccessAccounts[k].GroupID,
ProjectID: conf.AccessAccounts[k].ProjectID,
})
}
return accs, nil
}
// ResolveEndpoint is used for on prem or non-aws endpoints
func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
URL: s.endpoint,
SigningRegion: s.region,
HostnameImmutable: true,
}, nil
}
func (s *IAMServiceS3) Shutdown() error {
return nil
}
func (s *IAMServiceS3) getConfig() (aws.Config, error) {
creds := credentials.NewStaticCredentialsProvider(s.access, s.secret, "")
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
}
client := &http.Client{Transport: tr}
opts := []func(*config.LoadOptions) error{
config.WithRegion(s.region),
config.WithCredentialsProvider(creds),
config.WithHTTPClient(client),
}
if s.endpoint != "" {
opts = append(opts,
config.WithEndpointResolverWithOptions(s))
}
if s.debug {
opts = append(opts,
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
}
return config.LoadDefaultConfig(context.Background(), opts...)
}
func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
obj := iamFile
out, err := s.client.GetObject(context.Background(), &s3.GetObjectInput{
Bucket: &s.bucket,
Key: &obj,
})
if err != nil {
// if the error is object not exists,
// init empty accounts stuct and return that
var nsk *types.NoSuchKey
if errors.As(err, &nsk) {
return iAMConfig{}, nil
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
if apiErr.ErrorCode() == "NotFound" {
return iAMConfig{}, nil
}
}
// all other errors, return the error
return iAMConfig{}, fmt.Errorf("get %v: %w", obj, err)
}
defer out.Body.Close()
b, err := io.ReadAll(out.Body)
if err != nil {
return iAMConfig{}, fmt.Errorf("read %v: %w", obj, err)
}
conf, err := parseIAM(b)
if err != nil {
return iAMConfig{}, fmt.Errorf("parse iam data: %w", err)
}
return conf, nil
}
func (s *IAMServiceS3) storeAccts(conf iAMConfig) error {
b, err := json.Marshal(conf)
if err != nil {
return fmt.Errorf("failed to serialize iam: %w", err)
}
obj := iamFile
uploader := manager.NewUploader(s.client)
upinfo := &s3.PutObjectInput{
Body: bytes.NewReader(b),
Bucket: &s.bucket,
Key: &obj,
}
_, err = uploader.Upload(context.Background(), upinfo)
if err != nil {
return fmt.Errorf("store accounts in %v: %w", iamFile, err)
}
return nil
}

View File

@@ -15,7 +15,6 @@
package backend
import (
"bufio"
"context"
"fmt"
"io"
@@ -23,7 +22,6 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
"github.com/versity/versitygw/s3select"
)
//go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
@@ -63,7 +61,7 @@ type Backend interface {
// special case object operations
RestoreObject(context.Context, *s3.RestoreObjectInput) error
SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
// object tags operations
GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
@@ -164,19 +162,8 @@ func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) e
func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
return s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
return func(w *bufio.Writer) {
var getProgress s3select.GetProgress
progress := input.RequestProgress
if progress != nil && *progress.Enabled {
getProgress = func() (bytesScanned int64, bytesProcessed int64) {
return -1, -1
}
}
mh := s3select.NewMessageHandler(ctx, w, getProgress)
apiErr := s3err.GetAPIError(s3err.ErrNotImplemented)
mh.FinishWithError(apiErr.Code, apiErr.Description)
}
func (BackendUnsupported) SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
return s3response.SelectObjectContentResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
}
func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {

View File

@@ -146,10 +146,6 @@ func (p *Posix) ListBuckets(_ context.Context, owner string, isAdmin bool) (s3re
}
func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
_, err := os.Lstat(*input.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -162,10 +158,6 @@ func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.He
}
func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput) error {
if input.Bucket == nil {
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
bucket := *input.Bucket
owner := string(input.ObjectOwnership)
@@ -191,10 +183,6 @@ func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput) err
}
func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) error {
if input.Bucket == nil {
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
names, err := os.ReadDir(*input.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -224,13 +212,6 @@ func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) err
}
func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
if mpu.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if mpu.Key == nil {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
bucket := *mpu.Bucket
object := *mpu.Key
@@ -288,19 +269,6 @@ func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipart
}
func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if input.UploadId == nil {
return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
if input.MultipartUpload == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
@@ -326,7 +294,7 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
partsize := int64(0)
var totalsize int64
for i, p := range parts {
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))
fi, err := os.Lstat(partPath)
if err != nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
@@ -358,7 +326,7 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
defer f.cleanup()
for _, p := range parts {
pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)))
pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)))
if err != nil {
return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err)
}
@@ -544,16 +512,6 @@ func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
}
func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUploadInput) error {
if mpu.Bucket == nil {
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if mpu.Key == nil {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if mpu.UploadId == nil {
return s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
bucket := *mpu.Bucket
object := *mpu.Key
uploadID := *mpu.UploadId
@@ -584,12 +542,6 @@ func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUp
}
func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
var lmu s3response.ListMultipartUploadsResult
if mpu.Bucket == nil {
return lmu, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
bucket := *mpu.Bucket
var delimiter string
if mpu.Delimiter != nil {
@@ -600,6 +552,8 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
prefix = *mpu.Prefix
}
var lmu s3response.ListMultipartUploadsResult
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
return lmu, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -672,16 +626,12 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
}
}
maxUploads := 0
if mpu.MaxUploads != nil {
maxUploads = int(*mpu.MaxUploads)
}
if (uploadIDMarker != "" && !uploadIdMarkerFound) || (keyMarker != "" && keyMarkerInd == -1) {
return s3response.ListMultipartUploadsResult{
Bucket: bucket,
Delimiter: delimiter,
KeyMarker: keyMarker,
MaxUploads: maxUploads,
MaxUploads: int(mpu.MaxUploads),
Prefix: prefix,
UploadIDMarker: uploadIDMarker,
Uploads: []s3response.Upload{},
@@ -693,18 +643,18 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
})
for i := keyMarkerInd + 1; i < len(uploads); i++ {
if maxUploads == 0 {
if mpu.MaxUploads == 0 {
break
}
if keyMarker != "" && uploadIDMarker != "" && uploads[i].UploadID < uploadIDMarker {
continue
}
if i != len(uploads)-1 && len(resultUpds) == maxUploads {
if i != len(uploads)-1 && len(resultUpds) == int(mpu.MaxUploads) {
return s3response.ListMultipartUploadsResult{
Bucket: bucket,
Delimiter: delimiter,
KeyMarker: keyMarker,
MaxUploads: maxUploads,
MaxUploads: int(mpu.MaxUploads),
NextKeyMarker: resultUpds[i-1].Key,
NextUploadIDMarker: resultUpds[i-1].UploadID,
IsTruncated: true,
@@ -721,7 +671,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
Bucket: bucket,
Delimiter: delimiter,
KeyMarker: keyMarker,
MaxUploads: maxUploads,
MaxUploads: int(mpu.MaxUploads),
Prefix: prefix,
UploadIDMarker: uploadIDMarker,
Uploads: resultUpds,
@@ -729,29 +679,13 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
}
func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
var lpr s3response.ListPartsResult
if input.Bucket == nil {
return lpr, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return lpr, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if input.UploadId == nil {
return lpr, s3err.GetAPIError(s3err.ErrNoSuchUpload)
}
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
stringMarker := ""
if input.PartNumberMarker != nil {
stringMarker = *input.PartNumberMarker
}
maxParts := 0
if input.MaxParts != nil {
maxParts = int(*input.MaxParts)
}
stringMarker := *input.PartNumberMarker
maxParts := int(input.MaxParts)
var lpr s3response.ListPartsResult
var partNumberMarker int
if stringMarker != "" {
@@ -843,21 +777,11 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon
}
func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string, error) {
if input.Bucket == nil {
return "", s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return "", s3err.GetAPIError(s3err.ErrNoSuchKey)
}
bucket := *input.Bucket
object := *input.Key
uploadID := *input.UploadId
part := input.PartNumber
length := int64(0)
if input.ContentLength != nil {
length = *input.ContentLength
}
length := input.ContentLength
r := input.Body
_, err := os.Stat(bucket)
@@ -879,7 +803,7 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
return "", fmt.Errorf("stat uploadid: %w", err)
}
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part))
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", part))
f, err := openTmpFile(filepath.Join(bucket, objdir),
bucket, partPath, length)
@@ -909,13 +833,6 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
}
func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
if upi.Bucket == nil {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if upi.Key == nil {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
_, err := os.Stat(*upi.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -935,7 +852,7 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
return s3response.CopyObjectResult{}, fmt.Errorf("stat uploadid: %w", err)
}
partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", *upi.PartNumber))
partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", upi.PartNumber))
substrs := strings.SplitN(*upi.CopySource, "/", 2)
if len(substrs) != 2 {
@@ -1021,13 +938,6 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
}
func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) {
if po.Bucket == nil {
return "", s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if po.Key == nil {
return "", s3err.GetAPIError(s3err.ErrNoSuchKey)
}
tagsStr := getString(po.Tagging)
tags := make(map[string]string)
_, err := os.Stat(*po.Bucket)
@@ -1054,13 +964,9 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
name := filepath.Join(*po.Bucket, *po.Key)
contentLength := int64(0)
if po.ContentLength != nil {
contentLength = *po.ContentLength
}
if strings.HasSuffix(*po.Key, "/") {
// object is directory
if contentLength != 0 {
if po.ContentLength != 0 {
// posix directories can't contain data, send error
// if reuests has a data payload associated with a
// directory object
@@ -1089,7 +995,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
}
f, err := openTmpFile(filepath.Join(*po.Bucket, metaTmpDir),
*po.Bucket, *po.Key, contentLength)
*po.Bucket, *po.Key, po.ContentLength)
if err != nil {
return "", fmt.Errorf("open temp file: %w", err)
}
@@ -1133,13 +1039,6 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
}
func (p *Posix) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) error {
if input.Bucket == nil {
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
bucket := *input.Bucket
object := *input.Key
@@ -1232,16 +1131,6 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput)
}
func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if input.Range == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidRange)
}
bucket := *input.Bucket
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
@@ -1303,17 +1192,15 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
return nil, fmt.Errorf("get object tags: %w", err)
}
tagCount := int32(len(tags))
return &s3.GetObjectOutput{
AcceptRanges: &acceptRange,
ContentLength: &length,
ContentLength: length,
ContentEncoding: &contentEncoding,
ContentType: &contentType,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
TagCount: &tagCount,
TagCount: int32(len(tags)),
ContentRange: &contentRange,
}, nil
}
@@ -1348,28 +1235,20 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
return nil, fmt.Errorf("get object tags: %w", err)
}
tagCount := int32(len(tags))
return &s3.GetObjectOutput{
AcceptRanges: &acceptRange,
ContentLength: &length,
ContentLength: length,
ContentEncoding: &contentEncoding,
ContentType: &contentType,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
TagCount: &tagCount,
TagCount: int32(len(tags)),
ContentRange: &contentRange,
}, nil
}
func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
bucket := *input.Bucket
object := *input.Key
@@ -1399,10 +1278,8 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He
etag = ""
}
size := fi.Size()
return &s3.HeadObjectOutput{
ContentLength: &size,
ContentLength: fi.Size(),
ContentType: &contentType,
ContentEncoding: &contentEncoding,
ETag: &etag,
@@ -1412,18 +1289,6 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He
}
func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
if input.Key == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
}
if input.CopySource == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource)
}
if input.ExpectedBucketOwner == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
}
srcBucket, srcObject, ok := strings.Cut(*input.CopySource, "/")
if !ok {
return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource)
@@ -1496,16 +1361,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
}
}
contentLength := fInfo.Size()
etag, err := p.PutObject(ctx,
&s3.PutObjectInput{
Bucket: &dstBucket,
Key: &dstObject,
Body: f,
ContentLength: &contentLength,
Metadata: meta,
})
etag, err := p.PutObject(ctx, &s3.PutObjectInput{Bucket: &dstBucket, Key: &dstObject, Body: f, ContentLength: fInfo.Size(), Metadata: meta})
if err != nil {
return nil, err
}
@@ -1524,26 +1380,11 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
}
func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
bucket := *input.Bucket
prefix := ""
if input.Prefix != nil {
prefix = *input.Prefix
}
marker := ""
if input.Marker != nil {
marker = *input.Marker
}
delim := ""
if input.Delimiter != nil {
delim = *input.Delimiter
}
maxkeys := int32(0)
if input.MaxKeys != nil {
maxkeys = *input.MaxKeys
}
prefix := *input.Prefix
marker := *input.Marker
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
@@ -1564,9 +1405,9 @@ func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.
CommonPrefixes: results.CommonPrefixes,
Contents: results.Objects,
Delimiter: &delim,
IsTruncated: &results.Truncated,
IsTruncated: results.Truncated,
Marker: &marker,
MaxKeys: &maxkeys,
MaxKeys: maxkeys,
Name: &bucket,
NextMarker: &results.NextMarker,
Prefix: &prefix,
@@ -1625,38 +1466,21 @@ func fileToObj(bucket string) backend.GetObjFunc {
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
size := fi.Size()
return types.Object{
ETag: &etag,
Key: &path,
LastModified: backend.GetTimePtr(fi.ModTime()),
Size: &size,
Size: fi.Size(),
}, nil
}
}
func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
bucket := *input.Bucket
prefix := ""
if input.Prefix != nil {
prefix = *input.Prefix
}
marker := ""
if input.ContinuationToken != nil {
marker = *input.ContinuationToken
}
delim := ""
if input.Delimiter != nil {
delim = *input.Delimiter
}
maxkeys := int32(0)
if input.MaxKeys != nil {
maxkeys = *input.MaxKeys
}
prefix := *input.Prefix
marker := *input.ContinuationToken
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
@@ -1667,25 +1491,23 @@ func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (
}
fileSystem := os.DirFS(bucket)
results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
fileToObj(bucket), []string{metaTmpDir})
if err != nil {
return nil, fmt.Errorf("walk %v: %w", bucket, err)
}
count := int32(len(results.Objects))
return &s3.ListObjectsV2Output{
CommonPrefixes: results.CommonPrefixes,
Contents: results.Objects,
Delimiter: &delim,
IsTruncated: &results.Truncated,
IsTruncated: results.Truncated,
ContinuationToken: &marker,
MaxKeys: &maxkeys,
MaxKeys: int32(maxkeys),
Name: &bucket,
NextContinuationToken: &results.NextMarker,
Prefix: &prefix,
KeyCount: &count,
KeyCount: int32(len(results.Objects)),
}, nil
}
@@ -1706,9 +1528,6 @@ func (p *Posix) PutBucketAcl(_ context.Context, bucket string, data []byte) erro
}
func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
_, err := os.Stat(*input.Bucket)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)

View File

@@ -17,6 +17,7 @@ package s3proxy
import (
"context"
"crypto/tls"
"fmt"
"net/http"
"github.com/aws/aws-sdk-go-v2/aws"
@@ -25,10 +26,16 @@ import (
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/smithy-go/middleware"
"github.com/versity/versitygw/auth"
)
func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
cfg, err := s.getConfig(ctx, s.access, s.secret)
func (s *S3be) getClientFromCtx(ctx context.Context) (*s3.Client, error) {
acct, ok := ctx.Value("account").(auth.Account)
if !ok {
return nil, fmt.Errorf("invalid account in context")
}
cfg, err := s.getConfig(ctx, acct.Access, acct.Secret)
if err != nil {
return nil, err
}
@@ -36,7 +43,7 @@ func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
return s3.NewFromConfig(cfg), nil
}
func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
func (s *S3be) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
creds := credentials.NewStaticCredentialsProvider(access, secret, "")
tr := &http.Transport{
@@ -69,7 +76,7 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
}
// ResolveEndpoint is used for on prem or non-aws endpoints
func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
func (s *S3be) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
PartitionID: "aws",
URL: s.endpoint,

View File

@@ -16,35 +16,21 @@ package s3proxy
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/backend"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3response"
)
type S3Proxy struct {
type S3be struct {
backend.BackendUnsupported
client *s3.Client
access string
secret string
endpoint string
awsRegion string
disableChecksum bool
@@ -52,27 +38,23 @@ type S3Proxy struct {
debug bool
}
func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
s := &S3Proxy{
access: access,
secret: secret,
func New(endpoint, region string, disableChecksum, sslSkipVerify, debug bool) *S3be {
return &S3be{
endpoint: endpoint,
awsRegion: region,
disableChecksum: disableChecksum,
sslSkipVerify: sslSkipVerify,
debug: debug,
}
client, err := s.getClientWithCtx(context.Background())
if err != nil {
return nil, err
}
s.client = client
return s, nil
}
func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{})
err = handleError(err)
func (s *S3be) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return s3response.ListAllMyBucketsResult{}, err
}
output, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
if err != nil {
return s3response.ListAllMyBucketsResult{}, err
}
@@ -87,7 +69,8 @@ func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (
return s3response.ListAllMyBucketsResult{
Owner: s3response.CanonicalUser{
ID: *output.Owner.ID,
ID: *output.Owner.ID,
DisplayName: *output.Owner.DisplayName,
},
Buckets: s3response.ListAllMyBucketsList{
Bucket: buckets,
@@ -95,44 +78,74 @@ func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (
}, nil
}
func (s *S3Proxy) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
out, err := s.client.HeadBucket(ctx, input)
err = handleError(err)
func (s *S3be) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
return out, err
return client.HeadBucket(ctx, input)
}
func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) error {
_, err := s.client.CreateBucket(ctx, input)
return handleError(err)
}
func (s *S3be) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return err
}
func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
_, err := s.client.DeleteBucket(ctx, input)
return handleError(err)
}
func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
out, err := s.client.CreateMultipartUpload(ctx, input)
err = handleError(err)
return out, err
}
func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
out, err := s.client.CompleteMultipartUpload(ctx, input)
err = handleError(err)
return out, err
}
func (s *S3Proxy) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
_, err := s.client.AbortMultipartUpload(ctx, input)
err = handleError(err)
_, err = client.CreateBucket(ctx, input)
return err
}
func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
output, err := s.client.ListMultipartUploads(ctx, input)
err = handleError(err)
func (s *S3be) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return err
}
_, err = client.DeleteBucket(ctx, input)
return err
}
func (s *S3be) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
return client.CreateMultipartUpload(ctx, input)
}
func (s *S3be) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
return client.CompleteMultipartUpload(ctx, input)
}
func (s *S3be) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return err
}
_, err = client.AbortMultipartUpload(ctx, input)
return err
}
const (
iso8601Format = "20060102T150405Z"
)
func (s *S3be) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return s3response.ListMultipartUploadsResult{}, err
}
output, err := client.ListMultipartUploads(ctx, input)
if err != nil {
return s3response.ListMultipartUploadsResult{}, err
}
@@ -151,7 +164,7 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
DisplayName: *u.Owner.DisplayName,
},
StorageClass: string(u.StorageClass),
Initiated: u.Initiated.Format(backend.RFC3339TimeFormat),
Initiated: u.Initiated.Format(iso8601Format),
})
}
@@ -171,16 +184,20 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
Delimiter: *output.Delimiter,
Prefix: *output.Prefix,
EncodingType: string(output.EncodingType),
MaxUploads: int(*output.MaxUploads),
IsTruncated: *output.IsTruncated,
MaxUploads: int(output.MaxUploads),
IsTruncated: output.IsTruncated,
Uploads: uploads,
CommonPrefixes: cps,
}, nil
}
func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
output, err := s.client.ListParts(ctx, input)
err = handleError(err)
func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return s3response.ListPartsResult{}, err
}
output, err := client.ListParts(ctx, input)
if err != nil {
return s3response.ListPartsResult{}, err
}
@@ -188,10 +205,10 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
var parts []s3response.Part
for _, p := range output.Parts {
parts = append(parts, s3response.Part{
PartNumber: int(*p.PartNumber),
LastModified: p.LastModified.Format(backend.RFC3339TimeFormat),
PartNumber: int(p.PartNumber),
LastModified: p.LastModified.Format(iso8601Format),
ETag: *p.ETag,
Size: *p.Size,
Size: p.Size,
})
}
pnm, err := strconv.Atoi(*output.PartNumberMarker)
@@ -221,19 +238,19 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
StorageClass: string(output.StorageClass),
PartNumberMarker: pnm,
NextPartNumberMarker: npmn,
MaxParts: int(*output.MaxParts),
IsTruncated: *output.IsTruncated,
MaxParts: int(output.MaxParts),
IsTruncated: output.IsTruncated,
Parts: parts,
}, nil
}
func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
// streaming backend is not seekable,
// use unsigned payload for streaming ops
output, err := s.client.UploadPart(ctx, input, s3.WithAPIOptions(
v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
))
err = handleError(err)
func (s *S3be) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return "", err
}
output, err := client.UploadPart(ctx, input)
if err != nil {
return "", err
}
@@ -241,9 +258,13 @@ func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (et
return *output.ETag, nil
}
func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
output, err := s.client.UploadPartCopy(ctx, input)
err = handleError(err)
func (s *S3be) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return s3response.CopyObjectResult{}, err
}
output, err := client.UploadPartCopy(ctx, input)
if err != nil {
return s3response.CopyObjectResult{}, err
}
@@ -254,13 +275,13 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn
}, nil
}
func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
// streaming backend is not seekable,
// use unsigned payload for streaming ops
output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions(
v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
))
err = handleError(err)
func (s *S3be) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return "", err
}
output, err := client.PutObject(ctx, input)
if err != nil {
return "", err
}
@@ -268,16 +289,22 @@ func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (stri
return *output.ETag, nil
}
func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
out, err := s.client.HeadObject(ctx, input)
err = handleError(err)
func (s *S3be) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
return out, err
return client.HeadObject(ctx, input)
}
func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
output, err := s.client.GetObject(ctx, input)
err = handleError(err)
func (s *S3be) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
output, err := client.GetObject(ctx, input)
if err != nil {
return nil, err
}
@@ -291,46 +318,59 @@ func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.
return output, nil
}
func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
out, err := s.client.GetObjectAttributes(ctx, input)
err = handleError(err)
return out, err
}
func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
out, err := s.client.CopyObject(ctx, input)
err = handleError(err)
return out, err
}
func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
out, err := s.client.ListObjects(ctx, input)
err = handleError(err)
return out, err
}
func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
out, err := s.client.ListObjectsV2(ctx, input)
err = handleError(err)
return out, err
}
func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
_, err := s.client.DeleteObject(ctx, input)
return handleError(err)
}
func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
if len(input.Delete.Objects) == 0 {
input.Delete.Objects = []types.ObjectIdentifier{}
func (s *S3be) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
output, err := s.client.DeleteObjects(ctx, input)
err = handleError(err)
return client.GetObjectAttributes(ctx, input)
}
func (s *S3be) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
return client.CopyObject(ctx, input)
}
func (s *S3be) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
return client.ListObjects(ctx, input)
}
func (s *S3be) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
return client.ListObjectsV2(ctx, input)
}
func (s *S3be) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return err
}
_, err = client.DeleteObject(ctx, input)
return err
}
func (s *S3be) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return s3response.DeleteObjectsResult{}, err
}
output, err := client.DeleteObjects(ctx, input)
if err != nil {
return s3response.DeleteObjectsResult{}, err
}
@@ -341,9 +381,13 @@ func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInpu
}, nil
}
func (s *S3Proxy) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
output, err := s.client.GetBucketAcl(ctx, input)
err = handleError(err)
func (s *S3be) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
output, err := client.GetBucketAcl(ctx, input)
if err != nil {
return nil, err
}
@@ -361,7 +405,12 @@ func (s *S3Proxy) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput)
return json.Marshal(acl)
}
func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
func (s S3be) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return err
}
acl, err := auth.ParseACL(data)
if err != nil {
return err
@@ -378,21 +427,25 @@ func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte)
}
for _, el := range acl.Grantees {
acc := el.Access
input.AccessControlPolicy.Grants = append(input.AccessControlPolicy.Grants, types.Grant{
Permission: el.Permission,
Grantee: &types.Grantee{
ID: &acc,
ID: &el.Access,
Type: types.TypeCanonicalUser,
},
})
}
_, err = s.client.PutBucketAcl(ctx, input)
return handleError(err)
_, err = client.PutBucketAcl(ctx, input)
return err
}
func (s *S3Proxy) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
func (s *S3be) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return err
}
tagging := &types.Tagging{
TagSet: []types.Tag{},
}
@@ -403,20 +456,24 @@ func (s *S3Proxy) PutObjectTagging(ctx context.Context, bucket, object string, t
})
}
_, err := s.client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
_, err = client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
Bucket: &bucket,
Key: &object,
Tagging: tagging,
})
return handleError(err)
return err
}
func (s *S3Proxy) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
output, err := s.client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
func (s *S3be) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return nil, err
}
output, err := client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
Bucket: &bucket,
Key: &object,
})
err = handleError(err)
if err != nil {
return nil, err
}
@@ -429,106 +486,15 @@ func (s *S3Proxy) GetObjectTagging(ctx context.Context, bucket, object string) (
return tags, nil
}
func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
_, err := s.client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
func (s *S3be) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
client, err := s.getClientFromCtx(ctx)
if err != nil {
return err
}
_, err = client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
Bucket: &bucket,
Key: &object,
})
return handleError(err)
}
func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, newOwner), nil)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
if signErr != nil {
return fmt.Errorf("failed to sign the request: %w", err)
}
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request: %w", err)
}
if resp.StatusCode > 300 {
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
defer resp.Body.Close()
return fmt.Errorf(string(body))
}
return nil
}
func (s *S3Proxy) ListBucketsAndOwners(ctx context.Context) ([]s3response.Bucket, error) {
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", s.endpoint), nil)
if err != nil {
return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
}
signer := v4.NewSigner()
hashedPayload := sha256.Sum256([]byte{})
hexPayload := hex.EncodeToString(hashedPayload[:])
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
if signErr != nil {
return []s3response.Bucket{}, fmt.Errorf("failed to sign the request: %w", err)
}
client := http.Client{}
resp, err := client.Do(req)
if err != nil {
return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return []s3response.Bucket{}, err
}
defer resp.Body.Close()
var buckets []s3response.Bucket
if err := json.Unmarshal(body, &buckets); err != nil {
return []s3response.Bucket{}, err
}
return buckets, nil
}
func handleError(err error) error {
if err == nil {
return nil
}
var ae smithy.APIError
if errors.As(err, &ae) {
apiErr := s3err.APIError{
Code: ae.ErrorCode(),
Description: ae.ErrorMessage(),
}
var re *awshttp.ResponseError
if errors.As(err, &re) {
apiErr.HTTPStatusCode = re.Response.StatusCode
}
return apiErr
}
return err
}

View File

@@ -415,10 +415,8 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.
}
}
contentLength := fi.Size()
return &s3.HeadObjectOutput{
ContentLength: &contentLength,
ContentLength: fi.Size(),
ContentType: &contentType,
ContentEncoding: &contentEncoding,
ETag: &etag,
@@ -509,17 +507,15 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
return nil, fmt.Errorf("get object tags: %w", err)
}
tagCount := int32(len(tags))
return &s3.GetObjectOutput{
AcceptRanges: &acceptRange,
ContentLength: &length,
ContentLength: length,
ContentEncoding: &contentEncoding,
ContentType: &contentType,
ETag: &etag,
LastModified: backend.GetTimePtr(fi.ModTime()),
Metadata: userMetaData,
TagCount: &tagCount,
TagCount: int32(len(tags)),
StorageClass: types.StorageClassStandard,
}, nil
}
@@ -546,26 +542,11 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error)
}
func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
bucket := *input.Bucket
prefix := ""
if input.Prefix != nil {
prefix = *input.Prefix
}
marker := ""
if input.Marker != nil {
marker = *input.Marker
}
delim := ""
if input.Delimiter != nil {
delim = *input.Delimiter
}
maxkeys := int32(0)
if input.MaxKeys != nil {
maxkeys = *input.MaxKeys
}
prefix := *input.Prefix
marker := *input.Marker
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
@@ -586,9 +567,9 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s
CommonPrefixes: results.CommonPrefixes,
Contents: results.Objects,
Delimiter: &delim,
IsTruncated: &results.Truncated,
IsTruncated: results.Truncated,
Marker: &marker,
MaxKeys: &maxkeys,
MaxKeys: maxkeys,
Name: &bucket,
NextMarker: &results.NextMarker,
Prefix: &prefix,
@@ -596,26 +577,11 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s
}
func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
if input.Bucket == nil {
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
}
bucket := *input.Bucket
prefix := ""
if input.Prefix != nil {
prefix = *input.Prefix
}
marker := ""
if input.ContinuationToken != nil {
marker = *input.ContinuationToken
}
delim := ""
if input.Delimiter != nil {
delim = *input.Delimiter
}
maxkeys := int32(0)
if input.MaxKeys != nil {
maxkeys = *input.MaxKeys
}
prefix := *input.Prefix
marker := *input.ContinuationToken
delim := *input.Delimiter
maxkeys := input.MaxKeys
_, err := os.Stat(bucket)
if errors.Is(err, fs.ErrNotExist) {
@@ -636,9 +602,9 @@ func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input)
CommonPrefixes: results.CommonPrefixes,
Contents: results.Objects,
Delimiter: &delim,
IsTruncated: &results.Truncated,
IsTruncated: results.Truncated,
ContinuationToken: &marker,
MaxKeys: &maxkeys,
MaxKeys: int32(maxkeys),
Name: &bucket,
NextContinuationToken: &results.NextMarker,
Prefix: &prefix,
@@ -711,13 +677,11 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
}
}
size := fi.Size()
return types.Object{
ETag: &etag,
Key: &path,
LastModified: backend.GetTimePtr(fi.ModTime()),
Size: &size,
Size: fi.Size(),
StorageClass: sc,
}, nil
}

View File

@@ -55,13 +55,11 @@ func getObj(path string, d fs.DirEntry) (types.Object, error) {
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
}
size := fi.Size()
return types.Object{
ETag: &etag,
Key: &path,
LastModified: backend.GetTimePtr(fi.ModTime()),
Size: &size,
Size: fi.Size(),
}, nil
}

View File

@@ -1,104 +0,0 @@
package main
import (
"context"
"log"
"os"
"path/filepath"
"sync"
"testing"
"github.com/versity/versitygw/backend/posix"
"github.com/versity/versitygw/integration"
)
const (
tdir = "tempdir"
)
var (
wg sync.WaitGroup
)
func initEnv(dir string) {
// both
debug = true
region = "us-east-1"
// server
rootUserAccess = "user"
rootUserSecret = "pass"
iamDir = dir
port = "127.0.0.1:7070"
// client
awsID = "user"
awsSecret = "pass"
endpoint = "http://127.0.0.1:7070"
}
func initPosix(ctx context.Context) {
path, err := os.Getwd()
if err != nil {
log.Fatalf("get current directory: %v", err)
}
tempdir := filepath.Join(path, tdir)
initEnv(tempdir)
err = os.RemoveAll(tempdir)
if err != nil {
log.Fatalf("remove temp directory: %v", err)
}
err = os.Mkdir(tempdir, 0755)
if err != nil {
log.Fatalf("make temp directory: %v", err)
}
be, err := posix.New(tempdir)
if err != nil {
log.Fatalf("init posix: %v", err)
}
wg.Add(1)
go func() {
err = runGateway(ctx, be)
if err != nil && err != context.Canceled {
log.Fatalf("run gateway: %v", err)
}
err := os.RemoveAll(tempdir)
if err != nil {
log.Fatalf("remove temp directory: %v", err)
}
wg.Done()
}()
}
func TestIntegration(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
initPosix(ctx)
opts := []integration.Option{
integration.WithAccess(awsID),
integration.WithSecret(awsSecret),
integration.WithRegion(region),
integration.WithEndpoint(endpoint),
}
if debug {
opts = append(opts, integration.WithDebug())
}
s := integration.NewS3Conf(opts...)
// replace below with desired test
err := integration.HeadBucket_non_existing_bucket(s)
if err != nil {
t.Error(err)
}
cancel()
wg.Wait()
}

View File

@@ -20,6 +20,7 @@ import (
"fmt"
"log"
"os"
"strconv"
"github.com/gofiber/fiber/v2"
"github.com/urfave/cli/v2"
@@ -43,15 +44,10 @@ var (
logWebhookURL string
accessLog string
debug bool
quiet bool
iamDir string
ldapURL, ldapBindDN, ldapPassword string
ldapQueryBase, ldapObjClasses string
ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
s3IamAccess, s3IamSecret string
s3IamRegion, s3IamBucket string
s3IamEndpoint string
s3IamSslNoVerify, s3IamDebug bool
iamCacheDisable bool
iamCacheTTL int
iamCachePrune int
@@ -177,12 +173,6 @@ func initFlags() []cli.Flag {
Usage: "enable debug output",
Destination: &debug,
},
&cli.BoolFlag{
Name: "quiet",
Usage: "silence stdout request logging output",
Destination: &quiet,
Aliases: []string{"q"},
},
&cli.StringFlag{
Name: "access-log",
Usage: "enable server access logging to specified file",
@@ -270,42 +260,6 @@ func initFlags() []cli.Flag {
Usage: "ldap server user role attribute name",
Destination: &ldapRoleAtr,
},
&cli.StringFlag{
Name: "s3-iam-access",
Usage: "s3 IAM access key",
Destination: &s3IamAccess,
},
&cli.StringFlag{
Name: "s3-iam-secret",
Usage: "s3 IAM secret key",
Destination: &s3IamSecret,
},
&cli.StringFlag{
Name: "s3-iam-region",
Usage: "s3 IAM region",
Destination: &s3IamRegion,
Value: "us-east-1",
},
&cli.StringFlag{
Name: "s3-iam-bucket",
Usage: "s3 IAM bucket",
Destination: &s3IamBucket,
},
&cli.StringFlag{
Name: "s3-iam-endpoint",
Usage: "s3 IAM endpoint",
Destination: &s3IamEndpoint,
},
&cli.BoolFlag{
Name: "s3-iam-noverify",
Usage: "s3 IAM disable ssl verification",
Destination: &s3IamSslNoVerify,
},
&cli.BoolFlag{
Name: "s3-iam-debug",
Usage: "s3 IAM debug output",
Destination: &s3IamDebug,
},
&cli.BoolFlag{
Name: "iam-cache-disable",
Usage: "disable local iam cache",
@@ -326,12 +280,18 @@ func initFlags() []cli.Flag {
}
}
func runGateway(ctx context.Context, be backend.Backend) error {
func runGateway(ctx *cli.Context, be backend.Backend) error {
// int32 max for 32 bit arch
blimit := int64(2*1024*1024*1024 - 1)
if strconv.IntSize > 32 {
// 5GB max for 64 bit arch
blimit = int64(5 * 1024 * 1024 * 1024)
}
app := fiber.New(fiber.Config{
AppName: "versitygw",
ServerHeader: "VERSITYGW",
StreamRequestBody: true,
DisableKeepalive: true,
AppName: "versitygw",
ServerHeader: "VERSITYGW",
BodyLimit: int(blimit),
})
var opts []s3api.Option
@@ -356,9 +316,6 @@ func runGateway(ctx context.Context, be backend.Backend) error {
if admPort == "" {
opts = append(opts, s3api.WithAdminServer())
}
if quiet {
opts = append(opts, s3api.WithQuiet())
}
admApp := fiber.New(fiber.Config{
AppName: "versitygw",
@@ -383,25 +340,18 @@ func runGateway(ctx context.Context, be backend.Backend) error {
}
iam, err := auth.New(&auth.Opts{
Dir: iamDir,
LDAPServerURL: ldapURL,
LDAPBindDN: ldapBindDN,
LDAPPassword: ldapPassword,
LDAPQueryBase: ldapQueryBase,
LDAPObjClasses: ldapObjClasses,
LDAPAccessAtr: ldapAccessAtr,
LDAPSecretAtr: ldapSecAtr,
LDAPRoleAtr: ldapRoleAtr,
S3Access: s3IamAccess,
S3Secret: s3IamSecret,
S3Region: s3IamRegion,
S3Bucket: s3IamBucket,
S3Endpoint: s3IamEndpoint,
S3DisableSSlVerfiy: s3IamSslNoVerify,
S3Debug: s3IamDebug,
CacheDisable: iamCacheDisable,
CacheTTL: iamCacheTTL,
CachePrune: iamCachePrune,
Dir: iamDir,
LDAPServerURL: ldapURL,
LDAPBindDN: ldapBindDN,
LDAPPassword: ldapPassword,
LDAPQueryBase: ldapQueryBase,
LDAPObjClasses: ldapObjClasses,
LDAPAccessAtr: ldapAccessAtr,
LDAPSecretAtr: ldapSecAtr,
LDAPRoleAtr: ldapRoleAtr,
CacheDisable: iamCacheDisable,
CacheTTL: iamCacheTTL,
CachePrune: iamCachePrune,
})
if err != nil {
return fmt.Errorf("setup iam: %w", err)

View File

@@ -49,5 +49,5 @@ func runPosix(ctx *cli.Context) error {
return fmt.Errorf("init posix: %v", err)
}
return runGateway(ctx.Context, be)
return runGateway(ctx, be)
}

View File

@@ -15,15 +15,11 @@
package main
import (
"fmt"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/backend/s3proxy"
)
var (
s3proxyAccess string
s3proxySecret string
s3proxyEndpoint string
s3proxyRegion string
s3proxyDisableChecksum bool
@@ -39,22 +35,6 @@ func s3Command() *cli.Command {
to an s3 storage backend service.`,
Action: runS3,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "access",
Usage: "s3 proxy server access key id",
Value: "",
Required: true,
Destination: &s3proxyAccess,
Aliases: []string{"a"},
},
&cli.StringFlag{
Name: "secret",
Usage: "s3 proxy server secret access key",
Value: "",
Required: true,
Destination: &s3proxySecret,
Aliases: []string{"s"},
},
&cli.StringFlag{
Name: "endpoint",
Usage: "s3 service endpoint, default AWS if not specified",
@@ -90,10 +70,7 @@ to an s3 storage backend service.`,
}
func runS3(ctx *cli.Context) error {
be, err := s3proxy.New(s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
be := s3proxy.New(s3proxyEndpoint, s3proxyRegion,
s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
if err != nil {
return fmt.Errorf("init s3 backend: %w", err)
}
return runGateway(ctx.Context, be)
return runGateway(ctx, be)
}

View File

@@ -69,5 +69,5 @@ func runScoutfs(ctx *cli.Context) error {
return fmt.Errorf("init scoutfs: %v", err)
}
return runGateway(ctx.Context, be)
return runGateway(ctx, be)
}

View File

@@ -2,6 +2,9 @@ package main
import (
"fmt"
"math"
"os"
"text/tabwriter"
"github.com/urfave/cli/v2"
"github.com/versity/versitygw/integration"
@@ -13,6 +16,7 @@ var (
endpoint string
prefix string
dstBucket string
proxyURL string
partSize int64
objSize int64
concurrency int
@@ -67,7 +71,7 @@ func initTestFlags() []cli.Flag {
}
func initTestCommands() []*cli.Command {
return append([]*cli.Command{
return []*cli.Command{
{
Name: "full-flow",
Usage: "Tests the full flow of gateway.",
@@ -118,6 +122,7 @@ func initTestCommands() []*cli.Command {
Name: "bucket",
Usage: "Destination bucket name to read/write data",
Destination: &dstBucket,
Required: true,
},
&cli.Int64Flag{
Name: "partSize",
@@ -143,6 +148,11 @@ func initTestCommands() []*cli.Command {
Value: false,
Destination: &checksumDisable,
},
&cli.StringFlag{
Name: "proxy-url",
Usage: "S3 proxy server url to compare",
Destination: &proxyURL,
},
},
Action: func(ctx *cli.Context) error {
if upload && download {
@@ -152,10 +162,6 @@ func initTestCommands() []*cli.Command {
return fmt.Errorf("must specify one of upload or download")
}
if dstBucket == "" {
return fmt.Errorf("must specify bucket")
}
opts := []integration.Option{
integration.WithAccess(awsID),
integration.WithSecret(awsSecret),
@@ -177,9 +183,47 @@ func initTestCommands() []*cli.Command {
s3conf := integration.NewS3Conf(opts...)
if upload {
return integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
if proxyURL == "" {
integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
return nil
} else {
size, elapsed, err := integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
opts = append(opts, integration.WithEndpoint(proxyURL))
proxyS3Conf := integration.NewS3Conf(opts...)
proxySize, proxyElapsed, proxyErr := integration.TestUpload(proxyS3Conf, files, objSize, dstBucket, prefix)
if err != nil || proxyErr != nil {
return nil
}
printProxyResultsTable([][4]string{
{" # ", "Total Size", "Time Taken", "Speed(MB/S)"},
{"---------", "----------", "----------", "-----------"},
{"S3 Server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
{"S3 Proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
})
return nil
}
} else {
return integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
if proxyURL == "" {
integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
return nil
} else {
size, elapsed, err := integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
opts = append(opts, integration.WithEndpoint(proxyURL))
proxyS3Conf := integration.NewS3Conf(opts...)
proxySize, proxyElapsed, proxyErr := integration.TestDownload(proxyS3Conf, files, objSize, dstBucket, prefix)
if err != nil || proxyErr != nil {
return nil
}
printProxyResultsTable([][4]string{
{" # ", "Total Size", "Time Taken", "Speed(MB/S)"},
{"---------", "----------", "----------", "-----------"},
{"S3 server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
{"S3 proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
})
return nil
}
}
},
},
@@ -211,12 +255,13 @@ func initTestCommands() []*cli.Command {
Value: false,
Destination: &checksumDisable,
},
&cli.StringFlag{
Name: "proxy-url",
Usage: "S3 proxy server url to compare",
Destination: &proxyURL,
},
},
Action: func(ctx *cli.Context) error {
if dstBucket == "" {
return fmt.Errorf("must specify the destination bucket")
}
opts := []integration.Option{
integration.WithAccess(awsID),
integration.WithSecret(awsSecret),
@@ -233,10 +278,30 @@ func initTestCommands() []*cli.Command {
s3conf := integration.NewS3Conf(opts...)
return integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
if proxyURL == "" {
_, _, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
return err
} else {
elapsed, rps, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
opts = append(opts, integration.WithEndpoint(proxyURL))
s3proxy := integration.NewS3Conf(opts...)
proxyElapsed, proxyRPS, proxyErr := integration.TestReqPerSec(s3proxy, totalReqs, dstBucket)
if err != nil || proxyErr != nil {
return nil
}
printProxyResultsTable([][4]string{
{" # ", "Total Requests", "Time Taken", "Requests Per Second(Req/Sec)"},
{"---------", "--------------", "----------", "----------------------------"},
{"S3 Server", fmt.Sprint(totalReqs), fmt.Sprintf("%v", elapsed), fmt.Sprint(rps)},
{"S3 Proxy", fmt.Sprint(totalReqs), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(proxyRPS)},
})
return nil
}
},
},
}, extractIntTests()...)
}
}
type testFunc func(*integration.S3Conf)
@@ -265,30 +330,12 @@ func getAction(tf testFunc) func(*cli.Context) error {
}
}
func extractIntTests() (commands []*cli.Command) {
tests := integration.GetIntTests()
for key, val := range tests {
testKey := key
testFunc := val
commands = append(commands, &cli.Command{
Name: testKey,
Usage: fmt.Sprintf("Runs %v integration test", testKey),
Action: func(ctx *cli.Context) error {
opts := []integration.Option{
integration.WithAccess(awsID),
integration.WithSecret(awsSecret),
integration.WithRegion(region),
integration.WithEndpoint(endpoint),
}
if debug {
opts = append(opts, integration.WithDebug())
}
s := integration.NewS3Conf(opts...)
err := testFunc(s)
return err
},
})
func printProxyResultsTable(stats [][4]string) {
w := new(tabwriter.Writer)
w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
for _, elem := range stats {
fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", elem[0], elem[1], elem[2], elem[3])
}
return
fmt.Fprintln(w)
w.Flush()
}

View File

@@ -1,23 +0,0 @@
version: "3"
services:
posix:
build:
context: .
dockerfile: ./Dockerfile.dev
args:
- IAM_DIR=${IAM_DIR}
- SETUP_DIR=${SETUP_DIR}
volumes:
- ./:/app
ports:
- "${POSIX_PORT}:${POSIX_PORT}"
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$POSIX_PORT -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR posix $SETUP_DIR"]
proxy:
build:
context: .
dockerfile: ./Dockerfile.dev
volumes:
- ./:/app
ports:
- "${PROXY_PORT}:${PROXY_PORT}"
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$PROXY_PORT s3 -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --endpoint http://posix:$POSIX_PORT"]

54
go.mod
View File

@@ -3,54 +3,54 @@ module github.com/versity/versitygw
go 1.20
require (
github.com/aws/aws-sdk-go-v2 v1.24.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7
github.com/aws/smithy-go v1.19.0
github.com/aws/aws-sdk-go-v2 v1.22.2
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1
github.com/aws/smithy-go v1.16.0
github.com/go-ldap/ldap/v3 v3.4.6
github.com/gofiber/fiber/v2 v2.51.0
github.com/google/uuid v1.5.0
github.com/gofiber/fiber/v2 v2.50.0
github.com/google/uuid v1.4.0
github.com/nats-io/nats.go v1.31.0
github.com/pkg/xattr v0.4.9
github.com/segmentio/kafka-go v0.4.47
github.com/urfave/cli/v2 v2.26.0
github.com/valyala/fasthttp v1.51.0
github.com/segmentio/kafka-go v0.4.44
github.com/urfave/cli/v2 v2.25.7
github.com/valyala/fasthttp v1.50.0
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
golang.org/x/sys v0.15.0
golang.org/x/sys v0.14.0
)
require (
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/nats-io/nkeys v0.4.6 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/stretchr/testify v1.8.1 // indirect
golang.org/x/crypto v0.17.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
)
require (
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
github.com/aws/aws-sdk-go-v2/config v1.26.2
github.com/aws/aws-sdk-go-v2/credentials v1.16.13
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.9
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 // indirect
github.com/aws/aws-sdk-go-v2/config v1.24.0
github.com/aws/aws-sdk-go-v2/credentials v1.15.2
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/klauspost/compress v1.17.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect

109
go.sum
View File

@@ -4,44 +4,44 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3Uu
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk=
github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo=
github.com/aws/aws-sdk-go-v2/config v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc=
github.com/aws/aws-sdk-go-v2/config v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8=
github.com/aws/aws-sdk-go-v2/credentials v1.16.13 h1:WLABQ4Cp4vXtXfOWOS3MEZKr6AAYUpMczLhgKtAjQ/8=
github.com/aws/aws-sdk-go-v2/credentials v1.16.13/go.mod h1:Qg6x82FXwW0sJHzYruxGiuApNo31UEtJvXVSZAXeWiw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.9 h1:5zA8qVCXMPGt6YneFnll5B157SfdK2SewU85PH9/yM0=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.9/go.mod h1:t4gy210hPxkbtYM8xOzrWdxVq1PyekR76OOKXy3s0Vs=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM=
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9/go.mod h1:YD0aYBWCrPENpHolhKw2XDlTIWae2GKXT1T4o6N6hiM=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7 h1:o0ASbVwUAIrfp/WcCac+6jioZt4Hd8k/1X8u7GJ/QeM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7/go.mod h1:vADO6Jn+Rq4nDtfwNjhgR84qkZwiC6FqCaXdw/kYwjA=
github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM=
github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38=
github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 h1:HJeiuZ2fldpd0WqngyMR6KW7ofkXNLyOaHwEIGm39Cs=
github.com/aws/aws-sdk-go-v2/service/sts v1.26.6/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU=
github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
github.com/aws/aws-sdk-go-v2 v1.22.2 h1:lV0U8fnhAnPz8YcdmZVV60+tr6CakHzqA6P8T46ExJI=
github.com/aws/aws-sdk-go-v2 v1.22.2/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 h1:hHgLiIrTRtddC0AKcJr5s7i/hLgcpTt+q/FKxf1Zayk=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0/go.mod h1:w4I/v3NOWgD+qvs1NPEwhd++1h3XPHFaVxasfY6HlYQ=
github.com/aws/aws-sdk-go-v2/config v1.24.0 h1:4LEk29JO3w+y9dEo/5Tq5QTP7uIEw+KQrKiHOs4xlu4=
github.com/aws/aws-sdk-go-v2/config v1.24.0/go.mod h1:11nNDAuK86kOUHeuEQo8f3CkcV5xuUxvPwFjTZE/PnQ=
github.com/aws/aws-sdk-go-v2/credentials v1.15.2 h1:rKH7khRMxPdD0u3dHecd0Q7NOVw3EUe7AqdkUOkiOGI=
github.com/aws/aws-sdk-go-v2/credentials v1.15.2/go.mod h1:tXM8wmaeAhfC7nZoCxb0FzM/aRaB1m1WQ7x0qlBLq80=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 h1:G5KawTAkyHH6WyKQCdHiW4h3PmAXNJpOgwKg3H7sDRE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3/go.mod h1:hugKmSFnZB+HgNI1sYGT14BUPZkO6alC/e0AWu+0IAQ=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6 h1:IpQbitxCZeC64C1ALz9QZu6AHHWundnU2evQ9xbp5k8=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6/go.mod h1:27jIVQK+al9s0yTo3pkMdahRinbscqSC6zNGfNWXPZc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 h1:AaQsr5vvGR7rmeSWBtTCcw16tT9r51mWijuCQhzLnq8=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2/go.mod h1:o1IiRn7CWocIFTXJjGKJDOwxv1ibL53NpcvcqGWyRBA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 h1:UZx8SXZ0YtzRiALzYAWcjb9Y9hZUR7MBKaBQ5ouOjPs=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2/go.mod h1:ipuRpcSaklmxR6C39G187TpBAO132gUfleTGccUPs8c=
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 h1:usgqiJtamuGIBj+OvYmMq89+Z1hIKkMJToz1WpoeNUY=
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 h1:pyVrNAf7Hwz0u39dLKN5t+n0+K/3rMYKuiOoIum3AsU=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2/go.mod h1:mydrfOb9uiOYCxuCPR8YHQNQyGQwUQ7gPMZGBKbH8NY=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 h1:CJxo7ZBbaIzmXfV3hjcx36n9V87gJsIUPJflwqEHl3Q=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0/go.mod h1:yjVfjuY4nD1EW9i387Kau+I6V5cBA5YnC/mWNopjZrI=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 h1:f2LhPofnjcdOQKRtumKjMvIHkfSQ8aH/rwKUDEQ/SB4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2/go.mod h1:q+xX0H4OfuWDuBy7y/LDi4v8IBOWuF+vtp8Z6ex+lw4=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 h1:h7j73yuAVVjic8pqswh+L/7r2IHP43QwRyOu6zcCDDE=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2/go.mod h1:H07AHdK5LSy8F7EJUQhoxyiCNkePoHj2D8P2yGTWafo=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 h1:gbIaOzpXixUpoPK+js/bCBK1QBDXM22SigsnzGZio0U=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2/go.mod h1:p+S7RNbdGN8qgHDSg2SCQJ9FeMAmvcETQiVpeGhYnNM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 h1:o6MCcX1rJW8Y3g+hvg2xpjF6JR6DftuYhfl3Nc1WV9Q=
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1/go.mod h1:UDtxEWbREX6y4KREapT+jjtjoH0TiVSS6f5nfaY1UaM=
github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 h1:km+ZNjtLtpXYf42RdaDZnNHm9s7SYAuDGTafy6nd89A=
github.com/aws/aws-sdk-go-v2/service/sso v1.17.1/go.mod h1:aHBr3pvBSD5MbzOvQtYutyPLLRPbl/y9x86XyJJnUXQ=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 h1:iRFNqZH4a67IqPvK8xxtyQYnyrlsvwmpHOe9r55ggBA=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1/go.mod h1:pTy5WM+6sNv2tB24JNKFtn6EvciQ5k40ZJ0pq/Iaxj0=
github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 h1:txgVXIXWPXyqdiVn92BV6a/rgtpX31HYdsOYj0sVQQQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.25.1/go.mod h1:VAiJiNaoP1L89STFlEMgmHX1bKixY+FaP+TpRFrmyZ4=
github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik=
github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -51,12 +51,12 @@ github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
github.com/gofiber/fiber/v2 v2.51.0 h1:JNACcZy5e2tGApWB2QrRpenTWn0fq0hkFm6k0C86gKQ=
github.com/gofiber/fiber/v2 v2.51.0/go.mod h1:xaQRZQJGqnKOQnbQw+ltvku3/h8QxvNi8o6JiJ7Ll0U=
github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -67,8 +67,8 @@ github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
@@ -89,8 +89,8 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
github.com/segmentio/kafka-go v0.4.44 h1:Vjjksniy0WSTZ7CuVJrz1k04UoZeTc77UV6Yyk6tLY4=
github.com/segmentio/kafka-go v0.4.44/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -98,12 +98,12 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI=
github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
@@ -120,9 +120,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -147,8 +146,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -161,8 +160,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

View File

@@ -17,16 +17,16 @@ type prefResult struct {
err error
}
func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
var sg sync.WaitGroup
results := make([]prefResult, files)
start := time.Now()
if objSize == 0 {
return fmt.Errorf("must specify object size for upload")
return 0, time.Since(start), fmt.Errorf("must specify object size for upload")
}
if objSize > (int64(10000) * s.PartSize) {
return fmt.Errorf("object size can not exceed 10000 * chunksize")
return 0, time.Since(start), fmt.Errorf("object size can not exceed 10000 * chunksize")
}
runF("performance test: upload objects")
@@ -45,13 +45,13 @@ func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) erro
}(i)
}
sg.Wait()
elapsed := time.Since(start)
elapsed = time.Since(start)
var tot int64
for i, res := range results {
if res.err != nil {
failF("%v: %v\n", i, res.err)
break
return 0, time.Since(start), res.err
}
tot += res.size
fmt.Printf("%v: %v in %v (%v MB/s)\n",
@@ -63,10 +63,10 @@ func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) erro
passF("run upload: %v in %v (%v MB/s)\n",
tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
return nil
return tot, time.Since(start), nil
}
func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
var sg sync.WaitGroup
results := make([]prefResult, files)
start := time.Now()
@@ -86,13 +86,13 @@ func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) er
}(i)
}
sg.Wait()
elapsed := time.Since(start)
elapsed = time.Since(start)
var tot int64
for i, res := range results {
if res.err != nil {
failF("%v: %v\n", i, res.err)
break
return 0, elapsed, err
}
tot += res.size
fmt.Printf("%v: %v in %v (%v MB/s)\n",
@@ -104,10 +104,10 @@ func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) er
passF("run download: %v in %v (%v MB/s)\n",
tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
return nil
return tot, elapsed, nil
}
func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) error {
func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) (time.Duration, int, error) {
client := s3.NewFromConfig(s.Config())
var wg sync.WaitGroup
var resErr error
@@ -132,11 +132,11 @@ func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) error {
wg.Wait()
if resErr != nil {
failF("performance test failed with error: %w", resErr)
return nil
return time.Since(startTime), 0, resErr
}
elapsedTime := time.Since(startTime)
rps := int(float64(totalReqs) / elapsedTime.Seconds())
passF("Success\nTotal Requests: %d,\nConcurrency Level: %d,\nTime Taken: %s,\nRequests Per Second: %dreq/sec", totalReqs, s.Concurrency, elapsedTime, rps)
return nil
return elapsedTime, rps, nil
}

View File

@@ -51,7 +51,6 @@ func TestPutObject(s *S3Conf) {
PutObject_special_chars(s)
PutObject_invalid_long_tags(s)
PutObject_success(s)
PutObject_invalid_credentials(s)
}
func TestHeadObject(s *S3Conf) {
@@ -223,133 +222,3 @@ func TestPosix(s *S3Conf) {
PutObject_dir_obj_with_data(s)
CreateMultipartUpload_dir_obj(s)
}
type IntTests map[string]func(s *S3Conf) error
func GetIntTests() IntTests {
return IntTests{
"Authentication_empty_auth_header": Authentication_empty_auth_header,
"Authentication_invalid_auth_header": Authentication_invalid_auth_header,
"Authentication_unsupported_signature_version": Authentication_unsupported_signature_version,
"Authentication_malformed_credentials": Authentication_malformed_credentials,
"Authentication_malformed_credentials_invalid_parts": Authentication_malformed_credentials_invalid_parts,
"Authentication_credentials_terminated_string": Authentication_credentials_terminated_string,
"Authentication_credentials_incorrect_service": Authentication_credentials_incorrect_service,
"Authentication_credentials_incorrect_region": Authentication_credentials_incorrect_region,
"Authentication_credentials_invalid_date": Authentication_credentials_invalid_date,
"Authentication_credentials_future_date": Authentication_credentials_future_date,
"Authentication_credentials_past_date": Authentication_credentials_past_date,
"Authentication_credentials_non_existing_access_key": Authentication_credentials_non_existing_access_key,
"Authentication_invalid_signed_headers": Authentication_invalid_signed_headers,
"Authentication_missing_date_header": Authentication_missing_date_header,
"Authentication_invalid_date_header": Authentication_invalid_date_header,
"Authentication_date_mismatch": Authentication_date_mismatch,
"Authentication_incorrect_payload_hash": Authentication_incorrect_payload_hash,
"Authentication_incorrect_md5": Authentication_incorrect_md5,
"Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key,
"CreateBucket_invalid_bucket_name": CreateBucket_invalid_bucket_name,
"CreateBucket_existing_bucket": CreateBucket_existing_bucket,
"CreateBucket_as_user": CreateBucket_as_user,
"CreateDeleteBucket_success": CreateDeleteBucket_success,
"HeadBucket_non_existing_bucket": HeadBucket_non_existing_bucket,
"HeadBucket_success": HeadBucket_success,
"ListBuckets_as_user": ListBuckets_as_user,
"ListBuckets_as_admin": ListBuckets_as_admin,
"ListBuckets_success": ListBuckets_success,
"DeleteBucket_non_existing_bucket": DeleteBucket_non_existing_bucket,
"DeleteBucket_non_empty_bucket": DeleteBucket_non_empty_bucket,
"DeleteBucket_success_status_code": DeleteBucket_success_status_code,
"PutObject_non_existing_bucket": PutObject_non_existing_bucket,
"PutObject_special_chars": PutObject_special_chars,
"PutObject_invalid_long_tags": PutObject_invalid_long_tags,
"PutObject_success": PutObject_success,
"PutObject_invalid_credentials": PutObject_invalid_credentials,
"HeadObject_non_existing_object": HeadObject_non_existing_object,
"HeadObject_success": HeadObject_success,
"GetObject_non_existing_key": GetObject_non_existing_key,
"GetObject_invalid_ranges": GetObject_invalid_ranges,
"GetObject_with_meta": GetObject_with_meta,
"GetObject_success": GetObject_success,
"GetObject_by_range_success": GetObject_by_range_success,
"ListObjects_non_existing_bucket": ListObjects_non_existing_bucket,
"ListObjects_with_prefix": ListObjects_with_prefix,
"ListObject_truncated": ListObject_truncated,
"ListObjects_invalid_max_keys": ListObjects_invalid_max_keys,
"ListObjects_max_keys_0": ListObjects_max_keys_0,
"ListObjects_delimiter": ListObjects_delimiter,
"ListObjects_max_keys_none": ListObjects_max_keys_none,
"ListObjects_marker_not_from_obj_list": ListObjects_marker_not_from_obj_list,
"DeleteObject_non_existing_object": DeleteObject_non_existing_object,
"DeleteObject_success": DeleteObject_success,
"DeleteObject_success_status_code": DeleteObject_success_status_code,
"DeleteObjects_empty_input": DeleteObjects_empty_input,
"DeleteObjects_non_existing_objects": DeleteObjects_non_existing_objects,
"DeleteObjects_success": DeleteObjects_success,
"CopyObject_non_existing_dst_bucket": CopyObject_non_existing_dst_bucket,
"CopyObject_not_owned_source_bucket": CopyObject_not_owned_source_bucket,
"CopyObject_copy_to_itself": CopyObject_copy_to_itself,
"CopyObject_to_itself_with_new_metadata": CopyObject_to_itself_with_new_metadata,
"CopyObject_success": CopyObject_success,
"PutObjectTagging_non_existing_object": PutObjectTagging_non_existing_object,
"PutObjectTagging_long_tags": PutObjectTagging_long_tags,
"PutObjectTagging_success": PutObjectTagging_success,
"GetObjectTagging_non_existing_object": GetObjectTagging_non_existing_object,
"GetObjectTagging_success": GetObjectTagging_success,
"DeleteObjectTagging_non_existing_object": DeleteObjectTagging_non_existing_object,
"DeleteObjectTagging_success_status": DeleteObjectTagging_success_status,
"DeleteObjectTagging_success": DeleteObjectTagging_success,
"CreateMultipartUpload_non_existing_bucket": CreateMultipartUpload_non_existing_bucket,
"CreateMultipartUpload_success": CreateMultipartUpload_success,
"UploadPart_non_existing_bucket": UploadPart_non_existing_bucket,
"UploadPart_invalid_part_number": UploadPart_invalid_part_number,
"UploadPart_non_existing_key": UploadPart_non_existing_key,
"UploadPart_non_existing_mp_upload": UploadPart_non_existing_mp_upload,
"UploadPart_success": UploadPart_success,
"UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket,
"UploadPartCopy_incorrect_uploadId": UploadPartCopy_incorrect_uploadId,
"UploadPartCopy_incorrect_object_key": UploadPartCopy_incorrect_object_key,
"UploadPartCopy_invalid_part_number": UploadPartCopy_invalid_part_number,
"UploadPartCopy_invalid_copy_source": UploadPartCopy_invalid_copy_source,
"UploadPartCopy_non_existing_source_bucket": UploadPartCopy_non_existing_source_bucket,
"UploadPartCopy_non_existing_source_object_key": UploadPartCopy_non_existing_source_object_key,
"UploadPartCopy_success": UploadPartCopy_success,
"UploadPartCopy_by_range_invalid_range": UploadPartCopy_by_range_invalid_range,
"UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size,
"UploadPartCopy_by_range_success": UploadPartCopy_by_range_success,
"ListParts_incorrect_uploadId": ListParts_incorrect_uploadId,
"ListParts_incorrect_object_key": ListParts_incorrect_object_key,
"ListParts_success": ListParts_success,
"ListMultipartUploads_non_existing_bucket": ListMultipartUploads_non_existing_bucket,
"ListMultipartUploads_empty_result": ListMultipartUploads_empty_result,
"ListMultipartUploads_invalid_max_uploads": ListMultipartUploads_invalid_max_uploads,
"ListMultipartUploads_max_uploads": ListMultipartUploads_max_uploads,
"ListMultipartUploads_incorrect_next_key_marker": ListMultipartUploads_incorrect_next_key_marker,
"ListMultipartUploads_ignore_upload_id_marker": ListMultipartUploads_ignore_upload_id_marker,
"ListMultipartUploads_success": ListMultipartUploads_success,
"AbortMultipartUpload_non_existing_bucket": AbortMultipartUpload_non_existing_bucket,
"AbortMultipartUpload_incorrect_uploadId": AbortMultipartUpload_incorrect_uploadId,
"AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key,
"AbortMultipartUpload_success": AbortMultipartUpload_success,
"AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code,
"CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket,
"CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number,
"CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag,
"CompleteMultipartUpload_success": CompleteMultipartUpload_success,
"PutBucketAcl_non_existing_bucket": PutBucketAcl_non_existing_bucket,
"PutBucketAcl_invalid_acl_canned_and_acp": PutBucketAcl_invalid_acl_canned_and_acp,
"PutBucketAcl_invalid_acl_canned_and_grants": PutBucketAcl_invalid_acl_canned_and_grants,
"PutBucketAcl_invalid_acl_acp_and_grants": PutBucketAcl_invalid_acl_acp_and_grants,
"PutBucketAcl_invalid_owner": PutBucketAcl_invalid_owner,
"PutBucketAcl_success_access_denied": PutBucketAcl_success_access_denied,
"PutBucketAcl_success_grants": PutBucketAcl_success_grants,
"PutBucketAcl_success_canned_acl": PutBucketAcl_success_canned_acl,
"PutBucketAcl_success_acp": PutBucketAcl_success_acp,
"GetBucketAcl_non_existing_bucket": GetBucketAcl_non_existing_bucket,
"GetBucketAcl_access_denied": GetBucketAcl_access_denied,
"GetBucketAcl_success": GetBucketAcl_success,
"PutObject_overwrite_dir_obj": PutObject_overwrite_dir_obj,
"PutObject_overwrite_file_obj": PutObject_overwrite_file_obj,
"PutObject_dir_obj_with_data": PutObject_dir_obj_with_data,
"CreateMultipartUpload_dir_obj": CreateMultipartUpload_dir_obj,
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -60,7 +60,7 @@ func teardown(s *S3Conf, bucket string) error {
})
cancel()
if err != nil {
return fmt.Errorf("failed to delete object %v: %w", *key, err)
return fmt.Errorf("failed to delete object %v: %v", *key, err)
}
return nil
}
@@ -71,7 +71,7 @@ func teardown(s *S3Conf, bucket string) error {
out, err := s3client.ListObjectsV2(ctx, in)
cancel()
if err != nil {
return fmt.Errorf("failed to list objects: %w", err)
return fmt.Errorf("failed to list objects: %v", err)
}
for _, item := range out.Contents {
@@ -81,7 +81,7 @@ func teardown(s *S3Conf, bucket string) error {
}
}
if *out.IsTruncated {
if out.IsTruncated {
in.ContinuationToken = out.ContinuationToken
} else {
break
@@ -96,32 +96,31 @@ func teardown(s *S3Conf, bucket string) error {
return err
}
func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) error {
func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) {
runF(testName)
bucketName := getBucketName()
err := setup(s, bucketName)
if err != nil {
failF("%v: failed to create a bucket: %v", testName, err)
return fmt.Errorf("%v: failed to create a bucket: %w", testName, err)
failF("%v: failed to create a bucket: %v", testName, err.Error())
return
}
client := s3.NewFromConfig(s.Config())
handlerErr := handler(client, bucketName)
if handlerErr != nil {
failF("%v: %v", testName, handlerErr)
failF("%v: %v", testName, handlerErr.Error())
}
err = teardown(s, bucketName)
if err != nil {
fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err)
if handlerErr == nil {
return fmt.Errorf("%v: failed to delete the bucket: %w", testName, err)
failF("%v: failed to delete the bucket: %v", testName, err.Error())
} else {
fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err.Error())
}
}
if handlerErr == nil {
passF(testName)
}
return handlerErr
}
type authConfig struct {
@@ -133,21 +132,20 @@ type authConfig struct {
date time.Time
}
func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) error {
func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) {
runF(cfg.testName)
req, err := createSignedReq(cfg.method, s.endpoint, cfg.path, s.awsID, s.awsSecret, cfg.service, s.awsRegion, cfg.body, cfg.date)
if err != nil {
failF("%v: %v", cfg.testName, err)
return fmt.Errorf("%v: %w", cfg.testName, err)
failF("%v: %v", cfg.testName, err.Error())
return
}
err = handler(req)
if err != nil {
failF("%v: %v", cfg.testName, err)
return fmt.Errorf("%v: %w", cfg.testName, err)
failF("%v: %v", cfg.testName, err.Error())
return
}
passF(cfg.testName)
return nil
}
func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
@@ -207,8 +205,9 @@ func checkApiErr(err error, apiErr s3err.APIError) error {
}
return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
} else {
return fmt.Errorf("expected aws api error, instead got: %v", err.Error())
}
return fmt.Errorf("expected aws api error, instead got: %w", err)
}
func checkSdkApiErr(err error, code string) error {
@@ -294,7 +293,7 @@ func compareParts(parts1, parts2 []types.Part) bool {
}
for i, prt := range parts1 {
if *prt.PartNumber != *parts2[i].PartNumber {
if prt.PartNumber != parts2[i].PartNumber {
return false
}
if *prt.ETag != *parts2[i].ETag {
@@ -485,23 +484,20 @@ func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId s
return parts, err
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
pn := int32(partNumber)
out, err := client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
Body: bytes.NewReader(partBuffer),
PartNumber: &pn,
PartNumber: int32(partNumber),
})
cancel()
if err != nil {
return parts, err
} else {
parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)})
offset += partSize
}
parts = append(parts, types.Part{
ETag: out.ETag,
PartNumber: &pn,
})
offset += partSize
}
return parts, err

View File

@@ -1,7 +1,6 @@
#!/bin/bash
# make temp dirs
rm -rf /tmp/gw
mkdir /tmp/gw
rm -rf /tmp/covdata
mkdir /tmp/covdata

View File

@@ -4,7 +4,6 @@
package controllers
import (
"bufio"
"context"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/versity/versitygw/backend"
@@ -107,7 +106,7 @@ var _ backend.Backend = &BackendMock{}
// RestoreObjectFunc: func(contextMoqParam context.Context, restoreObjectInput *s3.RestoreObjectInput) error {
// panic("mock out the RestoreObject method")
// },
// SelectObjectContentFunc: func(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
// SelectObjectContentFunc: func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
// panic("mock out the SelectObjectContent method")
// },
// ShutdownFunc: func() {
@@ -214,7 +213,7 @@ type BackendMock struct {
RestoreObjectFunc func(contextMoqParam context.Context, restoreObjectInput *s3.RestoreObjectInput) error
// SelectObjectContentFunc mocks the SelectObjectContent method.
SelectObjectContentFunc func(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
SelectObjectContentFunc func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
// ShutdownFunc mocks the Shutdown method.
ShutdownFunc func()
@@ -442,10 +441,10 @@ type BackendMock struct {
}
// SelectObjectContent holds details about calls to the SelectObjectContent method.
SelectObjectContent []struct {
// Ctx is the ctx argument value.
Ctx context.Context
// Input is the input argument value.
Input *s3.SelectObjectContentInput
// ContextMoqParam is the contextMoqParam argument value.
ContextMoqParam context.Context
// SelectObjectContentInput is the selectObjectContentInput argument value.
SelectObjectContentInput *s3.SelectObjectContentInput
}
// Shutdown holds details about calls to the Shutdown method.
Shutdown []struct {
@@ -1540,21 +1539,21 @@ func (mock *BackendMock) RestoreObjectCalls() []struct {
}
// SelectObjectContent calls SelectObjectContentFunc.
func (mock *BackendMock) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
func (mock *BackendMock) SelectObjectContent(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
if mock.SelectObjectContentFunc == nil {
panic("BackendMock.SelectObjectContentFunc: method is nil but Backend.SelectObjectContent was just called")
}
callInfo := struct {
Ctx context.Context
Input *s3.SelectObjectContentInput
ContextMoqParam context.Context
SelectObjectContentInput *s3.SelectObjectContentInput
}{
Ctx: ctx,
Input: input,
ContextMoqParam: contextMoqParam,
SelectObjectContentInput: selectObjectContentInput,
}
mock.lockSelectObjectContent.Lock()
mock.calls.SelectObjectContent = append(mock.calls.SelectObjectContent, callInfo)
mock.lockSelectObjectContent.Unlock()
return mock.SelectObjectContentFunc(ctx, input)
return mock.SelectObjectContentFunc(contextMoqParam, selectObjectContentInput)
}
// SelectObjectContentCalls gets all the calls that were made to SelectObjectContent.
@@ -1562,12 +1561,12 @@ func (mock *BackendMock) SelectObjectContent(ctx context.Context, input *s3.Sele
//
// len(mockedBackend.SelectObjectContentCalls())
func (mock *BackendMock) SelectObjectContentCalls() []struct {
Ctx context.Context
Input *s3.SelectObjectContentInput
ContextMoqParam context.Context
SelectObjectContentInput *s3.SelectObjectContentInput
} {
var calls []struct {
Ctx context.Context
Input *s3.SelectObjectContentInput
ContextMoqParam context.Context
SelectObjectContentInput *s3.SelectObjectContentInput
}
mock.lockSelectObjectContent.RLock()
calls = mock.calls.SelectObjectContent

View File

@@ -17,7 +17,6 @@ package controllers
import (
"bytes"
"encoding/xml"
"errors"
"fmt"
"io"
"log"
@@ -64,7 +63,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
key := ctx.Params("key")
keyEnd := ctx.Params("*1")
uploadId := ctx.Query("uploadId")
maxParts := int32(ctx.QueryInt("max-parts", -1))
maxParts := ctx.QueryInt("max-parts", 0)
partNumberMarker := ctx.Query("part-number-marker")
acceptRange := ctx.Get("Range")
acct := ctx.Locals("account").(auth.Account)
@@ -93,7 +92,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
}
if uploadId != "" {
if maxParts < 0 && ctx.Request().URI().QueryArgs().Has("max-parts") {
if maxParts < 0 || (maxParts == 0 && ctx.Query("max-parts") != "") {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts), &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
}
if partNumberMarker != "" {
@@ -106,17 +105,13 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
}
var mxParts *int32
if ctx.Request().URI().QueryArgs().Has("max-parts") {
mxParts = &maxParts
}
res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{
Bucket: &bucket,
Key: &key,
UploadId: &uploadId,
PartNumberMarker: &partNumberMarker,
MaxParts: mxParts,
MaxParts: int32(maxParts),
})
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
}
@@ -174,7 +169,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
{
Key: "Content-Length",
Value: fmt.Sprint(getint64(res.ContentLength)),
Value: fmt.Sprint(res.ContentLength),
},
{
Key: "Content-Type",
@@ -204,17 +199,11 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
Key: "accept-ranges",
Value: getstring(res.AcceptRanges),
},
{
Key: "x-amz-tagging-count",
Value: fmt.Sprint(res.TagCount),
},
})
if res.TagCount != nil {
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
{
Key: "x-amz-tagging-count",
Value: fmt.Sprint(*res.TagCount),
},
})
}
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner})
}
@@ -225,13 +214,6 @@ func getstring(s *string) string {
return *s
}
func getint64(i *int64) int64 {
if i == nil {
return 0
}
return *i
}
func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
bucket := ctx.Params("bucket")
prefix := ctx.Query("prefix")
@@ -277,7 +259,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
Delimiter: &delimiter,
Prefix: &prefix,
UploadIdMarker: &uploadIdMarker,
MaxUploads: &maxUploads,
MaxUploads: maxUploads,
KeyMarker: &keyMarker,
})
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner})
@@ -300,7 +282,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
Prefix: &prefix,
ContinuationToken: &cToken,
Delimiter: &delimiter,
MaxKeys: &maxkeys,
MaxKeys: maxkeys,
})
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner})
}
@@ -323,7 +305,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
Prefix: &prefix,
Marker: &marker,
Delimiter: &delimiter,
MaxKeys: &maxkeys,
MaxKeys: maxkeys,
})
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner})
}
@@ -485,7 +467,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") && copySource != "" {
partNumber := int32(ctx.QueryInt("partNumber", -1))
partNumber := ctx.QueryInt("partNumber", -1)
if partNumber < 1 || partNumber > 10000 {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPartCopy", BucketOwner: parsedAcl.Owner})
}
@@ -494,7 +476,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
Bucket: &bucket,
Key: &keyStart,
CopySource: &copySource,
PartNumber: &partNumber,
PartNumber: int32(partNumber),
UploadId: &uploadId,
ExpectedBucketOwner: &bucketOwner,
CopySourceRange: &copySrcRange,
@@ -503,7 +485,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
}
if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") {
partNumber := int32(ctx.QueryInt("partNumber", -1))
partNumber := ctx.QueryInt("partNumber", -1)
if partNumber < 1 || partNumber > 10000 {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
}
@@ -517,21 +499,14 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
}
var body io.Reader
bodyi := ctx.Locals("body-reader")
if bodyi != nil {
body = bodyi.(io.Reader)
} else {
body = bytes.NewReader([]byte{})
}
body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body())))
ctx.Locals("logReqBody", false)
etag, err := c.be.UploadPart(ctx.Context(), &s3.UploadPartInput{
Bucket: &bucket,
Key: &keyStart,
UploadId: &uploadId,
PartNumber: &partNumber,
ContentLength: &contentLength,
PartNumber: int32(partNumber),
ContentLength: contentLength,
Body: body,
})
ctx.Response().Header.Set("Etag", etag)
@@ -603,21 +578,20 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
}
var mtime *time.Time
var umtime *time.Time
var mtime time.Time
var err error
if copySrcModifSince != "" {
tm, err := time.Parse(iso8601Format, copySrcModifSince)
mtime, err = time.Parse(iso8601Format, copySrcModifSince)
if err != nil {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidCopySource), &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
}
mtime = &tm
}
if copySrcUnmodifSince != "" {
tm, err := time.Parse(iso8601Format, copySrcUnmodifSince)
var umtime time.Time
if copySrcModifSince != "" {
mtime, err = time.Parse(iso8601Format, copySrcUnmodifSince)
if err != nil {
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidCopySource), &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
}
umtime = &tm
}
metadata := utils.GetUserMetaData(&ctx.Request().Header)
@@ -628,8 +602,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
CopySource: &copySource,
CopySourceIfMatch: &copySrcIfMatch,
CopySourceIfNoneMatch: &copySrcIfNoneMatch,
CopySourceIfModifiedSince: mtime,
CopySourceIfUnmodifiedSince: umtime,
CopySourceIfModifiedSince: &mtime,
CopySourceIfUnmodifiedSince: &umtime,
ExpectedBucketOwner: &acct.Access,
Metadata: metadata,
})
@@ -663,21 +637,13 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutObject", BucketOwner: parsedAcl.Owner})
}
var body io.Reader
bodyi := ctx.Locals("body-reader")
if bodyi != nil {
body = bodyi.(io.Reader)
} else {
body = bytes.NewReader([]byte{})
}
ctx.Locals("logReqBody", false)
etag, err := c.be.PutObject(ctx.Context(), &s3.PutObjectInput{
Bucket: &bucket,
Key: &keyStart,
ContentLength: &contentLength,
ContentLength: contentLength,
Metadata: metadata,
Body: body,
Body: bytes.NewReader(ctx.Request().Body()),
Tagging: &tagging,
})
ctx.Response().Header.Set("ETag", etag)
@@ -839,7 +805,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
{
Key: "Content-Length",
Value: fmt.Sprint(getint64(res.ContentLength)),
Value: fmt.Sprint(res.ContentLength),
},
{
Key: "Content-Type",
@@ -927,7 +893,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "SelectObjectContent", BucketOwner: parsedAcl.Owner})
}
sw := c.be.SelectObjectContent(ctx.Context(), &s3.SelectObjectContentInput{
res, err := c.be.SelectObjectContent(ctx.Context(), &s3.SelectObjectContentInput{
Bucket: &bucket,
Key: &key,
Expression: payload.Expression,
@@ -937,10 +903,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
RequestProgress: payload.RequestProgress,
ScanRange: payload.ScanRange,
})
ctx.Context().SetBodyStreamWriter(sw)
return nil
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "SelectObjectContent", BucketOwner: parsedAcl.Owner})
}
if uploadId != "" {
@@ -1017,10 +980,10 @@ func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error {
})
}
if err != nil {
var apierr s3err.APIError
if errors.As(err, &apierr) {
ctx.Status(apierr.HTTPStatusCode)
return ctx.Send(s3err.GetAPIErrorResponse(apierr, "", "", ""))
serr, ok := err.(s3err.APIError)
if ok {
ctx.Status(serr.HTTPStatusCode)
return ctx.Send(s3err.GetAPIErrorResponse(serr, "", "", ""))
}
log.Printf("Internal Error, %v", err)

View File

@@ -15,7 +15,6 @@
package controllers
import (
"bufio"
"context"
"encoding/json"
"fmt"
@@ -175,7 +174,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
now := time.Now()
app := fiber.New()
contentLength := int64(1000)
s3ApiController := S3ApiController{
be: &BackendMock{
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -196,7 +194,7 @@ func TestS3ApiController_GetActions(t *testing.T) {
ContentType: getPtr("application/xml"),
ContentEncoding: getPtr("gzip"),
ETag: getPtr("98sda7f97sa9df798sd79f8as9df"),
ContentLength: &contentLength,
ContentLength: 1000,
LastModified: &now,
StorageClass: "storage class",
}, nil
@@ -1200,7 +1198,6 @@ func TestS3ApiController_HeadObject(t *testing.T) {
contentType := "application/xml"
eTag := "Valid etag"
lastModifie := time.Now()
contentLength := int64(64)
s3ApiController := S3ApiController{
be: &BackendMock{
@@ -1210,7 +1207,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
return &s3.HeadObjectOutput{
ContentEncoding: &contentEncoding,
ContentLength: &contentLength,
ContentLength: 64,
ContentType: &contentType,
LastModified: &lastModifie,
ETag: &eTag,
@@ -1309,8 +1306,8 @@ func TestS3ApiController_CreateActions(t *testing.T) {
CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
return &s3.CreateMultipartUploadOutput{}, nil
},
SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) {
return func(w *bufio.Writer) {}
SelectObjectContentFunc: func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
return s3response.SelectObjectContentResult{}, nil
},
},
}

View File

@@ -18,11 +18,15 @@ import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"math"
"net/http"
"strconv"
"os"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/smithy-go/logging"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/auth"
"github.com/versity/versitygw/s3api/controllers"
@@ -33,6 +37,7 @@ import (
const (
iso8601Format = "20060102T150405Z"
YYYYMMDD = "20060102"
)
type RootUserConfig struct {
@@ -48,93 +53,137 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
ctx.Locals("startTime", time.Now())
authorization := ctx.Get("Authorization")
if authorization == "" {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger)
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), &controllers.MetaOpts{Logger: logger})
}
authData, err := utils.ParseAuthorization(authorization)
if err != nil {
return sendResponse(ctx, err, logger)
// Check the signature version
authParts := strings.Split(authorization, ",")
for i, el := range authParts {
authParts[i] = strings.TrimSpace(el)
}
if authData.Algorithm != "AWS4-HMAC-SHA256" {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger)
if len(authParts) != 3 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
}
if authData.Region != region {
return sendResponse(ctx, s3err.APIError{
startParts := strings.Split(authParts[0], " ")
if startParts[0] != "AWS4-HMAC-SHA256" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), &controllers.MetaOpts{Logger: logger})
}
credKv := strings.Split(startParts[1], "=")
if len(credKv) != 2 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
}
// Credential variables validation
creds := strings.Split(credKv[1], "/")
if len(creds) != 5 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
}
if creds[4] != "aws4_request" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureTerminationStr), &controllers.MetaOpts{Logger: logger})
}
if creds[3] != "s3" {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureIncorrService), &controllers.MetaOpts{Logger: logger})
}
if creds[2] != region {
return controllers.SendResponse(ctx, s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region),
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),
HTTPStatusCode: http.StatusForbidden,
}, logger)
}, &controllers.MetaOpts{Logger: logger})
}
ctx.Locals("isRoot", authData.Access == root.Access)
ctx.Locals("isRoot", creds[0] == root.Access)
account, err := acct.getAccount(authData.Access)
_, err := time.Parse(YYYYMMDD, creds[1])
if err != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
}
signHdrKv := strings.Split(authParts[1], "=")
if len(signHdrKv) != 2 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
}
signedHdrs := strings.Split(signHdrKv[1], ";")
account, err := acct.getAccount(creds[0])
if err == auth.ErrNoSuchUser {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), &controllers.MetaOpts{Logger: logger})
}
if err != nil {
return sendResponse(ctx, err, logger)
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}
ctx.Locals("account", account)
// Check X-Amz-Date header
date := ctx.Get("X-Amz-Date")
if date == "" {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger)
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), &controllers.MetaOpts{Logger: logger})
}
// Parse the date and check the date validity
tdate, err := time.Parse(iso8601Format, date)
if err != nil {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger)
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), &controllers.MetaOpts{Logger: logger})
}
if date[:8] != authData.Date {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger)
if date[:8] != creds[1] {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
}
// Validate the dates difference
err = validateDate(tdate)
if err != nil {
return sendResponse(ctx, err, logger)
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}
if utils.IsBigDataAction(ctx) {
// for streaming PUT actions, authorization is deferred
// until end of stream due to need to get length and
// checksum of the stream to validate authorization
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
return utils.NewAuthReader(ctx, r, authData, account.Secret, debug)
})
return ctx.Next()
}
hashPayloadHeader := ctx.Get("X-Amz-Content-Sha256")
ok := isSpecialPayload(hashPayloadHeader)
hashPayload := ctx.Get("X-Amz-Content-Sha256")
if !utils.IsSpecialPayload(hashPayload) {
if !ok {
// Calculate the hash of the request payload
hashedPayload := sha256.Sum256(ctx.Body())
hexPayload := hex.EncodeToString(hashedPayload[:])
// Compare the calculated hash with the hash provided
if hashPayload != hexPayload {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger)
if hashPayloadHeader != hexPayload {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), &controllers.MetaOpts{Logger: logger})
}
}
var contentLength int64
contentLengthStr := ctx.Get("Content-Length")
if contentLengthStr != "" {
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger)
}
}
err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
// Create a new http request instance from fasthttp request
req, err := utils.CreateHttpRequestFromCtx(ctx, signedHdrs)
if err != nil {
return sendResponse(ctx, err, logger)
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
}
signer := v4.NewSigner()
signErr := signer.SignHTTP(req.Context(), aws.Credentials{
AccessKeyID: creds[0],
SecretAccessKey: account.Secret,
}, req, hashPayloadHeader, creds[3], region, tdate, func(options *v4.SignerOptions) {
options.DisableURIPathEscaping = true
if debug {
options.LogSigning = true
options.Logger = logging.NewStandardLogger(os.Stderr)
}
})
if signErr != nil {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
}
parts := strings.Split(req.Header.Get("Authorization"), " ")
if len(parts) < 4 {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
}
calculatedSign := strings.Split(parts[3], "=")[1]
expectedSign := strings.Split(authParts[2], "=")[1]
if expectedSign != calculatedSign {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch), &controllers.MetaOpts{Logger: logger})
}
return ctx.Next()
@@ -158,29 +207,39 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
return a.iam.GetUserAccount(access)
}
func isSpecialPayload(str string) bool {
specialValues := map[string]bool{
"UNSIGNED-PAYLOAD": true,
"STREAMING-UNSIGNED-PAYLOAD-TRAILER": true,
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD": true,
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER": true,
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD": true,
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
}
return specialValues[str]
}
func validateDate(date time.Time) error {
now := time.Now().UTC()
diff := date.Unix() - now.Unix()
// Checks the dates difference to be less than a minute
if diff > 60 {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
}
}
if diff < -60 {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
if math.Abs(float64(diff)) > 60 {
if diff > 0 {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
}
} else {
return s3err.APIError{
Code: "SignatureDoesNotMatch",
Description: fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
HTTPStatusCode: http.StatusForbidden,
}
}
}
return nil
}
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
}

View File

@@ -1,31 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package middlewares
import (
"io"
"github.com/gofiber/fiber/v2"
)
func wrapBodyReader(ctx *fiber.Ctx, wr func(io.Reader) io.Reader) {
r, ok := ctx.Locals("body-reader").(io.Reader)
if !ok {
r = ctx.Request().BodyStream()
}
r = wr(r)
ctx.Locals("body-reader", r)
}

View File

@@ -16,11 +16,10 @@ package middlewares
import (
"crypto/md5"
"io"
"encoding/base64"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3api/controllers"
"github.com/versity/versitygw/s3api/utils"
"github.com/versity/versitygw/s3err"
"github.com/versity/versitygw/s3log"
)
@@ -32,16 +31,8 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
return ctx.Next()
}
if utils.IsBigDataAction(ctx) {
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
r, _ = utils.NewHashReader(r, incomingSum, utils.HashTypeMd5)
return r
})
return ctx.Next()
}
sum := md5.Sum(ctx.Body())
calculatedSum := utils.Md5SumString(sum[:])
calculatedSum := base64.StdEncoding.EncodeToString(sum[:])
if incomingSum != calculatedSum {
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})

View File

@@ -32,7 +32,6 @@ type S3ApiServer struct {
router *S3ApiRouter
port string
cert *tls.Certificate
quiet bool
debug bool
}
@@ -49,9 +48,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
}
// Logging middlewares
if !server.quiet {
app.Use(logger.New())
}
app.Use(logger.New())
app.Use(middlewares.DecodeURL(l))
app.Use(middlewares.RequestLogger(server.debug))
@@ -83,11 +80,6 @@ func WithDebug() Option {
return func(s *S3ApiServer) { s.debug = true }
}
// WithQuiet silences default logging output
func WithQuiet() Option {
return func(s *S3ApiServer) { s.quiet = true }
}
func (sa *S3ApiServer) Serve() (err error) {
if sa.cert != nil {
return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)

View File

@@ -1,261 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
"errors"
"fmt"
"io"
"os"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/smithy-go/logging"
"github.com/gofiber/fiber/v2"
"github.com/versity/versitygw/s3err"
)
const (
iso8601Format = "20060102T150405Z"
yyyymmdd = "20060102"
)
// AuthReader is an io.Reader that validates the request authorization
// once the underlying reader returns io.EOF. This is needed for streaming
// data requests where the data size and checksum are not known until
// the data is completely read.
type AuthReader struct {
ctx *fiber.Ctx
auth AuthData
secret string
size int
r *HashReader
debug bool
}
// NewAuthReader initializes an io.Reader that will verify the request
// v4 auth when the underlying reader returns io.EOF. This postpones the
// authorization check until the reader is consumed. So it is important that
// the consumer of this reader checks for the auth errors while reading.
func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, debug bool) *AuthReader {
var hr *HashReader
hashPayload := ctx.Get("X-Amz-Content-Sha256")
if !IsSpecialPayload(hashPayload) {
hr, _ = NewHashReader(r, "", HashTypeSha256)
} else {
hr, _ = NewHashReader(r, "", HashTypeNone)
}
return &AuthReader{
ctx: ctx,
r: hr,
auth: auth,
secret: secret,
debug: debug,
}
}
// Read allows *AuthReader to be used as an io.Reader
func (ar *AuthReader) Read(p []byte) (int, error) {
n, err := ar.r.Read(p)
ar.size += n
if errors.Is(err, io.EOF) {
verr := ar.validateSignature()
if verr != nil {
return n, verr
}
}
return n, err
}
func (ar *AuthReader) validateSignature() error {
date := ar.ctx.Get("X-Amz-Date")
if date == "" {
return s3err.GetAPIError(s3err.ErrMissingDateHeader)
}
hashPayload := ar.ctx.Get("X-Amz-Content-Sha256")
if !IsSpecialPayload(hashPayload) {
hexPayload := ar.r.Sum()
// Compare the calculated hash with the hash provided
if hashPayload != hexPayload {
return s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
}
}
// Parse the date and check the date validity
tdate, err := time.Parse(iso8601Format, date)
if err != nil {
return s3err.GetAPIError(s3err.ErrMalformedDate)
}
return CheckValidSignature(ar.ctx, ar.auth, ar.secret, hashPayload, tdate, int64(ar.size), ar.debug)
}
const (
service = "s3"
)
// CheckValidSignature validates the ctx v4 auth signature
func CheckValidSignature(ctx *fiber.Ctx, auth AuthData, secret, checksum string, tdate time.Time, contentLen int64, debug bool) error {
signedHdrs := strings.Split(auth.SignedHeaders, ";")
// Create a new http request instance from fasthttp request
req, err := createHttpRequestFromCtx(ctx, signedHdrs, contentLen)
if err != nil {
return fmt.Errorf("create http request from context: %w", err)
}
signer := v4.NewSigner()
signErr := signer.SignHTTP(req.Context(), aws.Credentials{
AccessKeyID: auth.Access,
SecretAccessKey: secret,
}, req, checksum, service, auth.Region, tdate, func(options *v4.SignerOptions) {
options.DisableURIPathEscaping = true
if debug {
options.LogSigning = true
options.Logger = logging.NewStandardLogger(os.Stderr)
}
})
if signErr != nil {
return fmt.Errorf("sign generated http request: %w", err)
}
genAuth, err := ParseAuthorization(req.Header.Get("Authorization"))
if err != nil {
return err
}
if auth.Signature != genAuth.Signature {
return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
}
return nil
}
// AuthData is the parsed authorization data from the header
type AuthData struct {
Algorithm string
Access string
Region string
SignedHeaders string
Signature string
Date string
}
// ParseAuthorization returns the parsed fields for the aws v4 auth header
// example authorization string from aws docs:
// Authorization: AWS4-HMAC-SHA256
// Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,
// SignedHeaders=host;range;x-amz-date,
// Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024
func ParseAuthorization(authorization string) (AuthData, error) {
a := AuthData{}
// authorization must start with:
// Authorization: <ALGORITHM>
// followed by key=value pairs separated by ","
authParts := strings.Fields(authorization)
for i, el := range authParts {
authParts[i] = strings.TrimSpace(el)
}
if len(authParts) < 3 {
return a, s3err.GetAPIError(s3err.ErrMissingFields)
}
algo := authParts[0]
kvData := strings.Join(authParts[1:], "")
kvPairs := strings.Split(kvData, ",")
// we are expecting at least Credential, SignedHeaders, and Signature
// key value pairs here
if len(kvPairs) < 3 {
return a, s3err.GetAPIError(s3err.ErrMissingFields)
}
var access, region, signedHeaders, signature, date string
for _, kv := range kvPairs {
keyValue := strings.Split(kv, "=")
if len(keyValue) != 2 {
switch {
case strings.HasPrefix(kv, "Credential"):
return a, s3err.GetAPIError(s3err.ErrCredMalformed)
case strings.HasPrefix(kv, "SignedHeaders"):
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
}
return a, s3err.GetAPIError(s3err.ErrMissingFields)
}
key := strings.TrimSpace(keyValue[0])
value := strings.TrimSpace(keyValue[1])
switch key {
case "Credential":
creds := strings.Split(value, "/")
if len(creds) != 5 {
return a, s3err.GetAPIError(s3err.ErrCredMalformed)
}
if creds[3] != "s3" {
return a, s3err.GetAPIError(s3err.ErrSignatureIncorrService)
}
if creds[4] != "aws4_request" {
return a, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)
}
_, err := time.Parse(yyyymmdd, creds[1])
if err != nil {
return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
}
access = creds[0]
date = creds[1]
region = creds[2]
case "SignedHeaders":
signedHeaders = value
case "Signature":
signature = value
}
}
return AuthData{
Algorithm: algo,
Access: access,
Region: region,
SignedHeaders: signedHeaders,
Signature: signature,
Date: date,
}, nil
}
var (
specialValues = map[string]bool{
"UNSIGNED-PAYLOAD": true,
"STREAMING-UNSIGNED-PAYLOAD-TRAILER": true,
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD": true,
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER": true,
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD": true,
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
}
)
// IsSpecialPayload checks for streaming/unsigned authorization types
func IsSpecialPayload(str string) bool {
return specialValues[str]
}

View File

@@ -1,130 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package utils
import (
"crypto/md5"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"hash"
"io"
"github.com/versity/versitygw/s3err"
)
// HashType identifies the checksum algorithm to be used
type HashType string
const (
// HashTypeMd5 generates MD5 checksum for the data stream
HashTypeMd5 = "md5"
// HashTypeSha256 generates SHA256 checksum for the data stream
HashTypeSha256 = "sha256"
// HashTypeNone is a no-op checksum for the data stream
HashTypeNone = "none"
)
// HashReader is an io.Reader that calculates the checksum
// as the data is read
type HashReader struct {
hashType HashType
hash hash.Hash
r io.Reader
sum string
}
var (
errInvalidHashType = errors.New("unsupported or invalid checksum type")
)
// NewHashReader intializes an io.Reader from an underlying io.Reader that
// calculates the checksum while the reader is being read from. If the
// sum provided is not "", the reader will return an error when the underlying
// reader returns io.EOF if the checksum does not match the provided expected
// checksum. If the provided sum is "", then the Sum() method can still
// be used to get the current checksum for the data read so far.
func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, error) {
var hash hash.Hash
switch ht {
case HashTypeMd5:
hash = md5.New()
case HashTypeSha256:
hash = sha256.New()
case HashTypeNone:
hash = noop{}
default:
return nil, errInvalidHashType
}
return &HashReader{
hash: hash,
r: r,
sum: expectedSum,
hashType: ht,
}, nil
}
// Read allows *HashReader to be used as an io.Reader
func (hr *HashReader) Read(p []byte) (int, error) {
n, readerr := hr.r.Read(p)
_, err := hr.hash.Write(p[:n])
if err != nil {
return n, err
}
if errors.Is(readerr, io.EOF) && hr.sum != "" {
switch hr.hashType {
case HashTypeMd5:
sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil))
if sum != hr.sum {
return n, s3err.GetAPIError(s3err.ErrInvalidDigest)
}
case HashTypeSha256:
sum := hex.EncodeToString(hr.hash.Sum(nil))
if sum != hr.sum {
return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
}
default:
return n, errInvalidHashType
}
}
return n, readerr
}
// Sum returns the checksum hash of the data read so far
func (hr *HashReader) Sum() string {
switch hr.hashType {
case HashTypeMd5:
return Md5SumString(hr.hash.Sum(nil))
case HashTypeSha256:
return hex.EncodeToString(hr.hash.Sum(nil))
default:
return ""
}
}
// Md5SumString converts the hash bytes to the string checksum value
func Md5SumString(b []byte) string {
return base64.StdEncoding.EncodeToString(b)
}
type noop struct{}
func (n noop) Write(p []byte) (int, error) { return 0, nil }
func (n noop) Sum(b []byte) []byte { return []byte{} }
func (n noop) Reset() {}
func (n noop) Size() int { return 0 }
func (n noop) BlockSize() int { return 1 }

View File

@@ -18,7 +18,6 @@ import (
"bytes"
"errors"
"fmt"
"io"
"net/http"
"regexp"
"strconv"
@@ -50,16 +49,10 @@ func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]strin
return
}
func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Request, error) {
req := ctx.Request()
var body io.Reader
if IsBigDataAction(ctx) {
body = req.BodyStream()
} else {
body = bytes.NewReader(req.Body())
}
httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), bytes.NewReader(req.Body()))
if err != nil {
return nil, errors.New("error in creating an http request")
}
@@ -76,8 +69,6 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
// If content length is non 0, then the header will be included
if !includeHeader("Content-Length", signedHdrs) {
httpReq.ContentLength = 0
} else {
httpReq.ContentLength = contentLength
}
// Set the Host header
@@ -140,12 +131,3 @@ func includeHeader(hdr string, signedHdrs []string) bool {
}
return false
}
func IsBigDataAction(ctx *fiber.Ctx) bool {
if ctx.Method() == http.MethodPut && len(strings.Split(ctx.Path(), "/")) >= 3 {
if !ctx.Request().URI().QueryArgs().Has("tagging") && ctx.Get("X-Amz-Copy-Source") == "" && !ctx.Request().URI().QueryArgs().Has("acl") {
return true
}
}
return false
}

View File

@@ -55,7 +55,7 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := createHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"}, 0)
got, err := CreateHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"})
if (err != nil) != tt.wantErr {
t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr)
return

View File

@@ -1,358 +0,0 @@
// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package s3select
import (
"bufio"
"context"
"encoding/binary"
"encoding/xml"
"fmt"
"hash/crc32"
"sync"
"sync/atomic"
"time"
)
// Protocol definition for messages can be found here:
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTSelectObjectAppendix.html
var (
// From ptotocol def:
// Enum indicating the header value type.
// For Amazon S3 Select, this is always 7.
headerValueType = byte(7)
)
func intToTwoBytes(i int) []byte {
return []byte{byte(i >> 8), byte(i)}
}
func generateHeader(messages ...string) []byte {
var header []byte
for i, message := range messages {
if i%2 == 1 {
header = append(header, headerValueType)
header = append(header, intToTwoBytes(len(message))...)
} else {
header = append(header, byte(len(message)))
}
header = append(header, message...)
}
return header
}
func generateOctetHeader(message string) []byte {
return generateHeader(
":message-type",
"event",
":content-type",
"application/octet-stream",
":event-type",
message)
}
func generateTextHeader(message string) []byte {
return generateHeader(
":message-type",
"event",
":content-type",
"text/xml",
":event-type",
message)
}
func generateNoContentHeader(message string) []byte {
return generateHeader(
":message-type",
"event",
":event-type",
message)
}
const (
// 4 bytes total byte len +
// 4 bytes headers bytes len +
// 4 bytes prelude CRC
preludeLen = 12
// CRC is uint32
msgCrcLen = 4
)
var (
recordsHeader = generateOctetHeader("Records")
continuationHeader = generateNoContentHeader("Cont")
continuationMessage = genMessage(continuationHeader, []byte{})
progressHeader = generateTextHeader("Progress")
statsHeader = generateTextHeader("Stats")
endHeader = generateNoContentHeader("End")
endMessage = genMessage(endHeader, []byte{})
)
func uintToBytes(n uint32) []byte {
b := make([]byte, 4)
binary.BigEndian.PutUint32(b, n)
return b
}
func generatePrelude(msgLen int, headerLen int) []byte {
prelude := make([]byte, 0, preludeLen)
// 4 bytes total byte len
prelude = append(prelude, uintToBytes(uint32(msgLen+headerLen+preludeLen+msgCrcLen))...)
// 4 bytes headers bytes len
prelude = append(prelude, uintToBytes(uint32(headerLen))...)
// 4 bytes prelude CRC
prelude = append(prelude, uintToBytes(crc32.ChecksumIEEE(prelude))...)
return prelude
}
const (
maxHeaderSize = 1024 * 1024
maxMessageSize = 5 * 1024 * 1024 * 1024
)
func genMessage(header, payload []byte) []byte {
var msg []byte
// below is always true since the size is validated
// in the send record
if len(header) <= maxHeaderSize && len(payload) <= maxMessageSize {
msglen := preludeLen + len(header) + len(payload) + msgCrcLen
msg = make([]byte, 0, msglen)
}
msg = append(msg, generatePrelude(len(payload), len(header))...)
msg = append(msg, header...)
msg = append(msg, payload...)
msg = append(msg, uintToBytes(crc32.ChecksumIEEE(msg))...)
return msg
}
func genRecordsMessage(payload []byte) []byte {
return genMessage(recordsHeader, payload)
}
type progress struct {
XMLName xml.Name `xml:"Progress"`
BytesScanned int64 `xml:"BytesScanned"`
BytesProcessed int64 `xml:"BytesProcessed"`
BytesReturned int64 `xml:"BytesReturned"`
}
func genProgressMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte {
progress := progress{
BytesScanned: bytesScanned,
BytesProcessed: bytesProcessed,
BytesReturned: bytesReturned,
}
xmlData, _ := xml.MarshalIndent(progress, "", " ")
payload := []byte(xml.Header + string(xmlData))
return genMessage(progressHeader, payload)
}
type stats struct {
XMLName xml.Name `xml:"Stats"`
BytesScanned int64 `xml:"BytesScanned"`
BytesProcessed int64 `xml:"BytesProcessed"`
BytesReturned int64 `xml:"BytesReturned"`
}
func genStatsMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte {
stats := stats{
BytesScanned: bytesScanned,
BytesProcessed: bytesProcessed,
BytesReturned: bytesReturned,
}
xmlData, _ := xml.MarshalIndent(stats, "", " ")
payload := []byte(xml.Header + string(xmlData))
return genMessage(statsHeader, payload)
}
func genErrorMessage(errorCode, errorMessage string) []byte {
return genMessage(generateHeader(
":error-code",
errorCode,
":error-message",
errorMessage,
":message-type",
"error",
), []byte{})
}
// GetProgress is a callback function that periodically retrieves the current
// values for the following if not nil. This is used to send Progress
// messages back to client.
// BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed).
// BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed).
type GetProgress func() (bytesScanned int64, bytesProcessed int64)
type MessageHandler struct {
sync.Mutex
ctx context.Context
cancel context.CancelFunc
writer *bufio.Writer
data chan []byte
getProgress GetProgress
stopCh chan bool
resetCh chan bool
bytesReturned int64
}
// NewMessageHandler creates a new MessageHandler instance and starts the event streaming
func NewMessageHandler(ctx context.Context, w *bufio.Writer, getProgressFunc GetProgress) *MessageHandler {
ctx, cancel := context.WithCancel(ctx)
mh := &MessageHandler{
ctx: ctx,
cancel: cancel,
writer: w,
data: make(chan []byte),
getProgress: getProgressFunc,
resetCh: make(chan bool),
stopCh: make(chan bool),
}
go mh.sendBackgroundMessages(mh.resetCh, mh.stopCh)
return mh
}
func (mh *MessageHandler) write(data []byte) error {
mh.Lock()
defer mh.Unlock()
mh.stopCh <- true
defer func() { mh.resetCh <- true }()
_, err := mh.writer.Write(data)
if err != nil {
return err
}
return mh.writer.Flush()
}
const (
continuationInterval = time.Second
progressInterval = time.Minute
)
func (mh *MessageHandler) sendBackgroundMessages(resetCh, stopCh <-chan bool) {
continuationTicker := time.NewTicker(continuationInterval)
defer continuationTicker.Stop()
var progressTicker *time.Ticker
var progressTickerChan <-chan time.Time
if mh.getProgress != nil {
progressTicker = time.NewTicker(progressInterval)
progressTickerChan = progressTicker.C
defer progressTicker.Stop()
}
Loop:
for {
select {
case <-mh.ctx.Done():
break Loop
case <-continuationTicker.C:
err := mh.write(continuationMessage)
if err != nil {
mh.cancel()
break Loop
}
case <-resetCh:
continuationTicker.Reset(continuationInterval)
case <-stopCh:
continuationTicker.Stop()
case <-progressTickerChan:
var bytesScanned, bytesProcessed int64
if mh.getProgress != nil {
bytesScanned, bytesProcessed = mh.getProgress()
}
bytesReturned := atomic.LoadInt64(&mh.bytesReturned)
err := mh.write(genProgressMessage(bytesScanned, bytesProcessed, bytesReturned))
if err != nil {
mh.cancel()
break Loop
}
}
}
}
// SendRecord sends a single Records message
func (mh *MessageHandler) SendRecord(payload []byte) error {
if mh.ctx.Err() != nil {
return mh.ctx.Err()
}
if len(payload) > maxMessageSize {
return fmt.Errorf("record max size exceeded")
}
err := mh.write(genRecordsMessage(payload))
if err != nil {
return err
}
atomic.AddInt64(&mh.bytesReturned, int64(len(payload)))
return nil
}
// Finish terminates message stream with Stats and End message
// generates stats and end message using function args based on:
// BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed).
// BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed).
func (mh *MessageHandler) Finish(bytesScanned, bytesProcessed int64) error {
if mh.ctx.Err() != nil {
return mh.ctx.Err()
}
bytesReturned := atomic.LoadInt64(&mh.bytesReturned)
err := mh.write(genStatsMessage(bytesScanned, bytesProcessed, bytesReturned))
if err != nil {
return err
}
err = mh.write(endMessage)
if err != nil {
return err
}
mh.cancel()
return nil
}
// FinishWithError terminates event stream with error
func (mh *MessageHandler) FinishWithError(errorCode, errorMessage string) error {
if mh.ctx.Err() != nil {
return mh.ctx.Err()
}
err := mh.write(genErrorMessage(errorCode, errorMessage))
if err != nil {
return err
}
mh.cancel()
return nil
}