mirror of
https://github.com/versity/versitygw.git
synced 2026-01-25 04:22:02 +00:00
Compare commits
1 Commits
v0.11
...
proxy-test
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
27a8aa66d9 |
@@ -1,46 +0,0 @@
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
cmd/versitygw/versitygw
|
||||
/versitygw
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
|
||||
# ignore IntelliJ directories
|
||||
.idea
|
||||
|
||||
# auto generated VERSION file
|
||||
VERSION
|
||||
|
||||
# build output
|
||||
/versitygw.spec
|
||||
/versitygw.spec.in
|
||||
*.tar
|
||||
*.tar.gz
|
||||
**/rand.data
|
||||
/profile.txt
|
||||
|
||||
dist/
|
||||
|
||||
# Release config files
|
||||
/.github
|
||||
|
||||
# Docker configuration files
|
||||
*Dockerfile
|
||||
/docker-compose.yml
|
||||
|
||||
# read files
|
||||
/LICENSE
|
||||
/NOTICE
|
||||
/CODE_OF_CONDUCT.md
|
||||
/README.md
|
||||
6
.env.dev
6
.env.dev
@@ -1,6 +0,0 @@
|
||||
POSIX_PORT=
|
||||
PROXY_PORT=
|
||||
ACCESS_KEY_ID=
|
||||
SECRET_ACCESS_KEY=
|
||||
IAM_DIR=
|
||||
SETUP_DIR=
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -25,9 +25,6 @@ go.work
|
||||
# ignore IntelliJ directories
|
||||
.idea
|
||||
|
||||
# ignore VS code directories
|
||||
.vscode
|
||||
|
||||
# auto generated VERSION file
|
||||
VERSION
|
||||
|
||||
|
||||
@@ -9,10 +9,6 @@ builds:
|
||||
# windows is untested, we can start doing windows releases
|
||||
# if someone is interested in taking on testing
|
||||
# - windows
|
||||
env:
|
||||
# disable cgo to fix glibc issues: https://github.com/golang/go/issues/58550
|
||||
# once we need to enable this, we will need to do per distro releases
|
||||
- CGO_ENABLED=0
|
||||
main: ./cmd/versitygw
|
||||
binary: ./cmd/versitygw
|
||||
id: versitygw
|
||||
|
||||
24
Dockerfile
24
Dockerfile
@@ -1,24 +0,0 @@
|
||||
FROM golang:1.20-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY ./ ./
|
||||
|
||||
WORKDIR /app/cmd/versitygw
|
||||
RUN go build -o versitygw
|
||||
|
||||
FROM alpine:latest
|
||||
|
||||
# These arguments can be overriden when building the image
|
||||
ARG IAM_DIR=/tmp/vgw
|
||||
ARG SETUP_DIR=/tmp/vgw
|
||||
|
||||
RUN mkdir -p $IAM_DIR
|
||||
RUN mkdir -p $SETUP_DIR
|
||||
|
||||
COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
|
||||
|
||||
ENTRYPOINT [ "/app/versitygw" ]
|
||||
@@ -1,17 +0,0 @@
|
||||
FROM golang:1.20
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY go.mod ./
|
||||
RUN go mod download
|
||||
|
||||
COPY ./ ./
|
||||
|
||||
ARG IAM_DIR=/tmp/vgw
|
||||
ARG SETUP_DIR=/tmp/vgw
|
||||
|
||||
RUN mkdir -p $IAM_DIR
|
||||
RUN mkdir -p $SETUP_DIR
|
||||
|
||||
RUN go get github.com/githubnemo/CompileDaemon
|
||||
RUN go install github.com/githubnemo/CompileDaemon
|
||||
15
Makefile
15
Makefile
@@ -74,18 +74,3 @@ dist: $(BIN).spec
|
||||
rm -f VERSION
|
||||
rm -f $(BIN).spec
|
||||
gzip -f $(TARFILE)
|
||||
|
||||
# Creates and runs S3 gateway instance in a docker container
|
||||
.PHONY: up-posix
|
||||
up-posix:
|
||||
docker compose --env-file .env.dev up posix
|
||||
|
||||
# Creates and runs S3 gateway proxy instance in a docker container
|
||||
.PHONY: up-proxy
|
||||
up-proxy:
|
||||
docker compose --env-file .env.dev up proxy
|
||||
|
||||
# Creates and runs both S3 gateway and proxy server instances in docker containers
|
||||
.PHONY: up-app
|
||||
up-app:
|
||||
docker compose --env-file .env.dev up
|
||||
|
||||
40
auth/iam.go
40
auth/iam.go
@@ -16,7 +16,6 @@ package auth
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -44,25 +43,18 @@ type IAMService interface {
|
||||
var ErrNoSuchUser = errors.New("user not found")
|
||||
|
||||
type Opts struct {
|
||||
Dir string
|
||||
LDAPServerURL string
|
||||
LDAPBindDN string
|
||||
LDAPPassword string
|
||||
LDAPQueryBase string
|
||||
LDAPObjClasses string
|
||||
LDAPAccessAtr string
|
||||
LDAPSecretAtr string
|
||||
LDAPRoleAtr string
|
||||
S3Access string
|
||||
S3Secret string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3Endpoint string
|
||||
S3DisableSSlVerfiy bool
|
||||
S3Debug bool
|
||||
CacheDisable bool
|
||||
CacheTTL int
|
||||
CachePrune int
|
||||
Dir string
|
||||
LDAPServerURL string
|
||||
LDAPBindDN string
|
||||
LDAPPassword string
|
||||
LDAPQueryBase string
|
||||
LDAPObjClasses string
|
||||
LDAPAccessAtr string
|
||||
LDAPSecretAtr string
|
||||
LDAPRoleAtr string
|
||||
CacheDisable bool
|
||||
CacheTTL int
|
||||
CachePrune int
|
||||
}
|
||||
|
||||
func New(o *Opts) (IAMService, error) {
|
||||
@@ -72,20 +64,12 @@ func New(o *Opts) (IAMService, error) {
|
||||
switch {
|
||||
case o.Dir != "":
|
||||
svc, err = NewInternal(o.Dir)
|
||||
fmt.Printf("initializing internal IAM with %q\n", o.Dir)
|
||||
case o.LDAPServerURL != "":
|
||||
svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
|
||||
o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
|
||||
o.LDAPObjClasses)
|
||||
fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
|
||||
case o.S3Endpoint != "":
|
||||
svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
|
||||
o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
|
||||
fmt.Printf("initializing S3 IAM with '%v/%v'\n",
|
||||
o.S3Endpoint, o.S3Bucket)
|
||||
default:
|
||||
// if no iam options selected, default to the single user mode
|
||||
fmt.Println("No IAM service configured, enabling single account mode")
|
||||
return IAMServiceSingle{}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,263 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package auth
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
)
|
||||
|
||||
// IAMServiceS3 stores user accounts in an S3 object
|
||||
// The endpoint, credentials, bucket, and region are provided
|
||||
// from cli configuration.
|
||||
// The object format and name is the same as the internal IAM service:
|
||||
// coming from iAMConfig and iamFile in iam_internal.
|
||||
|
||||
type IAMServiceS3 struct {
|
||||
access string
|
||||
secret string
|
||||
region string
|
||||
bucket string
|
||||
endpoint string
|
||||
sslSkipVerify bool
|
||||
debug bool
|
||||
client *s3.Client
|
||||
}
|
||||
|
||||
var _ IAMService = &IAMServiceS3{}
|
||||
|
||||
func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
|
||||
if access == "" {
|
||||
return nil, fmt.Errorf("must provide s3 IAM service access key")
|
||||
}
|
||||
if secret == "" {
|
||||
return nil, fmt.Errorf("must provide s3 IAM service secret key")
|
||||
}
|
||||
if region == "" {
|
||||
return nil, fmt.Errorf("must provide s3 IAM service region")
|
||||
}
|
||||
if bucket == "" {
|
||||
return nil, fmt.Errorf("must provide s3 IAM service bucket")
|
||||
}
|
||||
if endpoint == "" {
|
||||
return nil, fmt.Errorf("must provide s3 IAM service endpoint")
|
||||
}
|
||||
|
||||
i := &IAMServiceS3{
|
||||
access: access,
|
||||
secret: secret,
|
||||
region: region,
|
||||
bucket: bucket,
|
||||
endpoint: endpoint,
|
||||
sslSkipVerify: sslSkipVerify,
|
||||
debug: debug,
|
||||
}
|
||||
|
||||
cfg, err := i.getConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init s3 IAM: %v", err)
|
||||
}
|
||||
|
||||
i.client = s3.NewFromConfig(cfg)
|
||||
return i, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) CreateAccount(account Account) error {
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, ok := conf.AccessAccounts[account.Access]
|
||||
if ok {
|
||||
return fmt.Errorf("account already exists")
|
||||
}
|
||||
conf.AccessAccounts[account.Access] = account
|
||||
|
||||
return s.storeAccts(conf)
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return Account{}, err
|
||||
}
|
||||
|
||||
acct, ok := conf.AccessAccounts[access]
|
||||
if !ok {
|
||||
return Account{}, ErrNoSuchUser
|
||||
}
|
||||
|
||||
return acct, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) DeleteUserAccount(access string) error {
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, ok := conf.AccessAccounts[access]
|
||||
if !ok {
|
||||
return fmt.Errorf("account does not exist")
|
||||
}
|
||||
delete(conf.AccessAccounts, access)
|
||||
|
||||
return s.storeAccts(conf)
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
|
||||
conf, err := s.getAccounts()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(conf.AccessAccounts))
|
||||
for k := range conf.AccessAccounts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var accs []Account
|
||||
for _, k := range keys {
|
||||
accs = append(accs, Account{
|
||||
Access: k,
|
||||
Secret: conf.AccessAccounts[k].Secret,
|
||||
Role: conf.AccessAccounts[k].Role,
|
||||
UserID: conf.AccessAccounts[k].UserID,
|
||||
GroupID: conf.AccessAccounts[k].GroupID,
|
||||
ProjectID: conf.AccessAccounts[k].ProjectID,
|
||||
})
|
||||
}
|
||||
|
||||
return accs, nil
|
||||
}
|
||||
|
||||
// ResolveEndpoint is used for on prem or non-aws endpoints
|
||||
func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
PartitionID: "aws",
|
||||
URL: s.endpoint,
|
||||
SigningRegion: s.region,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) Shutdown() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) getConfig() (aws.Config, error) {
|
||||
creds := credentials.NewStaticCredentialsProvider(s.access, s.secret, "")
|
||||
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
|
||||
}
|
||||
client := &http.Client{Transport: tr}
|
||||
|
||||
opts := []func(*config.LoadOptions) error{
|
||||
config.WithRegion(s.region),
|
||||
config.WithCredentialsProvider(creds),
|
||||
config.WithHTTPClient(client),
|
||||
}
|
||||
|
||||
if s.endpoint != "" {
|
||||
opts = append(opts,
|
||||
config.WithEndpointResolverWithOptions(s))
|
||||
}
|
||||
|
||||
if s.debug {
|
||||
opts = append(opts,
|
||||
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
|
||||
}
|
||||
|
||||
return config.LoadDefaultConfig(context.Background(), opts...)
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
|
||||
obj := iamFile
|
||||
|
||||
out, err := s.client.GetObject(context.Background(), &s3.GetObjectInput{
|
||||
Bucket: &s.bucket,
|
||||
Key: &obj,
|
||||
})
|
||||
if err != nil {
|
||||
// if the error is object not exists,
|
||||
// init empty accounts stuct and return that
|
||||
var nsk *types.NoSuchKey
|
||||
if errors.As(err, &nsk) {
|
||||
return iAMConfig{}, nil
|
||||
}
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
if apiErr.ErrorCode() == "NotFound" {
|
||||
return iAMConfig{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// all other errors, return the error
|
||||
return iAMConfig{}, fmt.Errorf("get %v: %w", obj, err)
|
||||
}
|
||||
|
||||
defer out.Body.Close()
|
||||
|
||||
b, err := io.ReadAll(out.Body)
|
||||
if err != nil {
|
||||
return iAMConfig{}, fmt.Errorf("read %v: %w", obj, err)
|
||||
}
|
||||
|
||||
conf, err := parseIAM(b)
|
||||
if err != nil {
|
||||
return iAMConfig{}, fmt.Errorf("parse iam data: %w", err)
|
||||
}
|
||||
|
||||
return conf, nil
|
||||
}
|
||||
|
||||
func (s *IAMServiceS3) storeAccts(conf iAMConfig) error {
|
||||
b, err := json.Marshal(conf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to serialize iam: %w", err)
|
||||
}
|
||||
|
||||
obj := iamFile
|
||||
uploader := manager.NewUploader(s.client)
|
||||
upinfo := &s3.PutObjectInput{
|
||||
Body: bytes.NewReader(b),
|
||||
Bucket: &s.bucket,
|
||||
Key: &obj,
|
||||
}
|
||||
_, err = uploader.Upload(context.Background(), upinfo)
|
||||
if err != nil {
|
||||
return fmt.Errorf("store accounts in %v: %w", iamFile, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -15,7 +15,6 @@
|
||||
package backend
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -23,7 +22,6 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
"github.com/versity/versitygw/s3select"
|
||||
)
|
||||
|
||||
//go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
|
||||
@@ -63,7 +61,7 @@ type Backend interface {
|
||||
|
||||
// special case object operations
|
||||
RestoreObject(context.Context, *s3.RestoreObjectInput) error
|
||||
SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
|
||||
SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
|
||||
|
||||
// object tags operations
|
||||
GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
|
||||
@@ -164,19 +162,8 @@ func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) e
|
||||
func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
|
||||
return s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
|
||||
return func(w *bufio.Writer) {
|
||||
var getProgress s3select.GetProgress
|
||||
progress := input.RequestProgress
|
||||
if progress != nil && *progress.Enabled {
|
||||
getProgress = func() (bytesScanned int64, bytesProcessed int64) {
|
||||
return -1, -1
|
||||
}
|
||||
}
|
||||
mh := s3select.NewMessageHandler(ctx, w, getProgress)
|
||||
apiErr := s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
mh.FinishWithError(apiErr.Code, apiErr.Description)
|
||||
}
|
||||
func (BackendUnsupported) SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
|
||||
return s3response.SelectObjectContentResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {
|
||||
|
||||
@@ -146,10 +146,6 @@ func (p *Posix) ListBuckets(_ context.Context, owner string, isAdmin bool) (s3re
|
||||
}
|
||||
|
||||
func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
|
||||
_, err := os.Lstat(*input.Bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
@@ -162,10 +158,6 @@ func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.He
|
||||
}
|
||||
|
||||
func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput) error {
|
||||
if input.Bucket == nil {
|
||||
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
|
||||
bucket := *input.Bucket
|
||||
owner := string(input.ObjectOwnership)
|
||||
|
||||
@@ -191,10 +183,6 @@ func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput) err
|
||||
}
|
||||
|
||||
func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) error {
|
||||
if input.Bucket == nil {
|
||||
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
|
||||
names, err := os.ReadDir(*input.Bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
@@ -224,13 +212,6 @@ func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) err
|
||||
}
|
||||
|
||||
func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
if mpu.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
if mpu.Key == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
bucket := *mpu.Bucket
|
||||
object := *mpu.Key
|
||||
|
||||
@@ -288,19 +269,6 @@ func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipart
|
||||
}
|
||||
|
||||
func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
if input.Key == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if input.UploadId == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload)
|
||||
}
|
||||
if input.MultipartUpload == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
uploadID := *input.UploadId
|
||||
@@ -326,7 +294,7 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
|
||||
partsize := int64(0)
|
||||
var totalsize int64
|
||||
for i, p := range parts {
|
||||
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))
|
||||
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))
|
||||
fi, err := os.Lstat(partPath)
|
||||
if err != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
|
||||
@@ -358,7 +326,7 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
|
||||
defer f.cleanup()
|
||||
|
||||
for _, p := range parts {
|
||||
pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)))
|
||||
pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err)
|
||||
}
|
||||
@@ -544,16 +512,6 @@ func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
|
||||
}
|
||||
|
||||
func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUploadInput) error {
|
||||
if mpu.Bucket == nil {
|
||||
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
if mpu.Key == nil {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if mpu.UploadId == nil {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchUpload)
|
||||
}
|
||||
|
||||
bucket := *mpu.Bucket
|
||||
object := *mpu.Key
|
||||
uploadID := *mpu.UploadId
|
||||
@@ -584,12 +542,6 @@ func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUp
|
||||
}
|
||||
|
||||
func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
|
||||
var lmu s3response.ListMultipartUploadsResult
|
||||
|
||||
if mpu.Bucket == nil {
|
||||
return lmu, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
|
||||
bucket := *mpu.Bucket
|
||||
var delimiter string
|
||||
if mpu.Delimiter != nil {
|
||||
@@ -600,6 +552,8 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
|
||||
prefix = *mpu.Prefix
|
||||
}
|
||||
|
||||
var lmu s3response.ListMultipartUploadsResult
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return lmu, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
@@ -672,16 +626,12 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
|
||||
}
|
||||
}
|
||||
|
||||
maxUploads := 0
|
||||
if mpu.MaxUploads != nil {
|
||||
maxUploads = int(*mpu.MaxUploads)
|
||||
}
|
||||
if (uploadIDMarker != "" && !uploadIdMarkerFound) || (keyMarker != "" && keyMarkerInd == -1) {
|
||||
return s3response.ListMultipartUploadsResult{
|
||||
Bucket: bucket,
|
||||
Delimiter: delimiter,
|
||||
KeyMarker: keyMarker,
|
||||
MaxUploads: maxUploads,
|
||||
MaxUploads: int(mpu.MaxUploads),
|
||||
Prefix: prefix,
|
||||
UploadIDMarker: uploadIDMarker,
|
||||
Uploads: []s3response.Upload{},
|
||||
@@ -693,18 +643,18 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
|
||||
})
|
||||
|
||||
for i := keyMarkerInd + 1; i < len(uploads); i++ {
|
||||
if maxUploads == 0 {
|
||||
if mpu.MaxUploads == 0 {
|
||||
break
|
||||
}
|
||||
if keyMarker != "" && uploadIDMarker != "" && uploads[i].UploadID < uploadIDMarker {
|
||||
continue
|
||||
}
|
||||
if i != len(uploads)-1 && len(resultUpds) == maxUploads {
|
||||
if i != len(uploads)-1 && len(resultUpds) == int(mpu.MaxUploads) {
|
||||
return s3response.ListMultipartUploadsResult{
|
||||
Bucket: bucket,
|
||||
Delimiter: delimiter,
|
||||
KeyMarker: keyMarker,
|
||||
MaxUploads: maxUploads,
|
||||
MaxUploads: int(mpu.MaxUploads),
|
||||
NextKeyMarker: resultUpds[i-1].Key,
|
||||
NextUploadIDMarker: resultUpds[i-1].UploadID,
|
||||
IsTruncated: true,
|
||||
@@ -721,7 +671,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
|
||||
Bucket: bucket,
|
||||
Delimiter: delimiter,
|
||||
KeyMarker: keyMarker,
|
||||
MaxUploads: maxUploads,
|
||||
MaxUploads: int(mpu.MaxUploads),
|
||||
Prefix: prefix,
|
||||
UploadIDMarker: uploadIDMarker,
|
||||
Uploads: resultUpds,
|
||||
@@ -729,29 +679,13 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
|
||||
}
|
||||
|
||||
func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
|
||||
var lpr s3response.ListPartsResult
|
||||
|
||||
if input.Bucket == nil {
|
||||
return lpr, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
if input.Key == nil {
|
||||
return lpr, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if input.UploadId == nil {
|
||||
return lpr, s3err.GetAPIError(s3err.ErrNoSuchUpload)
|
||||
}
|
||||
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
uploadID := *input.UploadId
|
||||
stringMarker := ""
|
||||
if input.PartNumberMarker != nil {
|
||||
stringMarker = *input.PartNumberMarker
|
||||
}
|
||||
maxParts := 0
|
||||
if input.MaxParts != nil {
|
||||
maxParts = int(*input.MaxParts)
|
||||
}
|
||||
stringMarker := *input.PartNumberMarker
|
||||
maxParts := int(input.MaxParts)
|
||||
|
||||
var lpr s3response.ListPartsResult
|
||||
|
||||
var partNumberMarker int
|
||||
if stringMarker != "" {
|
||||
@@ -843,21 +777,11 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon
|
||||
}
|
||||
|
||||
func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string, error) {
|
||||
if input.Bucket == nil {
|
||||
return "", s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
if input.Key == nil {
|
||||
return "", s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
uploadID := *input.UploadId
|
||||
part := input.PartNumber
|
||||
length := int64(0)
|
||||
if input.ContentLength != nil {
|
||||
length = *input.ContentLength
|
||||
}
|
||||
length := input.ContentLength
|
||||
r := input.Body
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
@@ -879,7 +803,7 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
|
||||
return "", fmt.Errorf("stat uploadid: %w", err)
|
||||
}
|
||||
|
||||
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part))
|
||||
partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", part))
|
||||
|
||||
f, err := openTmpFile(filepath.Join(bucket, objdir),
|
||||
bucket, partPath, length)
|
||||
@@ -909,13 +833,6 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
|
||||
}
|
||||
|
||||
func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
|
||||
if upi.Bucket == nil {
|
||||
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
if upi.Key == nil {
|
||||
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
_, err := os.Stat(*upi.Bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
@@ -935,7 +852,7 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
|
||||
return s3response.CopyObjectResult{}, fmt.Errorf("stat uploadid: %w", err)
|
||||
}
|
||||
|
||||
partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", *upi.PartNumber))
|
||||
partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", upi.PartNumber))
|
||||
|
||||
substrs := strings.SplitN(*upi.CopySource, "/", 2)
|
||||
if len(substrs) != 2 {
|
||||
@@ -1021,13 +938,6 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
|
||||
}
|
||||
|
||||
func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) {
|
||||
if po.Bucket == nil {
|
||||
return "", s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
if po.Key == nil {
|
||||
return "", s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
tagsStr := getString(po.Tagging)
|
||||
tags := make(map[string]string)
|
||||
_, err := os.Stat(*po.Bucket)
|
||||
@@ -1054,13 +964,9 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
|
||||
|
||||
name := filepath.Join(*po.Bucket, *po.Key)
|
||||
|
||||
contentLength := int64(0)
|
||||
if po.ContentLength != nil {
|
||||
contentLength = *po.ContentLength
|
||||
}
|
||||
if strings.HasSuffix(*po.Key, "/") {
|
||||
// object is directory
|
||||
if contentLength != 0 {
|
||||
if po.ContentLength != 0 {
|
||||
// posix directories can't contain data, send error
|
||||
// if reuests has a data payload associated with a
|
||||
// directory object
|
||||
@@ -1089,7 +995,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
|
||||
}
|
||||
|
||||
f, err := openTmpFile(filepath.Join(*po.Bucket, metaTmpDir),
|
||||
*po.Bucket, *po.Key, contentLength)
|
||||
*po.Bucket, *po.Key, po.ContentLength)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("open temp file: %w", err)
|
||||
}
|
||||
@@ -1133,13 +1039,6 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
|
||||
}
|
||||
|
||||
func (p *Posix) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) error {
|
||||
if input.Bucket == nil {
|
||||
return s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
if input.Key == nil {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
|
||||
@@ -1232,16 +1131,6 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput)
|
||||
}
|
||||
|
||||
func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
if input.Key == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if input.Range == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRange)
|
||||
}
|
||||
|
||||
bucket := *input.Bucket
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
@@ -1303,17 +1192,15 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
|
||||
return nil, fmt.Errorf("get object tags: %w", err)
|
||||
}
|
||||
|
||||
tagCount := int32(len(tags))
|
||||
|
||||
return &s3.GetObjectOutput{
|
||||
AcceptRanges: &acceptRange,
|
||||
ContentLength: &length,
|
||||
ContentLength: length,
|
||||
ContentEncoding: &contentEncoding,
|
||||
ContentType: &contentType,
|
||||
ETag: &etag,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Metadata: userMetaData,
|
||||
TagCount: &tagCount,
|
||||
TagCount: int32(len(tags)),
|
||||
ContentRange: &contentRange,
|
||||
}, nil
|
||||
}
|
||||
@@ -1348,28 +1235,20 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
|
||||
return nil, fmt.Errorf("get object tags: %w", err)
|
||||
}
|
||||
|
||||
tagCount := int32(len(tags))
|
||||
|
||||
return &s3.GetObjectOutput{
|
||||
AcceptRanges: &acceptRange,
|
||||
ContentLength: &length,
|
||||
ContentLength: length,
|
||||
ContentEncoding: &contentEncoding,
|
||||
ContentType: &contentType,
|
||||
ETag: &etag,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Metadata: userMetaData,
|
||||
TagCount: &tagCount,
|
||||
TagCount: int32(len(tags)),
|
||||
ContentRange: &contentRange,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
if input.Key == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
|
||||
@@ -1399,10 +1278,8 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He
|
||||
etag = ""
|
||||
}
|
||||
|
||||
size := fi.Size()
|
||||
|
||||
return &s3.HeadObjectOutput{
|
||||
ContentLength: &size,
|
||||
ContentLength: fi.Size(),
|
||||
ContentType: &contentType,
|
||||
ContentEncoding: &contentEncoding,
|
||||
ETag: &etag,
|
||||
@@ -1412,18 +1289,6 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He
|
||||
}
|
||||
|
||||
func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
if input.Key == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
|
||||
}
|
||||
if input.CopySource == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource)
|
||||
}
|
||||
if input.ExpectedBucketOwner == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
|
||||
}
|
||||
srcBucket, srcObject, ok := strings.Cut(*input.CopySource, "/")
|
||||
if !ok {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource)
|
||||
@@ -1496,16 +1361,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := fInfo.Size()
|
||||
|
||||
etag, err := p.PutObject(ctx,
|
||||
&s3.PutObjectInput{
|
||||
Bucket: &dstBucket,
|
||||
Key: &dstObject,
|
||||
Body: f,
|
||||
ContentLength: &contentLength,
|
||||
Metadata: meta,
|
||||
})
|
||||
etag, err := p.PutObject(ctx, &s3.PutObjectInput{Bucket: &dstBucket, Key: &dstObject, Body: f, ContentLength: fInfo.Size(), Metadata: meta})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1524,26 +1380,11 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
|
||||
}
|
||||
|
||||
func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
bucket := *input.Bucket
|
||||
prefix := ""
|
||||
if input.Prefix != nil {
|
||||
prefix = *input.Prefix
|
||||
}
|
||||
marker := ""
|
||||
if input.Marker != nil {
|
||||
marker = *input.Marker
|
||||
}
|
||||
delim := ""
|
||||
if input.Delimiter != nil {
|
||||
delim = *input.Delimiter
|
||||
}
|
||||
maxkeys := int32(0)
|
||||
if input.MaxKeys != nil {
|
||||
maxkeys = *input.MaxKeys
|
||||
}
|
||||
prefix := *input.Prefix
|
||||
marker := *input.Marker
|
||||
delim := *input.Delimiter
|
||||
maxkeys := input.MaxKeys
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
@@ -1564,9 +1405,9 @@ func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.
|
||||
CommonPrefixes: results.CommonPrefixes,
|
||||
Contents: results.Objects,
|
||||
Delimiter: &delim,
|
||||
IsTruncated: &results.Truncated,
|
||||
IsTruncated: results.Truncated,
|
||||
Marker: &marker,
|
||||
MaxKeys: &maxkeys,
|
||||
MaxKeys: maxkeys,
|
||||
Name: &bucket,
|
||||
NextMarker: &results.NextMarker,
|
||||
Prefix: &prefix,
|
||||
@@ -1625,38 +1466,21 @@ func fileToObj(bucket string) backend.GetObjFunc {
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
size := fi.Size()
|
||||
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Size: &size,
|
||||
Size: fi.Size(),
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
bucket := *input.Bucket
|
||||
prefix := ""
|
||||
if input.Prefix != nil {
|
||||
prefix = *input.Prefix
|
||||
}
|
||||
marker := ""
|
||||
if input.ContinuationToken != nil {
|
||||
marker = *input.ContinuationToken
|
||||
}
|
||||
delim := ""
|
||||
if input.Delimiter != nil {
|
||||
delim = *input.Delimiter
|
||||
}
|
||||
maxkeys := int32(0)
|
||||
if input.MaxKeys != nil {
|
||||
maxkeys = *input.MaxKeys
|
||||
}
|
||||
prefix := *input.Prefix
|
||||
marker := *input.ContinuationToken
|
||||
delim := *input.Delimiter
|
||||
maxkeys := input.MaxKeys
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
@@ -1667,25 +1491,23 @@ func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (
|
||||
}
|
||||
|
||||
fileSystem := os.DirFS(bucket)
|
||||
results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
|
||||
results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
|
||||
fileToObj(bucket), []string{metaTmpDir})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("walk %v: %w", bucket, err)
|
||||
}
|
||||
|
||||
count := int32(len(results.Objects))
|
||||
|
||||
return &s3.ListObjectsV2Output{
|
||||
CommonPrefixes: results.CommonPrefixes,
|
||||
Contents: results.Objects,
|
||||
Delimiter: &delim,
|
||||
IsTruncated: &results.Truncated,
|
||||
IsTruncated: results.Truncated,
|
||||
ContinuationToken: &marker,
|
||||
MaxKeys: &maxkeys,
|
||||
MaxKeys: int32(maxkeys),
|
||||
Name: &bucket,
|
||||
NextContinuationToken: &results.NextMarker,
|
||||
Prefix: &prefix,
|
||||
KeyCount: &count,
|
||||
KeyCount: int32(len(results.Objects)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -1706,9 +1528,6 @@ func (p *Posix) PutBucketAcl(_ context.Context, bucket string, data []byte) erro
|
||||
}
|
||||
|
||||
func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
_, err := os.Stat(*input.Bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
|
||||
|
||||
@@ -17,6 +17,7 @@ package s3proxy
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
@@ -25,10 +26,16 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/smithy-go/middleware"
|
||||
"github.com/versity/versitygw/auth"
|
||||
)
|
||||
|
||||
func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
|
||||
cfg, err := s.getConfig(ctx, s.access, s.secret)
|
||||
func (s *S3be) getClientFromCtx(ctx context.Context) (*s3.Client, error) {
|
||||
acct, ok := ctx.Value("account").(auth.Account)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid account in context")
|
||||
}
|
||||
|
||||
cfg, err := s.getConfig(ctx, acct.Access, acct.Secret)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -36,7 +43,7 @@ func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
|
||||
return s3.NewFromConfig(cfg), nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
|
||||
func (s *S3be) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
|
||||
creds := credentials.NewStaticCredentialsProvider(access, secret, "")
|
||||
|
||||
tr := &http.Transport{
|
||||
@@ -69,7 +76,7 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
|
||||
}
|
||||
|
||||
// ResolveEndpoint is used for on prem or non-aws endpoints
|
||||
func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
func (s *S3be) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
PartitionID: "aws",
|
||||
URL: s.endpoint,
|
||||
|
||||
@@ -16,35 +16,21 @@ package s3proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/backend"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
type S3Proxy struct {
|
||||
type S3be struct {
|
||||
backend.BackendUnsupported
|
||||
|
||||
client *s3.Client
|
||||
|
||||
access string
|
||||
secret string
|
||||
endpoint string
|
||||
awsRegion string
|
||||
disableChecksum bool
|
||||
@@ -52,27 +38,23 @@ type S3Proxy struct {
|
||||
debug bool
|
||||
}
|
||||
|
||||
func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
|
||||
s := &S3Proxy{
|
||||
access: access,
|
||||
secret: secret,
|
||||
func New(endpoint, region string, disableChecksum, sslSkipVerify, debug bool) *S3be {
|
||||
return &S3be{
|
||||
endpoint: endpoint,
|
||||
awsRegion: region,
|
||||
disableChecksum: disableChecksum,
|
||||
sslSkipVerify: sslSkipVerify,
|
||||
debug: debug,
|
||||
}
|
||||
client, err := s.getClientWithCtx(context.Background())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.client = client
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
|
||||
output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{})
|
||||
err = handleError(err)
|
||||
func (s *S3be) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return s3response.ListAllMyBucketsResult{}, err
|
||||
}
|
||||
|
||||
output, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
|
||||
if err != nil {
|
||||
return s3response.ListAllMyBucketsResult{}, err
|
||||
}
|
||||
@@ -87,7 +69,8 @@ func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (
|
||||
|
||||
return s3response.ListAllMyBucketsResult{
|
||||
Owner: s3response.CanonicalUser{
|
||||
ID: *output.Owner.ID,
|
||||
ID: *output.Owner.ID,
|
||||
DisplayName: *output.Owner.DisplayName,
|
||||
},
|
||||
Buckets: s3response.ListAllMyBucketsList{
|
||||
Bucket: buckets,
|
||||
@@ -95,44 +78,74 @@ func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
|
||||
out, err := s.client.HeadBucket(ctx, input)
|
||||
err = handleError(err)
|
||||
func (s *S3be) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return out, err
|
||||
return client.HeadBucket(ctx, input)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) error {
|
||||
_, err := s.client.CreateBucket(ctx, input)
|
||||
return handleError(err)
|
||||
}
|
||||
func (s *S3be) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) error {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
|
||||
_, err := s.client.DeleteBucket(ctx, input)
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
out, err := s.client.CreateMultipartUpload(ctx, input)
|
||||
err = handleError(err)
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
out, err := s.client.CompleteMultipartUpload(ctx, input)
|
||||
err = handleError(err)
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (s *S3Proxy) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
|
||||
_, err := s.client.AbortMultipartUpload(ctx, input)
|
||||
err = handleError(err)
|
||||
_, err = client.CreateBucket(ctx, input)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
|
||||
output, err := s.client.ListMultipartUploads(ctx, input)
|
||||
err = handleError(err)
|
||||
func (s *S3be) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = client.DeleteBucket(ctx, input)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3be) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return client.CreateMultipartUpload(ctx, input)
|
||||
}
|
||||
|
||||
func (s *S3be) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return client.CompleteMultipartUpload(ctx, input)
|
||||
}
|
||||
|
||||
func (s *S3be) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = client.AbortMultipartUpload(ctx, input)
|
||||
return err
|
||||
}
|
||||
|
||||
const (
|
||||
iso8601Format = "20060102T150405Z"
|
||||
)
|
||||
|
||||
func (s *S3be) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return s3response.ListMultipartUploadsResult{}, err
|
||||
}
|
||||
|
||||
output, err := client.ListMultipartUploads(ctx, input)
|
||||
if err != nil {
|
||||
return s3response.ListMultipartUploadsResult{}, err
|
||||
}
|
||||
@@ -151,7 +164,7 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
|
||||
DisplayName: *u.Owner.DisplayName,
|
||||
},
|
||||
StorageClass: string(u.StorageClass),
|
||||
Initiated: u.Initiated.Format(backend.RFC3339TimeFormat),
|
||||
Initiated: u.Initiated.Format(iso8601Format),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -171,16 +184,20 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
|
||||
Delimiter: *output.Delimiter,
|
||||
Prefix: *output.Prefix,
|
||||
EncodingType: string(output.EncodingType),
|
||||
MaxUploads: int(*output.MaxUploads),
|
||||
IsTruncated: *output.IsTruncated,
|
||||
MaxUploads: int(output.MaxUploads),
|
||||
IsTruncated: output.IsTruncated,
|
||||
Uploads: uploads,
|
||||
CommonPrefixes: cps,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
|
||||
output, err := s.client.ListParts(ctx, input)
|
||||
err = handleError(err)
|
||||
func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return s3response.ListPartsResult{}, err
|
||||
}
|
||||
|
||||
output, err := client.ListParts(ctx, input)
|
||||
if err != nil {
|
||||
return s3response.ListPartsResult{}, err
|
||||
}
|
||||
@@ -188,10 +205,10 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
|
||||
var parts []s3response.Part
|
||||
for _, p := range output.Parts {
|
||||
parts = append(parts, s3response.Part{
|
||||
PartNumber: int(*p.PartNumber),
|
||||
LastModified: p.LastModified.Format(backend.RFC3339TimeFormat),
|
||||
PartNumber: int(p.PartNumber),
|
||||
LastModified: p.LastModified.Format(iso8601Format),
|
||||
ETag: *p.ETag,
|
||||
Size: *p.Size,
|
||||
Size: p.Size,
|
||||
})
|
||||
}
|
||||
pnm, err := strconv.Atoi(*output.PartNumberMarker)
|
||||
@@ -221,19 +238,19 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
|
||||
StorageClass: string(output.StorageClass),
|
||||
PartNumberMarker: pnm,
|
||||
NextPartNumberMarker: npmn,
|
||||
MaxParts: int(*output.MaxParts),
|
||||
IsTruncated: *output.IsTruncated,
|
||||
MaxParts: int(output.MaxParts),
|
||||
IsTruncated: output.IsTruncated,
|
||||
Parts: parts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
|
||||
// streaming backend is not seekable,
|
||||
// use unsigned payload for streaming ops
|
||||
output, err := s.client.UploadPart(ctx, input, s3.WithAPIOptions(
|
||||
v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
|
||||
))
|
||||
err = handleError(err)
|
||||
func (s *S3be) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
output, err := client.UploadPart(ctx, input)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -241,9 +258,13 @@ func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (et
|
||||
return *output.ETag, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
|
||||
output, err := s.client.UploadPartCopy(ctx, input)
|
||||
err = handleError(err)
|
||||
func (s *S3be) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectResult{}, err
|
||||
}
|
||||
|
||||
output, err := client.UploadPartCopy(ctx, input)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectResult{}, err
|
||||
}
|
||||
@@ -254,13 +275,13 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
|
||||
// streaming backend is not seekable,
|
||||
// use unsigned payload for streaming ops
|
||||
output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions(
|
||||
v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
|
||||
))
|
||||
err = handleError(err)
|
||||
func (s *S3be) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
output, err := client.PutObject(ctx, input)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -268,16 +289,22 @@ func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (stri
|
||||
return *output.ETag, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
out, err := s.client.HeadObject(ctx, input)
|
||||
err = handleError(err)
|
||||
func (s *S3be) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return out, err
|
||||
return client.HeadObject(ctx, input)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
|
||||
output, err := s.client.GetObject(ctx, input)
|
||||
err = handleError(err)
|
||||
func (s *S3be) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
output, err := client.GetObject(ctx, input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -291,46 +318,59 @@ func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
|
||||
out, err := s.client.GetObjectAttributes(ctx, input)
|
||||
err = handleError(err)
|
||||
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
|
||||
out, err := s.client.CopyObject(ctx, input)
|
||||
err = handleError(err)
|
||||
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
out, err := s.client.ListObjects(ctx, input)
|
||||
err = handleError(err)
|
||||
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
out, err := s.client.ListObjectsV2(ctx, input)
|
||||
err = handleError(err)
|
||||
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
|
||||
_, err := s.client.DeleteObject(ctx, input)
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
|
||||
if len(input.Delete.Objects) == 0 {
|
||||
input.Delete.Objects = []types.ObjectIdentifier{}
|
||||
func (s *S3be) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
output, err := s.client.DeleteObjects(ctx, input)
|
||||
err = handleError(err)
|
||||
return client.GetObjectAttributes(ctx, input)
|
||||
}
|
||||
|
||||
func (s *S3be) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return client.CopyObject(ctx, input)
|
||||
}
|
||||
|
||||
func (s *S3be) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return client.ListObjects(ctx, input)
|
||||
}
|
||||
|
||||
func (s *S3be) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return client.ListObjectsV2(ctx, input)
|
||||
}
|
||||
|
||||
func (s *S3be) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = client.DeleteObject(ctx, input)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3be) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return s3response.DeleteObjectsResult{}, err
|
||||
}
|
||||
|
||||
output, err := client.DeleteObjects(ctx, input)
|
||||
if err != nil {
|
||||
return s3response.DeleteObjectsResult{}, err
|
||||
}
|
||||
@@ -341,9 +381,13 @@ func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInpu
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
|
||||
output, err := s.client.GetBucketAcl(ctx, input)
|
||||
err = handleError(err)
|
||||
func (s *S3be) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
output, err := client.GetBucketAcl(ctx, input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -361,7 +405,12 @@ func (s *S3Proxy) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput)
|
||||
return json.Marshal(acl)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
|
||||
func (s S3be) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
acl, err := auth.ParseACL(data)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -378,21 +427,25 @@ func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte)
|
||||
}
|
||||
|
||||
for _, el := range acl.Grantees {
|
||||
acc := el.Access
|
||||
input.AccessControlPolicy.Grants = append(input.AccessControlPolicy.Grants, types.Grant{
|
||||
Permission: el.Permission,
|
||||
Grantee: &types.Grantee{
|
||||
ID: &acc,
|
||||
ID: &el.Access,
|
||||
Type: types.TypeCanonicalUser,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
_, err = s.client.PutBucketAcl(ctx, input)
|
||||
return handleError(err)
|
||||
_, err = client.PutBucketAcl(ctx, input)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3Proxy) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
|
||||
func (s *S3be) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tagging := &types.Tagging{
|
||||
TagSet: []types.Tag{},
|
||||
}
|
||||
@@ -403,20 +456,24 @@ func (s *S3Proxy) PutObjectTagging(ctx context.Context, bucket, object string, t
|
||||
})
|
||||
}
|
||||
|
||||
_, err := s.client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
|
||||
_, err = client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
Tagging: tagging,
|
||||
})
|
||||
return handleError(err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3Proxy) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
|
||||
output, err := s.client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
|
||||
func (s *S3be) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
output, err := client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
})
|
||||
err = handleError(err)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -429,106 +486,15 @@ func (s *S3Proxy) GetObjectTagging(ctx context.Context, bucket, object string) (
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
|
||||
_, err := s.client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
|
||||
func (s *S3be) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
|
||||
client, err := s.getClientFromCtx(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
|
||||
Bucket: &bucket,
|
||||
Key: &object,
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, newOwner), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256([]byte{})
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode > 300 {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return fmt.Errorf(string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3Proxy) ListBucketsAndOwners(ctx context.Context) ([]s3response.Bucket, error) {
|
||||
req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", s.endpoint), nil)
|
||||
if err != nil {
|
||||
return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
hashedPayload := sha256.Sum256([]byte{})
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", hexPayload)
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
|
||||
if signErr != nil {
|
||||
return []s3response.Bucket{}, fmt.Errorf("failed to sign the request: %w", err)
|
||||
}
|
||||
|
||||
client := http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return []s3response.Bucket{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var buckets []s3response.Bucket
|
||||
if err := json.Unmarshal(body, &buckets); err != nil {
|
||||
return []s3response.Bucket{}, err
|
||||
}
|
||||
|
||||
return buckets, nil
|
||||
}
|
||||
|
||||
func handleError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var ae smithy.APIError
|
||||
if errors.As(err, &ae) {
|
||||
apiErr := s3err.APIError{
|
||||
Code: ae.ErrorCode(),
|
||||
Description: ae.ErrorMessage(),
|
||||
}
|
||||
var re *awshttp.ResponseError
|
||||
if errors.As(err, &re) {
|
||||
apiErr.HTTPStatusCode = re.Response.StatusCode
|
||||
}
|
||||
return apiErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -415,10 +415,8 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := fi.Size()
|
||||
|
||||
return &s3.HeadObjectOutput{
|
||||
ContentLength: &contentLength,
|
||||
ContentLength: fi.Size(),
|
||||
ContentType: &contentType,
|
||||
ContentEncoding: &contentEncoding,
|
||||
ETag: &etag,
|
||||
@@ -509,17 +507,15 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
|
||||
return nil, fmt.Errorf("get object tags: %w", err)
|
||||
}
|
||||
|
||||
tagCount := int32(len(tags))
|
||||
|
||||
return &s3.GetObjectOutput{
|
||||
AcceptRanges: &acceptRange,
|
||||
ContentLength: &length,
|
||||
ContentLength: length,
|
||||
ContentEncoding: &contentEncoding,
|
||||
ContentType: &contentType,
|
||||
ETag: &etag,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Metadata: userMetaData,
|
||||
TagCount: &tagCount,
|
||||
TagCount: int32(len(tags)),
|
||||
StorageClass: types.StorageClassStandard,
|
||||
}, nil
|
||||
}
|
||||
@@ -546,26 +542,11 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error)
|
||||
}
|
||||
|
||||
func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
bucket := *input.Bucket
|
||||
prefix := ""
|
||||
if input.Prefix != nil {
|
||||
prefix = *input.Prefix
|
||||
}
|
||||
marker := ""
|
||||
if input.Marker != nil {
|
||||
marker = *input.Marker
|
||||
}
|
||||
delim := ""
|
||||
if input.Delimiter != nil {
|
||||
delim = *input.Delimiter
|
||||
}
|
||||
maxkeys := int32(0)
|
||||
if input.MaxKeys != nil {
|
||||
maxkeys = *input.MaxKeys
|
||||
}
|
||||
prefix := *input.Prefix
|
||||
marker := *input.Marker
|
||||
delim := *input.Delimiter
|
||||
maxkeys := input.MaxKeys
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
@@ -586,9 +567,9 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s
|
||||
CommonPrefixes: results.CommonPrefixes,
|
||||
Contents: results.Objects,
|
||||
Delimiter: &delim,
|
||||
IsTruncated: &results.Truncated,
|
||||
IsTruncated: results.Truncated,
|
||||
Marker: &marker,
|
||||
MaxKeys: &maxkeys,
|
||||
MaxKeys: maxkeys,
|
||||
Name: &bucket,
|
||||
NextMarker: &results.NextMarker,
|
||||
Prefix: &prefix,
|
||||
@@ -596,26 +577,11 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s
|
||||
}
|
||||
|
||||
func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
if input.Bucket == nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
|
||||
}
|
||||
bucket := *input.Bucket
|
||||
prefix := ""
|
||||
if input.Prefix != nil {
|
||||
prefix = *input.Prefix
|
||||
}
|
||||
marker := ""
|
||||
if input.ContinuationToken != nil {
|
||||
marker = *input.ContinuationToken
|
||||
}
|
||||
delim := ""
|
||||
if input.Delimiter != nil {
|
||||
delim = *input.Delimiter
|
||||
}
|
||||
maxkeys := int32(0)
|
||||
if input.MaxKeys != nil {
|
||||
maxkeys = *input.MaxKeys
|
||||
}
|
||||
prefix := *input.Prefix
|
||||
marker := *input.ContinuationToken
|
||||
delim := *input.Delimiter
|
||||
maxkeys := input.MaxKeys
|
||||
|
||||
_, err := os.Stat(bucket)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
@@ -636,9 +602,9 @@ func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input)
|
||||
CommonPrefixes: results.CommonPrefixes,
|
||||
Contents: results.Objects,
|
||||
Delimiter: &delim,
|
||||
IsTruncated: &results.Truncated,
|
||||
IsTruncated: results.Truncated,
|
||||
ContinuationToken: &marker,
|
||||
MaxKeys: &maxkeys,
|
||||
MaxKeys: int32(maxkeys),
|
||||
Name: &bucket,
|
||||
NextContinuationToken: &results.NextMarker,
|
||||
Prefix: &prefix,
|
||||
@@ -711,13 +677,11 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
|
||||
}
|
||||
}
|
||||
|
||||
size := fi.Size()
|
||||
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Size: &size,
|
||||
Size: fi.Size(),
|
||||
StorageClass: sc,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -55,13 +55,11 @@ func getObj(path string, d fs.DirEntry) (types.Object, error) {
|
||||
return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
||||
}
|
||||
|
||||
size := fi.Size()
|
||||
|
||||
return types.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
LastModified: backend.GetTimePtr(fi.ModTime()),
|
||||
Size: &size,
|
||||
Size: fi.Size(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/versity/versitygw/backend/posix"
|
||||
"github.com/versity/versitygw/integration"
|
||||
)
|
||||
|
||||
const (
|
||||
tdir = "tempdir"
|
||||
)
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
|
||||
func initEnv(dir string) {
|
||||
// both
|
||||
debug = true
|
||||
region = "us-east-1"
|
||||
|
||||
// server
|
||||
rootUserAccess = "user"
|
||||
rootUserSecret = "pass"
|
||||
iamDir = dir
|
||||
port = "127.0.0.1:7070"
|
||||
|
||||
// client
|
||||
awsID = "user"
|
||||
awsSecret = "pass"
|
||||
endpoint = "http://127.0.0.1:7070"
|
||||
}
|
||||
|
||||
func initPosix(ctx context.Context) {
|
||||
path, err := os.Getwd()
|
||||
if err != nil {
|
||||
log.Fatalf("get current directory: %v", err)
|
||||
}
|
||||
|
||||
tempdir := filepath.Join(path, tdir)
|
||||
initEnv(tempdir)
|
||||
|
||||
err = os.RemoveAll(tempdir)
|
||||
if err != nil {
|
||||
log.Fatalf("remove temp directory: %v", err)
|
||||
}
|
||||
|
||||
err = os.Mkdir(tempdir, 0755)
|
||||
if err != nil {
|
||||
log.Fatalf("make temp directory: %v", err)
|
||||
}
|
||||
|
||||
be, err := posix.New(tempdir)
|
||||
if err != nil {
|
||||
log.Fatalf("init posix: %v", err)
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
err = runGateway(ctx, be)
|
||||
if err != nil && err != context.Canceled {
|
||||
log.Fatalf("run gateway: %v", err)
|
||||
}
|
||||
|
||||
err := os.RemoveAll(tempdir)
|
||||
if err != nil {
|
||||
log.Fatalf("remove temp directory: %v", err)
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
func TestIntegration(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
initPosix(ctx)
|
||||
|
||||
opts := []integration.Option{
|
||||
integration.WithAccess(awsID),
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
|
||||
s := integration.NewS3Conf(opts...)
|
||||
|
||||
// replace below with desired test
|
||||
err := integration.HeadBucket_non_existing_bucket(s)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
cancel()
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/urfave/cli/v2"
|
||||
@@ -43,15 +44,10 @@ var (
|
||||
logWebhookURL string
|
||||
accessLog string
|
||||
debug bool
|
||||
quiet bool
|
||||
iamDir string
|
||||
ldapURL, ldapBindDN, ldapPassword string
|
||||
ldapQueryBase, ldapObjClasses string
|
||||
ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
|
||||
s3IamAccess, s3IamSecret string
|
||||
s3IamRegion, s3IamBucket string
|
||||
s3IamEndpoint string
|
||||
s3IamSslNoVerify, s3IamDebug bool
|
||||
iamCacheDisable bool
|
||||
iamCacheTTL int
|
||||
iamCachePrune int
|
||||
@@ -177,12 +173,6 @@ func initFlags() []cli.Flag {
|
||||
Usage: "enable debug output",
|
||||
Destination: &debug,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "quiet",
|
||||
Usage: "silence stdout request logging output",
|
||||
Destination: &quiet,
|
||||
Aliases: []string{"q"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "access-log",
|
||||
Usage: "enable server access logging to specified file",
|
||||
@@ -270,42 +260,6 @@ func initFlags() []cli.Flag {
|
||||
Usage: "ldap server user role attribute name",
|
||||
Destination: &ldapRoleAtr,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "s3-iam-access",
|
||||
Usage: "s3 IAM access key",
|
||||
Destination: &s3IamAccess,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "s3-iam-secret",
|
||||
Usage: "s3 IAM secret key",
|
||||
Destination: &s3IamSecret,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "s3-iam-region",
|
||||
Usage: "s3 IAM region",
|
||||
Destination: &s3IamRegion,
|
||||
Value: "us-east-1",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "s3-iam-bucket",
|
||||
Usage: "s3 IAM bucket",
|
||||
Destination: &s3IamBucket,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "s3-iam-endpoint",
|
||||
Usage: "s3 IAM endpoint",
|
||||
Destination: &s3IamEndpoint,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "s3-iam-noverify",
|
||||
Usage: "s3 IAM disable ssl verification",
|
||||
Destination: &s3IamSslNoVerify,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "s3-iam-debug",
|
||||
Usage: "s3 IAM debug output",
|
||||
Destination: &s3IamDebug,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "iam-cache-disable",
|
||||
Usage: "disable local iam cache",
|
||||
@@ -326,12 +280,18 @@ func initFlags() []cli.Flag {
|
||||
}
|
||||
}
|
||||
|
||||
func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
func runGateway(ctx *cli.Context, be backend.Backend) error {
|
||||
// int32 max for 32 bit arch
|
||||
blimit := int64(2*1024*1024*1024 - 1)
|
||||
if strconv.IntSize > 32 {
|
||||
// 5GB max for 64 bit arch
|
||||
blimit = int64(5 * 1024 * 1024 * 1024)
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
AppName: "versitygw",
|
||||
ServerHeader: "VERSITYGW",
|
||||
StreamRequestBody: true,
|
||||
DisableKeepalive: true,
|
||||
AppName: "versitygw",
|
||||
ServerHeader: "VERSITYGW",
|
||||
BodyLimit: int(blimit),
|
||||
})
|
||||
|
||||
var opts []s3api.Option
|
||||
@@ -356,9 +316,6 @@ func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
if admPort == "" {
|
||||
opts = append(opts, s3api.WithAdminServer())
|
||||
}
|
||||
if quiet {
|
||||
opts = append(opts, s3api.WithQuiet())
|
||||
}
|
||||
|
||||
admApp := fiber.New(fiber.Config{
|
||||
AppName: "versitygw",
|
||||
@@ -383,25 +340,18 @@ func runGateway(ctx context.Context, be backend.Backend) error {
|
||||
}
|
||||
|
||||
iam, err := auth.New(&auth.Opts{
|
||||
Dir: iamDir,
|
||||
LDAPServerURL: ldapURL,
|
||||
LDAPBindDN: ldapBindDN,
|
||||
LDAPPassword: ldapPassword,
|
||||
LDAPQueryBase: ldapQueryBase,
|
||||
LDAPObjClasses: ldapObjClasses,
|
||||
LDAPAccessAtr: ldapAccessAtr,
|
||||
LDAPSecretAtr: ldapSecAtr,
|
||||
LDAPRoleAtr: ldapRoleAtr,
|
||||
S3Access: s3IamAccess,
|
||||
S3Secret: s3IamSecret,
|
||||
S3Region: s3IamRegion,
|
||||
S3Bucket: s3IamBucket,
|
||||
S3Endpoint: s3IamEndpoint,
|
||||
S3DisableSSlVerfiy: s3IamSslNoVerify,
|
||||
S3Debug: s3IamDebug,
|
||||
CacheDisable: iamCacheDisable,
|
||||
CacheTTL: iamCacheTTL,
|
||||
CachePrune: iamCachePrune,
|
||||
Dir: iamDir,
|
||||
LDAPServerURL: ldapURL,
|
||||
LDAPBindDN: ldapBindDN,
|
||||
LDAPPassword: ldapPassword,
|
||||
LDAPQueryBase: ldapQueryBase,
|
||||
LDAPObjClasses: ldapObjClasses,
|
||||
LDAPAccessAtr: ldapAccessAtr,
|
||||
LDAPSecretAtr: ldapSecAtr,
|
||||
LDAPRoleAtr: ldapRoleAtr,
|
||||
CacheDisable: iamCacheDisable,
|
||||
CacheTTL: iamCacheTTL,
|
||||
CachePrune: iamCachePrune,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("setup iam: %w", err)
|
||||
|
||||
@@ -49,5 +49,5 @@ func runPosix(ctx *cli.Context) error {
|
||||
return fmt.Errorf("init posix: %v", err)
|
||||
}
|
||||
|
||||
return runGateway(ctx.Context, be)
|
||||
return runGateway(ctx, be)
|
||||
}
|
||||
|
||||
@@ -15,15 +15,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/backend/s3proxy"
|
||||
)
|
||||
|
||||
var (
|
||||
s3proxyAccess string
|
||||
s3proxySecret string
|
||||
s3proxyEndpoint string
|
||||
s3proxyRegion string
|
||||
s3proxyDisableChecksum bool
|
||||
@@ -39,22 +35,6 @@ func s3Command() *cli.Command {
|
||||
to an s3 storage backend service.`,
|
||||
Action: runS3,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "access",
|
||||
Usage: "s3 proxy server access key id",
|
||||
Value: "",
|
||||
Required: true,
|
||||
Destination: &s3proxyAccess,
|
||||
Aliases: []string{"a"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "secret",
|
||||
Usage: "s3 proxy server secret access key",
|
||||
Value: "",
|
||||
Required: true,
|
||||
Destination: &s3proxySecret,
|
||||
Aliases: []string{"s"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "endpoint",
|
||||
Usage: "s3 service endpoint, default AWS if not specified",
|
||||
@@ -90,10 +70,7 @@ to an s3 storage backend service.`,
|
||||
}
|
||||
|
||||
func runS3(ctx *cli.Context) error {
|
||||
be, err := s3proxy.New(s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
|
||||
be := s3proxy.New(s3proxyEndpoint, s3proxyRegion,
|
||||
s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init s3 backend: %w", err)
|
||||
}
|
||||
return runGateway(ctx.Context, be)
|
||||
return runGateway(ctx, be)
|
||||
}
|
||||
|
||||
@@ -69,5 +69,5 @@ func runScoutfs(ctx *cli.Context) error {
|
||||
return fmt.Errorf("init scoutfs: %v", err)
|
||||
}
|
||||
|
||||
return runGateway(ctx.Context, be)
|
||||
return runGateway(ctx, be)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
"github.com/versity/versitygw/integration"
|
||||
@@ -13,6 +16,7 @@ var (
|
||||
endpoint string
|
||||
prefix string
|
||||
dstBucket string
|
||||
proxyURL string
|
||||
partSize int64
|
||||
objSize int64
|
||||
concurrency int
|
||||
@@ -67,7 +71,7 @@ func initTestFlags() []cli.Flag {
|
||||
}
|
||||
|
||||
func initTestCommands() []*cli.Command {
|
||||
return append([]*cli.Command{
|
||||
return []*cli.Command{
|
||||
{
|
||||
Name: "full-flow",
|
||||
Usage: "Tests the full flow of gateway.",
|
||||
@@ -118,6 +122,7 @@ func initTestCommands() []*cli.Command {
|
||||
Name: "bucket",
|
||||
Usage: "Destination bucket name to read/write data",
|
||||
Destination: &dstBucket,
|
||||
Required: true,
|
||||
},
|
||||
&cli.Int64Flag{
|
||||
Name: "partSize",
|
||||
@@ -143,6 +148,11 @@ func initTestCommands() []*cli.Command {
|
||||
Value: false,
|
||||
Destination: &checksumDisable,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "proxy-url",
|
||||
Usage: "S3 proxy server url to compare",
|
||||
Destination: &proxyURL,
|
||||
},
|
||||
},
|
||||
Action: func(ctx *cli.Context) error {
|
||||
if upload && download {
|
||||
@@ -152,10 +162,6 @@ func initTestCommands() []*cli.Command {
|
||||
return fmt.Errorf("must specify one of upload or download")
|
||||
}
|
||||
|
||||
if dstBucket == "" {
|
||||
return fmt.Errorf("must specify bucket")
|
||||
}
|
||||
|
||||
opts := []integration.Option{
|
||||
integration.WithAccess(awsID),
|
||||
integration.WithSecret(awsSecret),
|
||||
@@ -177,9 +183,47 @@ func initTestCommands() []*cli.Command {
|
||||
s3conf := integration.NewS3Conf(opts...)
|
||||
|
||||
if upload {
|
||||
return integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
|
||||
if proxyURL == "" {
|
||||
integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
|
||||
return nil
|
||||
} else {
|
||||
size, elapsed, err := integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
|
||||
opts = append(opts, integration.WithEndpoint(proxyURL))
|
||||
proxyS3Conf := integration.NewS3Conf(opts...)
|
||||
proxySize, proxyElapsed, proxyErr := integration.TestUpload(proxyS3Conf, files, objSize, dstBucket, prefix)
|
||||
if err != nil || proxyErr != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
printProxyResultsTable([][4]string{
|
||||
{" # ", "Total Size", "Time Taken", "Speed(MB/S)"},
|
||||
{"---------", "----------", "----------", "-----------"},
|
||||
{"S3 Server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
|
||||
{"S3 Proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
|
||||
})
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
return integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
|
||||
if proxyURL == "" {
|
||||
integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
|
||||
return nil
|
||||
} else {
|
||||
size, elapsed, err := integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
|
||||
opts = append(opts, integration.WithEndpoint(proxyURL))
|
||||
proxyS3Conf := integration.NewS3Conf(opts...)
|
||||
proxySize, proxyElapsed, proxyErr := integration.TestDownload(proxyS3Conf, files, objSize, dstBucket, prefix)
|
||||
if err != nil || proxyErr != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
printProxyResultsTable([][4]string{
|
||||
{" # ", "Total Size", "Time Taken", "Speed(MB/S)"},
|
||||
{"---------", "----------", "----------", "-----------"},
|
||||
{"S3 server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
|
||||
{"S3 proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
|
||||
})
|
||||
return nil
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
@@ -211,12 +255,13 @@ func initTestCommands() []*cli.Command {
|
||||
Value: false,
|
||||
Destination: &checksumDisable,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "proxy-url",
|
||||
Usage: "S3 proxy server url to compare",
|
||||
Destination: &proxyURL,
|
||||
},
|
||||
},
|
||||
Action: func(ctx *cli.Context) error {
|
||||
if dstBucket == "" {
|
||||
return fmt.Errorf("must specify the destination bucket")
|
||||
}
|
||||
|
||||
opts := []integration.Option{
|
||||
integration.WithAccess(awsID),
|
||||
integration.WithSecret(awsSecret),
|
||||
@@ -233,10 +278,30 @@ func initTestCommands() []*cli.Command {
|
||||
|
||||
s3conf := integration.NewS3Conf(opts...)
|
||||
|
||||
return integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
|
||||
if proxyURL == "" {
|
||||
_, _, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
|
||||
return err
|
||||
} else {
|
||||
elapsed, rps, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
|
||||
opts = append(opts, integration.WithEndpoint(proxyURL))
|
||||
s3proxy := integration.NewS3Conf(opts...)
|
||||
proxyElapsed, proxyRPS, proxyErr := integration.TestReqPerSec(s3proxy, totalReqs, dstBucket)
|
||||
if err != nil || proxyErr != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
printProxyResultsTable([][4]string{
|
||||
{" # ", "Total Requests", "Time Taken", "Requests Per Second(Req/Sec)"},
|
||||
{"---------", "--------------", "----------", "----------------------------"},
|
||||
{"S3 Server", fmt.Sprint(totalReqs), fmt.Sprintf("%v", elapsed), fmt.Sprint(rps)},
|
||||
{"S3 Proxy", fmt.Sprint(totalReqs), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(proxyRPS)},
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
},
|
||||
},
|
||||
}, extractIntTests()...)
|
||||
}
|
||||
}
|
||||
|
||||
type testFunc func(*integration.S3Conf)
|
||||
@@ -265,30 +330,12 @@ func getAction(tf testFunc) func(*cli.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
func extractIntTests() (commands []*cli.Command) {
|
||||
tests := integration.GetIntTests()
|
||||
for key, val := range tests {
|
||||
testKey := key
|
||||
testFunc := val
|
||||
commands = append(commands, &cli.Command{
|
||||
Name: testKey,
|
||||
Usage: fmt.Sprintf("Runs %v integration test", testKey),
|
||||
Action: func(ctx *cli.Context) error {
|
||||
opts := []integration.Option{
|
||||
integration.WithAccess(awsID),
|
||||
integration.WithSecret(awsSecret),
|
||||
integration.WithRegion(region),
|
||||
integration.WithEndpoint(endpoint),
|
||||
}
|
||||
if debug {
|
||||
opts = append(opts, integration.WithDebug())
|
||||
}
|
||||
|
||||
s := integration.NewS3Conf(opts...)
|
||||
err := testFunc(s)
|
||||
return err
|
||||
},
|
||||
})
|
||||
func printProxyResultsTable(stats [][4]string) {
|
||||
w := new(tabwriter.Writer)
|
||||
w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
|
||||
for _, elem := range stats {
|
||||
fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", elem[0], elem[1], elem[2], elem[3])
|
||||
}
|
||||
return
|
||||
fmt.Fprintln(w)
|
||||
w.Flush()
|
||||
}
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
version: "3"
|
||||
services:
|
||||
posix:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: ./Dockerfile.dev
|
||||
args:
|
||||
- IAM_DIR=${IAM_DIR}
|
||||
- SETUP_DIR=${SETUP_DIR}
|
||||
volumes:
|
||||
- ./:/app
|
||||
ports:
|
||||
- "${POSIX_PORT}:${POSIX_PORT}"
|
||||
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$POSIX_PORT -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR posix $SETUP_DIR"]
|
||||
proxy:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: ./Dockerfile.dev
|
||||
volumes:
|
||||
- ./:/app
|
||||
ports:
|
||||
- "${PROXY_PORT}:${PROXY_PORT}"
|
||||
command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$PROXY_PORT s3 -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --endpoint http://posix:$POSIX_PORT"]
|
||||
54
go.mod
54
go.mod
@@ -3,54 +3,54 @@ module github.com/versity/versitygw
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.0
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7
|
||||
github.com/aws/smithy-go v1.19.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.22.2
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1
|
||||
github.com/aws/smithy-go v1.16.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.6
|
||||
github.com/gofiber/fiber/v2 v2.51.0
|
||||
github.com/google/uuid v1.5.0
|
||||
github.com/gofiber/fiber/v2 v2.50.0
|
||||
github.com/google/uuid v1.4.0
|
||||
github.com/nats-io/nats.go v1.31.0
|
||||
github.com/pkg/xattr v0.4.9
|
||||
github.com/segmentio/kafka-go v0.4.47
|
||||
github.com/urfave/cli/v2 v2.26.0
|
||||
github.com/valyala/fasthttp v1.51.0
|
||||
github.com/segmentio/kafka-go v0.4.44
|
||||
github.com/urfave/cli/v2 v2.25.7
|
||||
github.com/valyala/fasthttp v1.50.0
|
||||
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
|
||||
golang.org/x/sys v0.15.0
|
||||
golang.org/x/sys v0.14.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.6 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.18 // indirect
|
||||
github.com/stretchr/testify v1.8.1 // indirect
|
||||
golang.org/x/crypto v0.17.0 // indirect
|
||||
golang.org/x/crypto v0.14.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.26.2
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.13
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.9
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.24.0
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.15.2
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||
github.com/klauspost/compress v1.17.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/rivo/uniseg v0.4.4 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
|
||||
109
go.sum
109
go.sum
@@ -4,44 +4,44 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3Uu
|
||||
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
|
||||
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.13 h1:WLABQ4Cp4vXtXfOWOS3MEZKr6AAYUpMczLhgKtAjQ/8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.13/go.mod h1:Qg6x82FXwW0sJHzYruxGiuApNo31UEtJvXVSZAXeWiw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.9 h1:5zA8qVCXMPGt6YneFnll5B157SfdK2SewU85PH9/yM0=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.9/go.mod h1:t4gy210hPxkbtYM8xOzrWdxVq1PyekR76OOKXy3s0Vs=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9/go.mod h1:YD0aYBWCrPENpHolhKw2XDlTIWae2GKXT1T4o6N6hiM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7 h1:o0ASbVwUAIrfp/WcCac+6jioZt4Hd8k/1X8u7GJ/QeM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7/go.mod h1:vADO6Jn+Rq4nDtfwNjhgR84qkZwiC6FqCaXdw/kYwjA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 h1:HJeiuZ2fldpd0WqngyMR6KW7ofkXNLyOaHwEIGm39Cs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.26.6/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU=
|
||||
github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
|
||||
github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
|
||||
github.com/aws/aws-sdk-go-v2 v1.22.2 h1:lV0U8fnhAnPz8YcdmZVV60+tr6CakHzqA6P8T46ExJI=
|
||||
github.com/aws/aws-sdk-go-v2 v1.22.2/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 h1:hHgLiIrTRtddC0AKcJr5s7i/hLgcpTt+q/FKxf1Zayk=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0/go.mod h1:w4I/v3NOWgD+qvs1NPEwhd++1h3XPHFaVxasfY6HlYQ=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.24.0 h1:4LEk29JO3w+y9dEo/5Tq5QTP7uIEw+KQrKiHOs4xlu4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.24.0/go.mod h1:11nNDAuK86kOUHeuEQo8f3CkcV5xuUxvPwFjTZE/PnQ=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.15.2 h1:rKH7khRMxPdD0u3dHecd0Q7NOVw3EUe7AqdkUOkiOGI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.15.2/go.mod h1:tXM8wmaeAhfC7nZoCxb0FzM/aRaB1m1WQ7x0qlBLq80=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 h1:G5KawTAkyHH6WyKQCdHiW4h3PmAXNJpOgwKg3H7sDRE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3/go.mod h1:hugKmSFnZB+HgNI1sYGT14BUPZkO6alC/e0AWu+0IAQ=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6 h1:IpQbitxCZeC64C1ALz9QZu6AHHWundnU2evQ9xbp5k8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6/go.mod h1:27jIVQK+al9s0yTo3pkMdahRinbscqSC6zNGfNWXPZc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 h1:AaQsr5vvGR7rmeSWBtTCcw16tT9r51mWijuCQhzLnq8=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2/go.mod h1:o1IiRn7CWocIFTXJjGKJDOwxv1ibL53NpcvcqGWyRBA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 h1:UZx8SXZ0YtzRiALzYAWcjb9Y9hZUR7MBKaBQ5ouOjPs=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2/go.mod h1:ipuRpcSaklmxR6C39G187TpBAO132gUfleTGccUPs8c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 h1:usgqiJtamuGIBj+OvYmMq89+Z1hIKkMJToz1WpoeNUY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 h1:pyVrNAf7Hwz0u39dLKN5t+n0+K/3rMYKuiOoIum3AsU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2/go.mod h1:mydrfOb9uiOYCxuCPR8YHQNQyGQwUQ7gPMZGBKbH8NY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 h1:CJxo7ZBbaIzmXfV3hjcx36n9V87gJsIUPJflwqEHl3Q=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0/go.mod h1:yjVfjuY4nD1EW9i387Kau+I6V5cBA5YnC/mWNopjZrI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 h1:f2LhPofnjcdOQKRtumKjMvIHkfSQ8aH/rwKUDEQ/SB4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2/go.mod h1:q+xX0H4OfuWDuBy7y/LDi4v8IBOWuF+vtp8Z6ex+lw4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 h1:h7j73yuAVVjic8pqswh+L/7r2IHP43QwRyOu6zcCDDE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2/go.mod h1:H07AHdK5LSy8F7EJUQhoxyiCNkePoHj2D8P2yGTWafo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 h1:gbIaOzpXixUpoPK+js/bCBK1QBDXM22SigsnzGZio0U=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2/go.mod h1:p+S7RNbdGN8qgHDSg2SCQJ9FeMAmvcETQiVpeGhYnNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 h1:o6MCcX1rJW8Y3g+hvg2xpjF6JR6DftuYhfl3Nc1WV9Q=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1/go.mod h1:UDtxEWbREX6y4KREapT+jjtjoH0TiVSS6f5nfaY1UaM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 h1:km+ZNjtLtpXYf42RdaDZnNHm9s7SYAuDGTafy6nd89A=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.17.1/go.mod h1:aHBr3pvBSD5MbzOvQtYutyPLLRPbl/y9x86XyJJnUXQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 h1:iRFNqZH4a67IqPvK8xxtyQYnyrlsvwmpHOe9r55ggBA=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1/go.mod h1:pTy5WM+6sNv2tB24JNKFtn6EvciQ5k40ZJ0pq/Iaxj0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 h1:txgVXIXWPXyqdiVn92BV6a/rgtpX31HYdsOYj0sVQQQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.25.1/go.mod h1:VAiJiNaoP1L89STFlEMgmHX1bKixY+FaP+TpRFrmyZ4=
|
||||
github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik=
|
||||
github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -51,12 +51,12 @@ github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
|
||||
github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
|
||||
github.com/gofiber/fiber/v2 v2.51.0 h1:JNACcZy5e2tGApWB2QrRpenTWn0fq0hkFm6k0C86gKQ=
|
||||
github.com/gofiber/fiber/v2 v2.51.0/go.mod h1:xaQRZQJGqnKOQnbQw+ltvku3/h8QxvNi8o6JiJ7Ll0U=
|
||||
github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
|
||||
github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
|
||||
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
|
||||
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
|
||||
github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
|
||||
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
@@ -67,8 +67,8 @@ github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
|
||||
@@ -89,8 +89,8 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
|
||||
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
|
||||
github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
|
||||
github.com/segmentio/kafka-go v0.4.44 h1:Vjjksniy0WSTZ7CuVJrz1k04UoZeTc77UV6Yyk6tLY4=
|
||||
github.com/segmentio/kafka-go v0.4.44/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
@@ -98,12 +98,12 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI=
|
||||
github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
|
||||
github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
|
||||
github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
|
||||
github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
|
||||
github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
|
||||
github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
|
||||
@@ -120,9 +120,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
|
||||
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
@@ -147,8 +146,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
|
||||
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
|
||||
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
@@ -161,8 +160,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
|
||||
@@ -17,16 +17,16 @@ type prefResult struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
|
||||
func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
|
||||
var sg sync.WaitGroup
|
||||
results := make([]prefResult, files)
|
||||
start := time.Now()
|
||||
if objSize == 0 {
|
||||
return fmt.Errorf("must specify object size for upload")
|
||||
return 0, time.Since(start), fmt.Errorf("must specify object size for upload")
|
||||
}
|
||||
|
||||
if objSize > (int64(10000) * s.PartSize) {
|
||||
return fmt.Errorf("object size can not exceed 10000 * chunksize")
|
||||
return 0, time.Since(start), fmt.Errorf("object size can not exceed 10000 * chunksize")
|
||||
}
|
||||
|
||||
runF("performance test: upload objects")
|
||||
@@ -45,13 +45,13 @@ func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) erro
|
||||
}(i)
|
||||
}
|
||||
sg.Wait()
|
||||
elapsed := time.Since(start)
|
||||
elapsed = time.Since(start)
|
||||
|
||||
var tot int64
|
||||
for i, res := range results {
|
||||
if res.err != nil {
|
||||
failF("%v: %v\n", i, res.err)
|
||||
break
|
||||
return 0, time.Since(start), res.err
|
||||
}
|
||||
tot += res.size
|
||||
fmt.Printf("%v: %v in %v (%v MB/s)\n",
|
||||
@@ -63,10 +63,10 @@ func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) erro
|
||||
passF("run upload: %v in %v (%v MB/s)\n",
|
||||
tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
|
||||
|
||||
return nil
|
||||
return tot, time.Since(start), nil
|
||||
}
|
||||
|
||||
func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
|
||||
func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
|
||||
var sg sync.WaitGroup
|
||||
results := make([]prefResult, files)
|
||||
start := time.Now()
|
||||
@@ -86,13 +86,13 @@ func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) er
|
||||
}(i)
|
||||
}
|
||||
sg.Wait()
|
||||
elapsed := time.Since(start)
|
||||
elapsed = time.Since(start)
|
||||
|
||||
var tot int64
|
||||
for i, res := range results {
|
||||
if res.err != nil {
|
||||
failF("%v: %v\n", i, res.err)
|
||||
break
|
||||
return 0, elapsed, err
|
||||
}
|
||||
tot += res.size
|
||||
fmt.Printf("%v: %v in %v (%v MB/s)\n",
|
||||
@@ -104,10 +104,10 @@ func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) er
|
||||
passF("run download: %v in %v (%v MB/s)\n",
|
||||
tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
|
||||
|
||||
return nil
|
||||
return tot, elapsed, nil
|
||||
}
|
||||
|
||||
func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) error {
|
||||
func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) (time.Duration, int, error) {
|
||||
client := s3.NewFromConfig(s.Config())
|
||||
var wg sync.WaitGroup
|
||||
var resErr error
|
||||
@@ -132,11 +132,11 @@ func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) error {
|
||||
wg.Wait()
|
||||
if resErr != nil {
|
||||
failF("performance test failed with error: %w", resErr)
|
||||
return nil
|
||||
return time.Since(startTime), 0, resErr
|
||||
}
|
||||
elapsedTime := time.Since(startTime)
|
||||
rps := int(float64(totalReqs) / elapsedTime.Seconds())
|
||||
|
||||
passF("Success\nTotal Requests: %d,\nConcurrency Level: %d,\nTime Taken: %s,\nRequests Per Second: %dreq/sec", totalReqs, s.Concurrency, elapsedTime, rps)
|
||||
return nil
|
||||
return elapsedTime, rps, nil
|
||||
}
|
||||
|
||||
@@ -51,7 +51,6 @@ func TestPutObject(s *S3Conf) {
|
||||
PutObject_special_chars(s)
|
||||
PutObject_invalid_long_tags(s)
|
||||
PutObject_success(s)
|
||||
PutObject_invalid_credentials(s)
|
||||
}
|
||||
|
||||
func TestHeadObject(s *S3Conf) {
|
||||
@@ -223,133 +222,3 @@ func TestPosix(s *S3Conf) {
|
||||
PutObject_dir_obj_with_data(s)
|
||||
CreateMultipartUpload_dir_obj(s)
|
||||
}
|
||||
|
||||
type IntTests map[string]func(s *S3Conf) error
|
||||
|
||||
func GetIntTests() IntTests {
|
||||
return IntTests{
|
||||
"Authentication_empty_auth_header": Authentication_empty_auth_header,
|
||||
"Authentication_invalid_auth_header": Authentication_invalid_auth_header,
|
||||
"Authentication_unsupported_signature_version": Authentication_unsupported_signature_version,
|
||||
"Authentication_malformed_credentials": Authentication_malformed_credentials,
|
||||
"Authentication_malformed_credentials_invalid_parts": Authentication_malformed_credentials_invalid_parts,
|
||||
"Authentication_credentials_terminated_string": Authentication_credentials_terminated_string,
|
||||
"Authentication_credentials_incorrect_service": Authentication_credentials_incorrect_service,
|
||||
"Authentication_credentials_incorrect_region": Authentication_credentials_incorrect_region,
|
||||
"Authentication_credentials_invalid_date": Authentication_credentials_invalid_date,
|
||||
"Authentication_credentials_future_date": Authentication_credentials_future_date,
|
||||
"Authentication_credentials_past_date": Authentication_credentials_past_date,
|
||||
"Authentication_credentials_non_existing_access_key": Authentication_credentials_non_existing_access_key,
|
||||
"Authentication_invalid_signed_headers": Authentication_invalid_signed_headers,
|
||||
"Authentication_missing_date_header": Authentication_missing_date_header,
|
||||
"Authentication_invalid_date_header": Authentication_invalid_date_header,
|
||||
"Authentication_date_mismatch": Authentication_date_mismatch,
|
||||
"Authentication_incorrect_payload_hash": Authentication_incorrect_payload_hash,
|
||||
"Authentication_incorrect_md5": Authentication_incorrect_md5,
|
||||
"Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key,
|
||||
"CreateBucket_invalid_bucket_name": CreateBucket_invalid_bucket_name,
|
||||
"CreateBucket_existing_bucket": CreateBucket_existing_bucket,
|
||||
"CreateBucket_as_user": CreateBucket_as_user,
|
||||
"CreateDeleteBucket_success": CreateDeleteBucket_success,
|
||||
"HeadBucket_non_existing_bucket": HeadBucket_non_existing_bucket,
|
||||
"HeadBucket_success": HeadBucket_success,
|
||||
"ListBuckets_as_user": ListBuckets_as_user,
|
||||
"ListBuckets_as_admin": ListBuckets_as_admin,
|
||||
"ListBuckets_success": ListBuckets_success,
|
||||
"DeleteBucket_non_existing_bucket": DeleteBucket_non_existing_bucket,
|
||||
"DeleteBucket_non_empty_bucket": DeleteBucket_non_empty_bucket,
|
||||
"DeleteBucket_success_status_code": DeleteBucket_success_status_code,
|
||||
"PutObject_non_existing_bucket": PutObject_non_existing_bucket,
|
||||
"PutObject_special_chars": PutObject_special_chars,
|
||||
"PutObject_invalid_long_tags": PutObject_invalid_long_tags,
|
||||
"PutObject_success": PutObject_success,
|
||||
"PutObject_invalid_credentials": PutObject_invalid_credentials,
|
||||
"HeadObject_non_existing_object": HeadObject_non_existing_object,
|
||||
"HeadObject_success": HeadObject_success,
|
||||
"GetObject_non_existing_key": GetObject_non_existing_key,
|
||||
"GetObject_invalid_ranges": GetObject_invalid_ranges,
|
||||
"GetObject_with_meta": GetObject_with_meta,
|
||||
"GetObject_success": GetObject_success,
|
||||
"GetObject_by_range_success": GetObject_by_range_success,
|
||||
"ListObjects_non_existing_bucket": ListObjects_non_existing_bucket,
|
||||
"ListObjects_with_prefix": ListObjects_with_prefix,
|
||||
"ListObject_truncated": ListObject_truncated,
|
||||
"ListObjects_invalid_max_keys": ListObjects_invalid_max_keys,
|
||||
"ListObjects_max_keys_0": ListObjects_max_keys_0,
|
||||
"ListObjects_delimiter": ListObjects_delimiter,
|
||||
"ListObjects_max_keys_none": ListObjects_max_keys_none,
|
||||
"ListObjects_marker_not_from_obj_list": ListObjects_marker_not_from_obj_list,
|
||||
"DeleteObject_non_existing_object": DeleteObject_non_existing_object,
|
||||
"DeleteObject_success": DeleteObject_success,
|
||||
"DeleteObject_success_status_code": DeleteObject_success_status_code,
|
||||
"DeleteObjects_empty_input": DeleteObjects_empty_input,
|
||||
"DeleteObjects_non_existing_objects": DeleteObjects_non_existing_objects,
|
||||
"DeleteObjects_success": DeleteObjects_success,
|
||||
"CopyObject_non_existing_dst_bucket": CopyObject_non_existing_dst_bucket,
|
||||
"CopyObject_not_owned_source_bucket": CopyObject_not_owned_source_bucket,
|
||||
"CopyObject_copy_to_itself": CopyObject_copy_to_itself,
|
||||
"CopyObject_to_itself_with_new_metadata": CopyObject_to_itself_with_new_metadata,
|
||||
"CopyObject_success": CopyObject_success,
|
||||
"PutObjectTagging_non_existing_object": PutObjectTagging_non_existing_object,
|
||||
"PutObjectTagging_long_tags": PutObjectTagging_long_tags,
|
||||
"PutObjectTagging_success": PutObjectTagging_success,
|
||||
"GetObjectTagging_non_existing_object": GetObjectTagging_non_existing_object,
|
||||
"GetObjectTagging_success": GetObjectTagging_success,
|
||||
"DeleteObjectTagging_non_existing_object": DeleteObjectTagging_non_existing_object,
|
||||
"DeleteObjectTagging_success_status": DeleteObjectTagging_success_status,
|
||||
"DeleteObjectTagging_success": DeleteObjectTagging_success,
|
||||
"CreateMultipartUpload_non_existing_bucket": CreateMultipartUpload_non_existing_bucket,
|
||||
"CreateMultipartUpload_success": CreateMultipartUpload_success,
|
||||
"UploadPart_non_existing_bucket": UploadPart_non_existing_bucket,
|
||||
"UploadPart_invalid_part_number": UploadPart_invalid_part_number,
|
||||
"UploadPart_non_existing_key": UploadPart_non_existing_key,
|
||||
"UploadPart_non_existing_mp_upload": UploadPart_non_existing_mp_upload,
|
||||
"UploadPart_success": UploadPart_success,
|
||||
"UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket,
|
||||
"UploadPartCopy_incorrect_uploadId": UploadPartCopy_incorrect_uploadId,
|
||||
"UploadPartCopy_incorrect_object_key": UploadPartCopy_incorrect_object_key,
|
||||
"UploadPartCopy_invalid_part_number": UploadPartCopy_invalid_part_number,
|
||||
"UploadPartCopy_invalid_copy_source": UploadPartCopy_invalid_copy_source,
|
||||
"UploadPartCopy_non_existing_source_bucket": UploadPartCopy_non_existing_source_bucket,
|
||||
"UploadPartCopy_non_existing_source_object_key": UploadPartCopy_non_existing_source_object_key,
|
||||
"UploadPartCopy_success": UploadPartCopy_success,
|
||||
"UploadPartCopy_by_range_invalid_range": UploadPartCopy_by_range_invalid_range,
|
||||
"UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size,
|
||||
"UploadPartCopy_by_range_success": UploadPartCopy_by_range_success,
|
||||
"ListParts_incorrect_uploadId": ListParts_incorrect_uploadId,
|
||||
"ListParts_incorrect_object_key": ListParts_incorrect_object_key,
|
||||
"ListParts_success": ListParts_success,
|
||||
"ListMultipartUploads_non_existing_bucket": ListMultipartUploads_non_existing_bucket,
|
||||
"ListMultipartUploads_empty_result": ListMultipartUploads_empty_result,
|
||||
"ListMultipartUploads_invalid_max_uploads": ListMultipartUploads_invalid_max_uploads,
|
||||
"ListMultipartUploads_max_uploads": ListMultipartUploads_max_uploads,
|
||||
"ListMultipartUploads_incorrect_next_key_marker": ListMultipartUploads_incorrect_next_key_marker,
|
||||
"ListMultipartUploads_ignore_upload_id_marker": ListMultipartUploads_ignore_upload_id_marker,
|
||||
"ListMultipartUploads_success": ListMultipartUploads_success,
|
||||
"AbortMultipartUpload_non_existing_bucket": AbortMultipartUpload_non_existing_bucket,
|
||||
"AbortMultipartUpload_incorrect_uploadId": AbortMultipartUpload_incorrect_uploadId,
|
||||
"AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key,
|
||||
"AbortMultipartUpload_success": AbortMultipartUpload_success,
|
||||
"AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code,
|
||||
"CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket,
|
||||
"CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number,
|
||||
"CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag,
|
||||
"CompleteMultipartUpload_success": CompleteMultipartUpload_success,
|
||||
"PutBucketAcl_non_existing_bucket": PutBucketAcl_non_existing_bucket,
|
||||
"PutBucketAcl_invalid_acl_canned_and_acp": PutBucketAcl_invalid_acl_canned_and_acp,
|
||||
"PutBucketAcl_invalid_acl_canned_and_grants": PutBucketAcl_invalid_acl_canned_and_grants,
|
||||
"PutBucketAcl_invalid_acl_acp_and_grants": PutBucketAcl_invalid_acl_acp_and_grants,
|
||||
"PutBucketAcl_invalid_owner": PutBucketAcl_invalid_owner,
|
||||
"PutBucketAcl_success_access_denied": PutBucketAcl_success_access_denied,
|
||||
"PutBucketAcl_success_grants": PutBucketAcl_success_grants,
|
||||
"PutBucketAcl_success_canned_acl": PutBucketAcl_success_canned_acl,
|
||||
"PutBucketAcl_success_acp": PutBucketAcl_success_acp,
|
||||
"GetBucketAcl_non_existing_bucket": GetBucketAcl_non_existing_bucket,
|
||||
"GetBucketAcl_access_denied": GetBucketAcl_access_denied,
|
||||
"GetBucketAcl_success": GetBucketAcl_success,
|
||||
"PutObject_overwrite_dir_obj": PutObject_overwrite_dir_obj,
|
||||
"PutObject_overwrite_file_obj": PutObject_overwrite_file_obj,
|
||||
"PutObject_dir_obj_with_data": PutObject_dir_obj_with_data,
|
||||
"CreateMultipartUpload_dir_obj": CreateMultipartUpload_dir_obj,
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -60,7 +60,7 @@ func teardown(s *S3Conf, bucket string) error {
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete object %v: %w", *key, err)
|
||||
return fmt.Errorf("failed to delete object %v: %v", *key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -71,7 +71,7 @@ func teardown(s *S3Conf, bucket string) error {
|
||||
out, err := s3client.ListObjectsV2(ctx, in)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list objects: %w", err)
|
||||
return fmt.Errorf("failed to list objects: %v", err)
|
||||
}
|
||||
|
||||
for _, item := range out.Contents {
|
||||
@@ -81,7 +81,7 @@ func teardown(s *S3Conf, bucket string) error {
|
||||
}
|
||||
}
|
||||
|
||||
if *out.IsTruncated {
|
||||
if out.IsTruncated {
|
||||
in.ContinuationToken = out.ContinuationToken
|
||||
} else {
|
||||
break
|
||||
@@ -96,32 +96,31 @@ func teardown(s *S3Conf, bucket string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) error {
|
||||
func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) {
|
||||
runF(testName)
|
||||
bucketName := getBucketName()
|
||||
err := setup(s, bucketName)
|
||||
if err != nil {
|
||||
failF("%v: failed to create a bucket: %v", testName, err)
|
||||
return fmt.Errorf("%v: failed to create a bucket: %w", testName, err)
|
||||
failF("%v: failed to create a bucket: %v", testName, err.Error())
|
||||
return
|
||||
}
|
||||
client := s3.NewFromConfig(s.Config())
|
||||
handlerErr := handler(client, bucketName)
|
||||
if handlerErr != nil {
|
||||
failF("%v: %v", testName, handlerErr)
|
||||
failF("%v: %v", testName, handlerErr.Error())
|
||||
}
|
||||
|
||||
err = teardown(s, bucketName)
|
||||
if err != nil {
|
||||
fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err)
|
||||
if handlerErr == nil {
|
||||
return fmt.Errorf("%v: failed to delete the bucket: %w", testName, err)
|
||||
failF("%v: failed to delete the bucket: %v", testName, err.Error())
|
||||
} else {
|
||||
fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err.Error())
|
||||
}
|
||||
}
|
||||
if handlerErr == nil {
|
||||
passF(testName)
|
||||
}
|
||||
|
||||
return handlerErr
|
||||
}
|
||||
|
||||
type authConfig struct {
|
||||
@@ -133,21 +132,20 @@ type authConfig struct {
|
||||
date time.Time
|
||||
}
|
||||
|
||||
func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) error {
|
||||
func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) {
|
||||
runF(cfg.testName)
|
||||
req, err := createSignedReq(cfg.method, s.endpoint, cfg.path, s.awsID, s.awsSecret, cfg.service, s.awsRegion, cfg.body, cfg.date)
|
||||
if err != nil {
|
||||
failF("%v: %v", cfg.testName, err)
|
||||
return fmt.Errorf("%v: %w", cfg.testName, err)
|
||||
failF("%v: %v", cfg.testName, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
err = handler(req)
|
||||
if err != nil {
|
||||
failF("%v: %v", cfg.testName, err)
|
||||
return fmt.Errorf("%v: %w", cfg.testName, err)
|
||||
failF("%v: %v", cfg.testName, err.Error())
|
||||
return
|
||||
}
|
||||
passF(cfg.testName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
|
||||
@@ -207,8 +205,9 @@ func checkApiErr(err error, apiErr s3err.APIError) error {
|
||||
}
|
||||
|
||||
return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
|
||||
} else {
|
||||
return fmt.Errorf("expected aws api error, instead got: %v", err.Error())
|
||||
}
|
||||
return fmt.Errorf("expected aws api error, instead got: %w", err)
|
||||
}
|
||||
|
||||
func checkSdkApiErr(err error, code string) error {
|
||||
@@ -294,7 +293,7 @@ func compareParts(parts1, parts2 []types.Part) bool {
|
||||
}
|
||||
|
||||
for i, prt := range parts1 {
|
||||
if *prt.PartNumber != *parts2[i].PartNumber {
|
||||
if prt.PartNumber != parts2[i].PartNumber {
|
||||
return false
|
||||
}
|
||||
if *prt.ETag != *parts2[i].ETag {
|
||||
@@ -485,23 +484,20 @@ func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId s
|
||||
return parts, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
pn := int32(partNumber)
|
||||
out, err := client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
UploadId: &uploadId,
|
||||
Body: bytes.NewReader(partBuffer),
|
||||
PartNumber: &pn,
|
||||
PartNumber: int32(partNumber),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return parts, err
|
||||
} else {
|
||||
parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)})
|
||||
offset += partSize
|
||||
}
|
||||
parts = append(parts, types.Part{
|
||||
ETag: out.ETag,
|
||||
PartNumber: &pn,
|
||||
})
|
||||
offset += partSize
|
||||
}
|
||||
|
||||
return parts, err
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
# make temp dirs
|
||||
rm -rf /tmp/gw
|
||||
mkdir /tmp/gw
|
||||
rm -rf /tmp/covdata
|
||||
mkdir /tmp/covdata
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/versity/versitygw/backend"
|
||||
@@ -107,7 +106,7 @@ var _ backend.Backend = &BackendMock{}
|
||||
// RestoreObjectFunc: func(contextMoqParam context.Context, restoreObjectInput *s3.RestoreObjectInput) error {
|
||||
// panic("mock out the RestoreObject method")
|
||||
// },
|
||||
// SelectObjectContentFunc: func(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
|
||||
// SelectObjectContentFunc: func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
|
||||
// panic("mock out the SelectObjectContent method")
|
||||
// },
|
||||
// ShutdownFunc: func() {
|
||||
@@ -214,7 +213,7 @@ type BackendMock struct {
|
||||
RestoreObjectFunc func(contextMoqParam context.Context, restoreObjectInput *s3.RestoreObjectInput) error
|
||||
|
||||
// SelectObjectContentFunc mocks the SelectObjectContent method.
|
||||
SelectObjectContentFunc func(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
|
||||
SelectObjectContentFunc func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
|
||||
|
||||
// ShutdownFunc mocks the Shutdown method.
|
||||
ShutdownFunc func()
|
||||
@@ -442,10 +441,10 @@ type BackendMock struct {
|
||||
}
|
||||
// SelectObjectContent holds details about calls to the SelectObjectContent method.
|
||||
SelectObjectContent []struct {
|
||||
// Ctx is the ctx argument value.
|
||||
Ctx context.Context
|
||||
// Input is the input argument value.
|
||||
Input *s3.SelectObjectContentInput
|
||||
// ContextMoqParam is the contextMoqParam argument value.
|
||||
ContextMoqParam context.Context
|
||||
// SelectObjectContentInput is the selectObjectContentInput argument value.
|
||||
SelectObjectContentInput *s3.SelectObjectContentInput
|
||||
}
|
||||
// Shutdown holds details about calls to the Shutdown method.
|
||||
Shutdown []struct {
|
||||
@@ -1540,21 +1539,21 @@ func (mock *BackendMock) RestoreObjectCalls() []struct {
|
||||
}
|
||||
|
||||
// SelectObjectContent calls SelectObjectContentFunc.
|
||||
func (mock *BackendMock) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
|
||||
func (mock *BackendMock) SelectObjectContent(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
|
||||
if mock.SelectObjectContentFunc == nil {
|
||||
panic("BackendMock.SelectObjectContentFunc: method is nil but Backend.SelectObjectContent was just called")
|
||||
}
|
||||
callInfo := struct {
|
||||
Ctx context.Context
|
||||
Input *s3.SelectObjectContentInput
|
||||
ContextMoqParam context.Context
|
||||
SelectObjectContentInput *s3.SelectObjectContentInput
|
||||
}{
|
||||
Ctx: ctx,
|
||||
Input: input,
|
||||
ContextMoqParam: contextMoqParam,
|
||||
SelectObjectContentInput: selectObjectContentInput,
|
||||
}
|
||||
mock.lockSelectObjectContent.Lock()
|
||||
mock.calls.SelectObjectContent = append(mock.calls.SelectObjectContent, callInfo)
|
||||
mock.lockSelectObjectContent.Unlock()
|
||||
return mock.SelectObjectContentFunc(ctx, input)
|
||||
return mock.SelectObjectContentFunc(contextMoqParam, selectObjectContentInput)
|
||||
}
|
||||
|
||||
// SelectObjectContentCalls gets all the calls that were made to SelectObjectContent.
|
||||
@@ -1562,12 +1561,12 @@ func (mock *BackendMock) SelectObjectContent(ctx context.Context, input *s3.Sele
|
||||
//
|
||||
// len(mockedBackend.SelectObjectContentCalls())
|
||||
func (mock *BackendMock) SelectObjectContentCalls() []struct {
|
||||
Ctx context.Context
|
||||
Input *s3.SelectObjectContentInput
|
||||
ContextMoqParam context.Context
|
||||
SelectObjectContentInput *s3.SelectObjectContentInput
|
||||
} {
|
||||
var calls []struct {
|
||||
Ctx context.Context
|
||||
Input *s3.SelectObjectContentInput
|
||||
ContextMoqParam context.Context
|
||||
SelectObjectContentInput *s3.SelectObjectContentInput
|
||||
}
|
||||
mock.lockSelectObjectContent.RLock()
|
||||
calls = mock.calls.SelectObjectContent
|
||||
|
||||
@@ -17,7 +17,6 @@ package controllers
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -64,7 +63,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
|
||||
key := ctx.Params("key")
|
||||
keyEnd := ctx.Params("*1")
|
||||
uploadId := ctx.Query("uploadId")
|
||||
maxParts := int32(ctx.QueryInt("max-parts", -1))
|
||||
maxParts := ctx.QueryInt("max-parts", 0)
|
||||
partNumberMarker := ctx.Query("part-number-marker")
|
||||
acceptRange := ctx.Get("Range")
|
||||
acct := ctx.Locals("account").(auth.Account)
|
||||
@@ -93,7 +92,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
|
||||
}
|
||||
|
||||
if uploadId != "" {
|
||||
if maxParts < 0 && ctx.Request().URI().QueryArgs().Has("max-parts") {
|
||||
if maxParts < 0 || (maxParts == 0 && ctx.Query("max-parts") != "") {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts), &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
if partNumberMarker != "" {
|
||||
@@ -106,17 +105,13 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
|
||||
if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
|
||||
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
var mxParts *int32
|
||||
if ctx.Request().URI().QueryArgs().Has("max-parts") {
|
||||
mxParts = &maxParts
|
||||
}
|
||||
|
||||
res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
UploadId: &uploadId,
|
||||
PartNumberMarker: &partNumberMarker,
|
||||
MaxParts: mxParts,
|
||||
MaxParts: int32(maxParts),
|
||||
})
|
||||
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
@@ -174,7 +169,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
|
||||
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
|
||||
{
|
||||
Key: "Content-Length",
|
||||
Value: fmt.Sprint(getint64(res.ContentLength)),
|
||||
Value: fmt.Sprint(res.ContentLength),
|
||||
},
|
||||
{
|
||||
Key: "Content-Type",
|
||||
@@ -204,17 +199,11 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
|
||||
Key: "accept-ranges",
|
||||
Value: getstring(res.AcceptRanges),
|
||||
},
|
||||
{
|
||||
Key: "x-amz-tagging-count",
|
||||
Value: fmt.Sprint(res.TagCount),
|
||||
},
|
||||
})
|
||||
|
||||
if res.TagCount != nil {
|
||||
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
|
||||
{
|
||||
Key: "x-amz-tagging-count",
|
||||
Value: fmt.Sprint(*res.TagCount),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
|
||||
@@ -225,13 +214,6 @@ func getstring(s *string) string {
|
||||
return *s
|
||||
}
|
||||
|
||||
func getint64(i *int64) int64 {
|
||||
if i == nil {
|
||||
return 0
|
||||
}
|
||||
return *i
|
||||
}
|
||||
|
||||
func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
|
||||
bucket := ctx.Params("bucket")
|
||||
prefix := ctx.Query("prefix")
|
||||
@@ -277,7 +259,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
|
||||
Delimiter: &delimiter,
|
||||
Prefix: &prefix,
|
||||
UploadIdMarker: &uploadIdMarker,
|
||||
MaxUploads: &maxUploads,
|
||||
MaxUploads: maxUploads,
|
||||
KeyMarker: &keyMarker,
|
||||
})
|
||||
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner})
|
||||
@@ -300,7 +282,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
|
||||
Prefix: &prefix,
|
||||
ContinuationToken: &cToken,
|
||||
Delimiter: &delimiter,
|
||||
MaxKeys: &maxkeys,
|
||||
MaxKeys: maxkeys,
|
||||
})
|
||||
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
@@ -323,7 +305,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
|
||||
Prefix: &prefix,
|
||||
Marker: &marker,
|
||||
Delimiter: &delimiter,
|
||||
MaxKeys: &maxkeys,
|
||||
MaxKeys: maxkeys,
|
||||
})
|
||||
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
@@ -485,7 +467,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
|
||||
}
|
||||
|
||||
if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") && copySource != "" {
|
||||
partNumber := int32(ctx.QueryInt("partNumber", -1))
|
||||
partNumber := ctx.QueryInt("partNumber", -1)
|
||||
if partNumber < 1 || partNumber > 10000 {
|
||||
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPartCopy", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
@@ -494,7 +476,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
|
||||
Bucket: &bucket,
|
||||
Key: &keyStart,
|
||||
CopySource: ©Source,
|
||||
PartNumber: &partNumber,
|
||||
PartNumber: int32(partNumber),
|
||||
UploadId: &uploadId,
|
||||
ExpectedBucketOwner: &bucketOwner,
|
||||
CopySourceRange: ©SrcRange,
|
||||
@@ -503,7 +485,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
|
||||
}
|
||||
|
||||
if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") {
|
||||
partNumber := int32(ctx.QueryInt("partNumber", -1))
|
||||
partNumber := ctx.QueryInt("partNumber", -1)
|
||||
if partNumber < 1 || partNumber > 10000 {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
@@ -517,21 +499,14 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
|
||||
var body io.Reader
|
||||
bodyi := ctx.Locals("body-reader")
|
||||
if bodyi != nil {
|
||||
body = bodyi.(io.Reader)
|
||||
} else {
|
||||
body = bytes.NewReader([]byte{})
|
||||
}
|
||||
|
||||
body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body())))
|
||||
ctx.Locals("logReqBody", false)
|
||||
etag, err := c.be.UploadPart(ctx.Context(), &s3.UploadPartInput{
|
||||
Bucket: &bucket,
|
||||
Key: &keyStart,
|
||||
UploadId: &uploadId,
|
||||
PartNumber: &partNumber,
|
||||
ContentLength: &contentLength,
|
||||
PartNumber: int32(partNumber),
|
||||
ContentLength: contentLength,
|
||||
Body: body,
|
||||
})
|
||||
ctx.Response().Header.Set("Etag", etag)
|
||||
@@ -603,21 +578,20 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
|
||||
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
|
||||
var mtime *time.Time
|
||||
var umtime *time.Time
|
||||
var mtime time.Time
|
||||
var err error
|
||||
if copySrcModifSince != "" {
|
||||
tm, err := time.Parse(iso8601Format, copySrcModifSince)
|
||||
mtime, err = time.Parse(iso8601Format, copySrcModifSince)
|
||||
if err != nil {
|
||||
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidCopySource), &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
mtime = &tm
|
||||
}
|
||||
if copySrcUnmodifSince != "" {
|
||||
tm, err := time.Parse(iso8601Format, copySrcUnmodifSince)
|
||||
var umtime time.Time
|
||||
if copySrcModifSince != "" {
|
||||
mtime, err = time.Parse(iso8601Format, copySrcUnmodifSince)
|
||||
if err != nil {
|
||||
return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidCopySource), &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
umtime = &tm
|
||||
}
|
||||
|
||||
metadata := utils.GetUserMetaData(&ctx.Request().Header)
|
||||
@@ -628,8 +602,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
|
||||
CopySource: ©Source,
|
||||
CopySourceIfMatch: ©SrcIfMatch,
|
||||
CopySourceIfNoneMatch: ©SrcIfNoneMatch,
|
||||
CopySourceIfModifiedSince: mtime,
|
||||
CopySourceIfUnmodifiedSince: umtime,
|
||||
CopySourceIfModifiedSince: &mtime,
|
||||
CopySourceIfUnmodifiedSince: &umtime,
|
||||
ExpectedBucketOwner: &acct.Access,
|
||||
Metadata: metadata,
|
||||
})
|
||||
@@ -663,21 +637,13 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
|
||||
return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutObject", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
|
||||
var body io.Reader
|
||||
bodyi := ctx.Locals("body-reader")
|
||||
if bodyi != nil {
|
||||
body = bodyi.(io.Reader)
|
||||
} else {
|
||||
body = bytes.NewReader([]byte{})
|
||||
}
|
||||
|
||||
ctx.Locals("logReqBody", false)
|
||||
etag, err := c.be.PutObject(ctx.Context(), &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &keyStart,
|
||||
ContentLength: &contentLength,
|
||||
ContentLength: contentLength,
|
||||
Metadata: metadata,
|
||||
Body: body,
|
||||
Body: bytes.NewReader(ctx.Request().Body()),
|
||||
Tagging: &tagging,
|
||||
})
|
||||
ctx.Response().Header.Set("ETag", etag)
|
||||
@@ -839,7 +805,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
|
||||
utils.SetResponseHeaders(ctx, []utils.CustomHeader{
|
||||
{
|
||||
Key: "Content-Length",
|
||||
Value: fmt.Sprint(getint64(res.ContentLength)),
|
||||
Value: fmt.Sprint(res.ContentLength),
|
||||
},
|
||||
{
|
||||
Key: "Content-Type",
|
||||
@@ -927,7 +893,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
|
||||
return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "SelectObjectContent", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
|
||||
sw := c.be.SelectObjectContent(ctx.Context(), &s3.SelectObjectContentInput{
|
||||
res, err := c.be.SelectObjectContent(ctx.Context(), &s3.SelectObjectContentInput{
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
Expression: payload.Expression,
|
||||
@@ -937,10 +903,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
|
||||
RequestProgress: payload.RequestProgress,
|
||||
ScanRange: payload.ScanRange,
|
||||
})
|
||||
|
||||
ctx.Context().SetBodyStreamWriter(sw)
|
||||
|
||||
return nil
|
||||
return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "SelectObjectContent", BucketOwner: parsedAcl.Owner})
|
||||
}
|
||||
|
||||
if uploadId != "" {
|
||||
@@ -1017,10 +980,10 @@ func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error {
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
var apierr s3err.APIError
|
||||
if errors.As(err, &apierr) {
|
||||
ctx.Status(apierr.HTTPStatusCode)
|
||||
return ctx.Send(s3err.GetAPIErrorResponse(apierr, "", "", ""))
|
||||
serr, ok := err.(s3err.APIError)
|
||||
if ok {
|
||||
ctx.Status(serr.HTTPStatusCode)
|
||||
return ctx.Send(s3err.GetAPIErrorResponse(serr, "", "", ""))
|
||||
}
|
||||
|
||||
log.Printf("Internal Error, %v", err)
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -175,7 +174,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
app := fiber.New()
|
||||
contentLength := int64(1000)
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
|
||||
@@ -196,7 +194,7 @@ func TestS3ApiController_GetActions(t *testing.T) {
|
||||
ContentType: getPtr("application/xml"),
|
||||
ContentEncoding: getPtr("gzip"),
|
||||
ETag: getPtr("98sda7f97sa9df798sd79f8as9df"),
|
||||
ContentLength: &contentLength,
|
||||
ContentLength: 1000,
|
||||
LastModified: &now,
|
||||
StorageClass: "storage class",
|
||||
}, nil
|
||||
@@ -1200,7 +1198,6 @@ func TestS3ApiController_HeadObject(t *testing.T) {
|
||||
contentType := "application/xml"
|
||||
eTag := "Valid etag"
|
||||
lastModifie := time.Now()
|
||||
contentLength := int64(64)
|
||||
|
||||
s3ApiController := S3ApiController{
|
||||
be: &BackendMock{
|
||||
@@ -1210,7 +1207,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
|
||||
HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
|
||||
return &s3.HeadObjectOutput{
|
||||
ContentEncoding: &contentEncoding,
|
||||
ContentLength: &contentLength,
|
||||
ContentLength: 64,
|
||||
ContentType: &contentType,
|
||||
LastModified: &lastModifie,
|
||||
ETag: &eTag,
|
||||
@@ -1309,8 +1306,8 @@ func TestS3ApiController_CreateActions(t *testing.T) {
|
||||
CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
|
||||
return &s3.CreateMultipartUploadOutput{}, nil
|
||||
},
|
||||
SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) {
|
||||
return func(w *bufio.Writer) {}
|
||||
SelectObjectContentFunc: func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
|
||||
return s3response.SelectObjectContentResult{}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -18,11 +18,15 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/smithy-go/logging"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/auth"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
@@ -33,6 +37,7 @@ import (
|
||||
|
||||
const (
|
||||
iso8601Format = "20060102T150405Z"
|
||||
YYYYMMDD = "20060102"
|
||||
)
|
||||
|
||||
type RootUserConfig struct {
|
||||
@@ -48,93 +53,137 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
|
||||
ctx.Locals("startTime", time.Now())
|
||||
authorization := ctx.Get("Authorization")
|
||||
if authorization == "" {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger)
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
authData, err := utils.ParseAuthorization(authorization)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
// Check the signature version
|
||||
authParts := strings.Split(authorization, ",")
|
||||
for i, el := range authParts {
|
||||
authParts[i] = strings.TrimSpace(el)
|
||||
}
|
||||
|
||||
if authData.Algorithm != "AWS4-HMAC-SHA256" {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger)
|
||||
if len(authParts) != 3 {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
if authData.Region != region {
|
||||
return sendResponse(ctx, s3err.APIError{
|
||||
startParts := strings.Split(authParts[0], " ")
|
||||
|
||||
if startParts[0] != "AWS4-HMAC-SHA256" {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
credKv := strings.Split(startParts[1], "=")
|
||||
if len(credKv) != 2 {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
// Credential variables validation
|
||||
creds := strings.Split(credKv[1], "/")
|
||||
if len(creds) != 5 {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
if creds[4] != "aws4_request" {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureTerminationStr), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
if creds[3] != "s3" {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureIncorrService), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
if creds[2] != region {
|
||||
return controllers.SendResponse(ctx, s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region),
|
||||
Description: fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}, logger)
|
||||
}, &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
ctx.Locals("isRoot", authData.Access == root.Access)
|
||||
ctx.Locals("isRoot", creds[0] == root.Access)
|
||||
|
||||
account, err := acct.getAccount(authData.Access)
|
||||
_, err := time.Parse(YYYYMMDD, creds[1])
|
||||
if err != nil {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
signHdrKv := strings.Split(authParts[1], "=")
|
||||
if len(signHdrKv) != 2 {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
signedHdrs := strings.Split(signHdrKv[1], ";")
|
||||
|
||||
account, err := acct.getAccount(creds[0])
|
||||
if err == auth.ErrNoSuchUser {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
ctx.Locals("account", account)
|
||||
|
||||
// Check X-Amz-Date header
|
||||
date := ctx.Get("X-Amz-Date")
|
||||
if date == "" {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger)
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
// Parse the date and check the date validity
|
||||
tdate, err := time.Parse(iso8601Format, date)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger)
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
if date[:8] != authData.Date {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger)
|
||||
if date[:8] != creds[1] {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
// Validate the dates difference
|
||||
err = validateDate(tdate)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
if utils.IsBigDataAction(ctx) {
|
||||
// for streaming PUT actions, authorization is deferred
|
||||
// until end of stream due to need to get length and
|
||||
// checksum of the stream to validate authorization
|
||||
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
|
||||
return utils.NewAuthReader(ctx, r, authData, account.Secret, debug)
|
||||
})
|
||||
return ctx.Next()
|
||||
}
|
||||
hashPayloadHeader := ctx.Get("X-Amz-Content-Sha256")
|
||||
ok := isSpecialPayload(hashPayloadHeader)
|
||||
|
||||
hashPayload := ctx.Get("X-Amz-Content-Sha256")
|
||||
if !utils.IsSpecialPayload(hashPayload) {
|
||||
if !ok {
|
||||
// Calculate the hash of the request payload
|
||||
hashedPayload := sha256.Sum256(ctx.Body())
|
||||
hexPayload := hex.EncodeToString(hashedPayload[:])
|
||||
|
||||
// Compare the calculated hash with the hash provided
|
||||
if hashPayload != hexPayload {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger)
|
||||
if hashPayloadHeader != hexPayload {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
}
|
||||
|
||||
var contentLength int64
|
||||
contentLengthStr := ctx.Get("Content-Length")
|
||||
if contentLengthStr != "" {
|
||||
contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger)
|
||||
}
|
||||
}
|
||||
|
||||
err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
|
||||
// Create a new http request instance from fasthttp request
|
||||
req, err := utils.CreateHttpRequestFromCtx(ctx, signedHdrs)
|
||||
if err != nil {
|
||||
return sendResponse(ctx, err, logger)
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{
|
||||
AccessKeyID: creds[0],
|
||||
SecretAccessKey: account.Secret,
|
||||
}, req, hashPayloadHeader, creds[3], region, tdate, func(options *v4.SignerOptions) {
|
||||
options.DisableURIPathEscaping = true
|
||||
if debug {
|
||||
options.LogSigning = true
|
||||
options.Logger = logging.NewStandardLogger(os.Stderr)
|
||||
}
|
||||
})
|
||||
if signErr != nil {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
parts := strings.Split(req.Header.Get("Authorization"), " ")
|
||||
if len(parts) < 4 {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
calculatedSign := strings.Split(parts[3], "=")[1]
|
||||
expectedSign := strings.Split(authParts[2], "=")[1]
|
||||
|
||||
if expectedSign != calculatedSign {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch), &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
return ctx.Next()
|
||||
@@ -158,29 +207,39 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
|
||||
return a.iam.GetUserAccount(access)
|
||||
}
|
||||
|
||||
func isSpecialPayload(str string) bool {
|
||||
specialValues := map[string]bool{
|
||||
"UNSIGNED-PAYLOAD": true,
|
||||
"STREAMING-UNSIGNED-PAYLOAD-TRAILER": true,
|
||||
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD": true,
|
||||
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER": true,
|
||||
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD": true,
|
||||
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
|
||||
}
|
||||
|
||||
return specialValues[str]
|
||||
}
|
||||
|
||||
func validateDate(date time.Time) error {
|
||||
now := time.Now().UTC()
|
||||
diff := date.Unix() - now.Unix()
|
||||
|
||||
// Checks the dates difference to be less than a minute
|
||||
if diff > 60 {
|
||||
return s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}
|
||||
}
|
||||
if diff < -60 {
|
||||
return s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
if math.Abs(float64(diff)) > 60 {
|
||||
if diff > 0 {
|
||||
return s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}
|
||||
} else {
|
||||
return s3err.APIError{
|
||||
Code: "SignatureDoesNotMatch",
|
||||
Description: fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
|
||||
return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
|
||||
}
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package middlewares
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
)
|
||||
|
||||
func wrapBodyReader(ctx *fiber.Ctx, wr func(io.Reader) io.Reader) {
|
||||
r, ok := ctx.Locals("body-reader").(io.Reader)
|
||||
if !ok {
|
||||
r = ctx.Request().BodyStream()
|
||||
}
|
||||
|
||||
r = wr(r)
|
||||
ctx.Locals("body-reader", r)
|
||||
}
|
||||
@@ -16,11 +16,10 @@ package middlewares
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"io"
|
||||
"encoding/base64"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/s3api/controllers"
|
||||
"github.com/versity/versitygw/s3api/utils"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"github.com/versity/versitygw/s3log"
|
||||
)
|
||||
@@ -32,16 +31,8 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
if utils.IsBigDataAction(ctx) {
|
||||
wrapBodyReader(ctx, func(r io.Reader) io.Reader {
|
||||
r, _ = utils.NewHashReader(r, incomingSum, utils.HashTypeMd5)
|
||||
return r
|
||||
})
|
||||
return ctx.Next()
|
||||
}
|
||||
|
||||
sum := md5.Sum(ctx.Body())
|
||||
calculatedSum := utils.Md5SumString(sum[:])
|
||||
calculatedSum := base64.StdEncoding.EncodeToString(sum[:])
|
||||
|
||||
if incomingSum != calculatedSum {
|
||||
return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})
|
||||
|
||||
@@ -32,7 +32,6 @@ type S3ApiServer struct {
|
||||
router *S3ApiRouter
|
||||
port string
|
||||
cert *tls.Certificate
|
||||
quiet bool
|
||||
debug bool
|
||||
}
|
||||
|
||||
@@ -49,9 +48,7 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
|
||||
}
|
||||
|
||||
// Logging middlewares
|
||||
if !server.quiet {
|
||||
app.Use(logger.New())
|
||||
}
|
||||
app.Use(logger.New())
|
||||
app.Use(middlewares.DecodeURL(l))
|
||||
app.Use(middlewares.RequestLogger(server.debug))
|
||||
|
||||
@@ -83,11 +80,6 @@ func WithDebug() Option {
|
||||
return func(s *S3ApiServer) { s.debug = true }
|
||||
}
|
||||
|
||||
// WithQuiet silences default logging output
|
||||
func WithQuiet() Option {
|
||||
return func(s *S3ApiServer) { s.quiet = true }
|
||||
}
|
||||
|
||||
func (sa *S3ApiServer) Serve() (err error) {
|
||||
if sa.cert != nil {
|
||||
return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)
|
||||
|
||||
@@ -1,261 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/smithy-go/logging"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
const (
|
||||
iso8601Format = "20060102T150405Z"
|
||||
yyyymmdd = "20060102"
|
||||
)
|
||||
|
||||
// AuthReader is an io.Reader that validates the request authorization
|
||||
// once the underlying reader returns io.EOF. This is needed for streaming
|
||||
// data requests where the data size and checksum are not known until
|
||||
// the data is completely read.
|
||||
type AuthReader struct {
|
||||
ctx *fiber.Ctx
|
||||
auth AuthData
|
||||
secret string
|
||||
size int
|
||||
r *HashReader
|
||||
debug bool
|
||||
}
|
||||
|
||||
// NewAuthReader initializes an io.Reader that will verify the request
|
||||
// v4 auth when the underlying reader returns io.EOF. This postpones the
|
||||
// authorization check until the reader is consumed. So it is important that
|
||||
// the consumer of this reader checks for the auth errors while reading.
|
||||
func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, debug bool) *AuthReader {
|
||||
var hr *HashReader
|
||||
hashPayload := ctx.Get("X-Amz-Content-Sha256")
|
||||
if !IsSpecialPayload(hashPayload) {
|
||||
hr, _ = NewHashReader(r, "", HashTypeSha256)
|
||||
} else {
|
||||
hr, _ = NewHashReader(r, "", HashTypeNone)
|
||||
}
|
||||
|
||||
return &AuthReader{
|
||||
ctx: ctx,
|
||||
r: hr,
|
||||
auth: auth,
|
||||
secret: secret,
|
||||
debug: debug,
|
||||
}
|
||||
}
|
||||
|
||||
// Read allows *AuthReader to be used as an io.Reader
|
||||
func (ar *AuthReader) Read(p []byte) (int, error) {
|
||||
n, err := ar.r.Read(p)
|
||||
ar.size += n
|
||||
|
||||
if errors.Is(err, io.EOF) {
|
||||
verr := ar.validateSignature()
|
||||
if verr != nil {
|
||||
return n, verr
|
||||
}
|
||||
}
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (ar *AuthReader) validateSignature() error {
|
||||
date := ar.ctx.Get("X-Amz-Date")
|
||||
if date == "" {
|
||||
return s3err.GetAPIError(s3err.ErrMissingDateHeader)
|
||||
}
|
||||
|
||||
hashPayload := ar.ctx.Get("X-Amz-Content-Sha256")
|
||||
if !IsSpecialPayload(hashPayload) {
|
||||
hexPayload := ar.r.Sum()
|
||||
|
||||
// Compare the calculated hash with the hash provided
|
||||
if hashPayload != hexPayload {
|
||||
return s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
|
||||
}
|
||||
}
|
||||
|
||||
// Parse the date and check the date validity
|
||||
tdate, err := time.Parse(iso8601Format, date)
|
||||
if err != nil {
|
||||
return s3err.GetAPIError(s3err.ErrMalformedDate)
|
||||
}
|
||||
|
||||
return CheckValidSignature(ar.ctx, ar.auth, ar.secret, hashPayload, tdate, int64(ar.size), ar.debug)
|
||||
}
|
||||
|
||||
const (
|
||||
service = "s3"
|
||||
)
|
||||
|
||||
// CheckValidSignature validates the ctx v4 auth signature
|
||||
func CheckValidSignature(ctx *fiber.Ctx, auth AuthData, secret, checksum string, tdate time.Time, contentLen int64, debug bool) error {
|
||||
signedHdrs := strings.Split(auth.SignedHeaders, ";")
|
||||
|
||||
// Create a new http request instance from fasthttp request
|
||||
req, err := createHttpRequestFromCtx(ctx, signedHdrs, contentLen)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create http request from context: %w", err)
|
||||
}
|
||||
|
||||
signer := v4.NewSigner()
|
||||
|
||||
signErr := signer.SignHTTP(req.Context(), aws.Credentials{
|
||||
AccessKeyID: auth.Access,
|
||||
SecretAccessKey: secret,
|
||||
}, req, checksum, service, auth.Region, tdate, func(options *v4.SignerOptions) {
|
||||
options.DisableURIPathEscaping = true
|
||||
if debug {
|
||||
options.LogSigning = true
|
||||
options.Logger = logging.NewStandardLogger(os.Stderr)
|
||||
}
|
||||
})
|
||||
if signErr != nil {
|
||||
return fmt.Errorf("sign generated http request: %w", err)
|
||||
}
|
||||
|
||||
genAuth, err := ParseAuthorization(req.Header.Get("Authorization"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if auth.Signature != genAuth.Signature {
|
||||
return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AuthData is the parsed authorization data from the header
|
||||
type AuthData struct {
|
||||
Algorithm string
|
||||
Access string
|
||||
Region string
|
||||
SignedHeaders string
|
||||
Signature string
|
||||
Date string
|
||||
}
|
||||
|
||||
// ParseAuthorization returns the parsed fields for the aws v4 auth header
|
||||
// example authorization string from aws docs:
|
||||
// Authorization: AWS4-HMAC-SHA256
|
||||
// Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,
|
||||
// SignedHeaders=host;range;x-amz-date,
|
||||
// Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024
|
||||
func ParseAuthorization(authorization string) (AuthData, error) {
|
||||
a := AuthData{}
|
||||
|
||||
// authorization must start with:
|
||||
// Authorization: <ALGORITHM>
|
||||
// followed by key=value pairs separated by ","
|
||||
authParts := strings.Fields(authorization)
|
||||
for i, el := range authParts {
|
||||
authParts[i] = strings.TrimSpace(el)
|
||||
}
|
||||
|
||||
if len(authParts) < 3 {
|
||||
return a, s3err.GetAPIError(s3err.ErrMissingFields)
|
||||
}
|
||||
|
||||
algo := authParts[0]
|
||||
|
||||
kvData := strings.Join(authParts[1:], "")
|
||||
kvPairs := strings.Split(kvData, ",")
|
||||
// we are expecting at least Credential, SignedHeaders, and Signature
|
||||
// key value pairs here
|
||||
if len(kvPairs) < 3 {
|
||||
return a, s3err.GetAPIError(s3err.ErrMissingFields)
|
||||
}
|
||||
|
||||
var access, region, signedHeaders, signature, date string
|
||||
|
||||
for _, kv := range kvPairs {
|
||||
keyValue := strings.Split(kv, "=")
|
||||
if len(keyValue) != 2 {
|
||||
switch {
|
||||
case strings.HasPrefix(kv, "Credential"):
|
||||
return a, s3err.GetAPIError(s3err.ErrCredMalformed)
|
||||
case strings.HasPrefix(kv, "SignedHeaders"):
|
||||
return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
|
||||
}
|
||||
return a, s3err.GetAPIError(s3err.ErrMissingFields)
|
||||
}
|
||||
key := strings.TrimSpace(keyValue[0])
|
||||
value := strings.TrimSpace(keyValue[1])
|
||||
|
||||
switch key {
|
||||
case "Credential":
|
||||
creds := strings.Split(value, "/")
|
||||
if len(creds) != 5 {
|
||||
return a, s3err.GetAPIError(s3err.ErrCredMalformed)
|
||||
}
|
||||
if creds[3] != "s3" {
|
||||
return a, s3err.GetAPIError(s3err.ErrSignatureIncorrService)
|
||||
}
|
||||
if creds[4] != "aws4_request" {
|
||||
return a, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)
|
||||
}
|
||||
_, err := time.Parse(yyyymmdd, creds[1])
|
||||
if err != nil {
|
||||
return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
|
||||
}
|
||||
access = creds[0]
|
||||
date = creds[1]
|
||||
region = creds[2]
|
||||
case "SignedHeaders":
|
||||
signedHeaders = value
|
||||
case "Signature":
|
||||
signature = value
|
||||
}
|
||||
}
|
||||
|
||||
return AuthData{
|
||||
Algorithm: algo,
|
||||
Access: access,
|
||||
Region: region,
|
||||
SignedHeaders: signedHeaders,
|
||||
Signature: signature,
|
||||
Date: date,
|
||||
}, nil
|
||||
}
|
||||
|
||||
var (
|
||||
specialValues = map[string]bool{
|
||||
"UNSIGNED-PAYLOAD": true,
|
||||
"STREAMING-UNSIGNED-PAYLOAD-TRAILER": true,
|
||||
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD": true,
|
||||
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER": true,
|
||||
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD": true,
|
||||
"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
|
||||
}
|
||||
)
|
||||
|
||||
// IsSpecialPayload checks for streaming/unsigned authorization types
|
||||
func IsSpecialPayload(str string) bool {
|
||||
return specialValues[str]
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package utils
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"hash"
|
||||
"io"
|
||||
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
// HashType identifies the checksum algorithm to be used
|
||||
type HashType string
|
||||
|
||||
const (
|
||||
// HashTypeMd5 generates MD5 checksum for the data stream
|
||||
HashTypeMd5 = "md5"
|
||||
// HashTypeSha256 generates SHA256 checksum for the data stream
|
||||
HashTypeSha256 = "sha256"
|
||||
// HashTypeNone is a no-op checksum for the data stream
|
||||
HashTypeNone = "none"
|
||||
)
|
||||
|
||||
// HashReader is an io.Reader that calculates the checksum
|
||||
// as the data is read
|
||||
type HashReader struct {
|
||||
hashType HashType
|
||||
hash hash.Hash
|
||||
r io.Reader
|
||||
sum string
|
||||
}
|
||||
|
||||
var (
|
||||
errInvalidHashType = errors.New("unsupported or invalid checksum type")
|
||||
)
|
||||
|
||||
// NewHashReader intializes an io.Reader from an underlying io.Reader that
|
||||
// calculates the checksum while the reader is being read from. If the
|
||||
// sum provided is not "", the reader will return an error when the underlying
|
||||
// reader returns io.EOF if the checksum does not match the provided expected
|
||||
// checksum. If the provided sum is "", then the Sum() method can still
|
||||
// be used to get the current checksum for the data read so far.
|
||||
func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, error) {
|
||||
var hash hash.Hash
|
||||
switch ht {
|
||||
case HashTypeMd5:
|
||||
hash = md5.New()
|
||||
case HashTypeSha256:
|
||||
hash = sha256.New()
|
||||
case HashTypeNone:
|
||||
hash = noop{}
|
||||
default:
|
||||
return nil, errInvalidHashType
|
||||
}
|
||||
|
||||
return &HashReader{
|
||||
hash: hash,
|
||||
r: r,
|
||||
sum: expectedSum,
|
||||
hashType: ht,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Read allows *HashReader to be used as an io.Reader
|
||||
func (hr *HashReader) Read(p []byte) (int, error) {
|
||||
n, readerr := hr.r.Read(p)
|
||||
_, err := hr.hash.Write(p[:n])
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
if errors.Is(readerr, io.EOF) && hr.sum != "" {
|
||||
switch hr.hashType {
|
||||
case HashTypeMd5:
|
||||
sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil))
|
||||
if sum != hr.sum {
|
||||
return n, s3err.GetAPIError(s3err.ErrInvalidDigest)
|
||||
}
|
||||
case HashTypeSha256:
|
||||
sum := hex.EncodeToString(hr.hash.Sum(nil))
|
||||
if sum != hr.sum {
|
||||
return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
|
||||
}
|
||||
default:
|
||||
return n, errInvalidHashType
|
||||
}
|
||||
}
|
||||
return n, readerr
|
||||
}
|
||||
|
||||
// Sum returns the checksum hash of the data read so far
|
||||
func (hr *HashReader) Sum() string {
|
||||
switch hr.hashType {
|
||||
case HashTypeMd5:
|
||||
return Md5SumString(hr.hash.Sum(nil))
|
||||
case HashTypeSha256:
|
||||
return hex.EncodeToString(hr.hash.Sum(nil))
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// Md5SumString converts the hash bytes to the string checksum value
|
||||
func Md5SumString(b []byte) string {
|
||||
return base64.StdEncoding.EncodeToString(b)
|
||||
}
|
||||
|
||||
type noop struct{}
|
||||
|
||||
func (n noop) Write(p []byte) (int, error) { return 0, nil }
|
||||
func (n noop) Sum(b []byte) []byte { return []byte{} }
|
||||
func (n noop) Reset() {}
|
||||
func (n noop) Size() int { return 0 }
|
||||
func (n noop) BlockSize() int { return 1 }
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
@@ -50,16 +49,10 @@ func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]strin
|
||||
return
|
||||
}
|
||||
|
||||
func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
|
||||
func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Request, error) {
|
||||
req := ctx.Request()
|
||||
var body io.Reader
|
||||
if IsBigDataAction(ctx) {
|
||||
body = req.BodyStream()
|
||||
} else {
|
||||
body = bytes.NewReader(req.Body())
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
|
||||
httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), bytes.NewReader(req.Body()))
|
||||
if err != nil {
|
||||
return nil, errors.New("error in creating an http request")
|
||||
}
|
||||
@@ -76,8 +69,6 @@ func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength
|
||||
// If content length is non 0, then the header will be included
|
||||
if !includeHeader("Content-Length", signedHdrs) {
|
||||
httpReq.ContentLength = 0
|
||||
} else {
|
||||
httpReq.ContentLength = contentLength
|
||||
}
|
||||
|
||||
// Set the Host header
|
||||
@@ -140,12 +131,3 @@ func includeHeader(hdr string, signedHdrs []string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func IsBigDataAction(ctx *fiber.Ctx) bool {
|
||||
if ctx.Method() == http.MethodPut && len(strings.Split(ctx.Path(), "/")) >= 3 {
|
||||
if !ctx.Request().URI().QueryArgs().Has("tagging") && ctx.Get("X-Amz-Copy-Source") == "" && !ctx.Request().URI().QueryArgs().Has("acl") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := createHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"}, 0)
|
||||
got, err := CreateHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"})
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
|
||||
@@ -1,358 +0,0 @@
|
||||
// Copyright 2023 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package s3select
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Protocol definition for messages can be found here:
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTSelectObjectAppendix.html
|
||||
|
||||
var (
|
||||
// From ptotocol def:
|
||||
// Enum indicating the header value type.
|
||||
// For Amazon S3 Select, this is always 7.
|
||||
headerValueType = byte(7)
|
||||
)
|
||||
|
||||
func intToTwoBytes(i int) []byte {
|
||||
return []byte{byte(i >> 8), byte(i)}
|
||||
}
|
||||
|
||||
func generateHeader(messages ...string) []byte {
|
||||
var header []byte
|
||||
|
||||
for i, message := range messages {
|
||||
if i%2 == 1 {
|
||||
header = append(header, headerValueType)
|
||||
header = append(header, intToTwoBytes(len(message))...)
|
||||
} else {
|
||||
header = append(header, byte(len(message)))
|
||||
}
|
||||
header = append(header, message...)
|
||||
}
|
||||
|
||||
return header
|
||||
}
|
||||
|
||||
func generateOctetHeader(message string) []byte {
|
||||
return generateHeader(
|
||||
":message-type",
|
||||
"event",
|
||||
":content-type",
|
||||
"application/octet-stream",
|
||||
":event-type",
|
||||
message)
|
||||
}
|
||||
|
||||
func generateTextHeader(message string) []byte {
|
||||
return generateHeader(
|
||||
":message-type",
|
||||
"event",
|
||||
":content-type",
|
||||
"text/xml",
|
||||
":event-type",
|
||||
message)
|
||||
}
|
||||
|
||||
func generateNoContentHeader(message string) []byte {
|
||||
return generateHeader(
|
||||
":message-type",
|
||||
"event",
|
||||
":event-type",
|
||||
message)
|
||||
}
|
||||
|
||||
const (
|
||||
// 4 bytes total byte len +
|
||||
// 4 bytes headers bytes len +
|
||||
// 4 bytes prelude CRC
|
||||
preludeLen = 12
|
||||
// CRC is uint32
|
||||
msgCrcLen = 4
|
||||
)
|
||||
|
||||
var (
|
||||
recordsHeader = generateOctetHeader("Records")
|
||||
continuationHeader = generateNoContentHeader("Cont")
|
||||
continuationMessage = genMessage(continuationHeader, []byte{})
|
||||
progressHeader = generateTextHeader("Progress")
|
||||
statsHeader = generateTextHeader("Stats")
|
||||
endHeader = generateNoContentHeader("End")
|
||||
endMessage = genMessage(endHeader, []byte{})
|
||||
)
|
||||
|
||||
func uintToBytes(n uint32) []byte {
|
||||
b := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(b, n)
|
||||
return b
|
||||
}
|
||||
|
||||
func generatePrelude(msgLen int, headerLen int) []byte {
|
||||
prelude := make([]byte, 0, preludeLen)
|
||||
|
||||
// 4 bytes total byte len
|
||||
prelude = append(prelude, uintToBytes(uint32(msgLen+headerLen+preludeLen+msgCrcLen))...)
|
||||
// 4 bytes headers bytes len
|
||||
prelude = append(prelude, uintToBytes(uint32(headerLen))...)
|
||||
// 4 bytes prelude CRC
|
||||
prelude = append(prelude, uintToBytes(crc32.ChecksumIEEE(prelude))...)
|
||||
|
||||
return prelude
|
||||
}
|
||||
|
||||
const (
|
||||
maxHeaderSize = 1024 * 1024
|
||||
maxMessageSize = 5 * 1024 * 1024 * 1024
|
||||
)
|
||||
|
||||
func genMessage(header, payload []byte) []byte {
|
||||
var msg []byte
|
||||
// below is always true since the size is validated
|
||||
// in the send record
|
||||
if len(header) <= maxHeaderSize && len(payload) <= maxMessageSize {
|
||||
msglen := preludeLen + len(header) + len(payload) + msgCrcLen
|
||||
msg = make([]byte, 0, msglen)
|
||||
}
|
||||
|
||||
msg = append(msg, generatePrelude(len(payload), len(header))...)
|
||||
msg = append(msg, header...)
|
||||
msg = append(msg, payload...)
|
||||
msg = append(msg, uintToBytes(crc32.ChecksumIEEE(msg))...)
|
||||
|
||||
return msg
|
||||
}
|
||||
|
||||
func genRecordsMessage(payload []byte) []byte {
|
||||
return genMessage(recordsHeader, payload)
|
||||
}
|
||||
|
||||
type progress struct {
|
||||
XMLName xml.Name `xml:"Progress"`
|
||||
BytesScanned int64 `xml:"BytesScanned"`
|
||||
BytesProcessed int64 `xml:"BytesProcessed"`
|
||||
BytesReturned int64 `xml:"BytesReturned"`
|
||||
}
|
||||
|
||||
func genProgressMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte {
|
||||
progress := progress{
|
||||
BytesScanned: bytesScanned,
|
||||
BytesProcessed: bytesProcessed,
|
||||
BytesReturned: bytesReturned,
|
||||
}
|
||||
|
||||
xmlData, _ := xml.MarshalIndent(progress, "", " ")
|
||||
payload := []byte(xml.Header + string(xmlData))
|
||||
return genMessage(progressHeader, payload)
|
||||
}
|
||||
|
||||
type stats struct {
|
||||
XMLName xml.Name `xml:"Stats"`
|
||||
BytesScanned int64 `xml:"BytesScanned"`
|
||||
BytesProcessed int64 `xml:"BytesProcessed"`
|
||||
BytesReturned int64 `xml:"BytesReturned"`
|
||||
}
|
||||
|
||||
func genStatsMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte {
|
||||
stats := stats{
|
||||
BytesScanned: bytesScanned,
|
||||
BytesProcessed: bytesProcessed,
|
||||
BytesReturned: bytesReturned,
|
||||
}
|
||||
|
||||
xmlData, _ := xml.MarshalIndent(stats, "", " ")
|
||||
payload := []byte(xml.Header + string(xmlData))
|
||||
return genMessage(statsHeader, payload)
|
||||
}
|
||||
|
||||
func genErrorMessage(errorCode, errorMessage string) []byte {
|
||||
return genMessage(generateHeader(
|
||||
":error-code",
|
||||
errorCode,
|
||||
":error-message",
|
||||
errorMessage,
|
||||
":message-type",
|
||||
"error",
|
||||
), []byte{})
|
||||
}
|
||||
|
||||
// GetProgress is a callback function that periodically retrieves the current
|
||||
// values for the following if not nil. This is used to send Progress
|
||||
// messages back to client.
|
||||
// BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed).
|
||||
// BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed).
|
||||
type GetProgress func() (bytesScanned int64, bytesProcessed int64)
|
||||
|
||||
type MessageHandler struct {
|
||||
sync.Mutex
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
writer *bufio.Writer
|
||||
data chan []byte
|
||||
getProgress GetProgress
|
||||
stopCh chan bool
|
||||
resetCh chan bool
|
||||
bytesReturned int64
|
||||
}
|
||||
|
||||
// NewMessageHandler creates a new MessageHandler instance and starts the event streaming
|
||||
func NewMessageHandler(ctx context.Context, w *bufio.Writer, getProgressFunc GetProgress) *MessageHandler {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
mh := &MessageHandler{
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
writer: w,
|
||||
data: make(chan []byte),
|
||||
getProgress: getProgressFunc,
|
||||
resetCh: make(chan bool),
|
||||
stopCh: make(chan bool),
|
||||
}
|
||||
|
||||
go mh.sendBackgroundMessages(mh.resetCh, mh.stopCh)
|
||||
return mh
|
||||
}
|
||||
|
||||
func (mh *MessageHandler) write(data []byte) error {
|
||||
mh.Lock()
|
||||
defer mh.Unlock()
|
||||
|
||||
mh.stopCh <- true
|
||||
defer func() { mh.resetCh <- true }()
|
||||
|
||||
_, err := mh.writer.Write(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return mh.writer.Flush()
|
||||
}
|
||||
|
||||
const (
|
||||
continuationInterval = time.Second
|
||||
progressInterval = time.Minute
|
||||
)
|
||||
|
||||
func (mh *MessageHandler) sendBackgroundMessages(resetCh, stopCh <-chan bool) {
|
||||
continuationTicker := time.NewTicker(continuationInterval)
|
||||
defer continuationTicker.Stop()
|
||||
|
||||
var progressTicker *time.Ticker
|
||||
var progressTickerChan <-chan time.Time
|
||||
if mh.getProgress != nil {
|
||||
progressTicker = time.NewTicker(progressInterval)
|
||||
progressTickerChan = progressTicker.C
|
||||
defer progressTicker.Stop()
|
||||
}
|
||||
|
||||
Loop:
|
||||
for {
|
||||
select {
|
||||
case <-mh.ctx.Done():
|
||||
break Loop
|
||||
|
||||
case <-continuationTicker.C:
|
||||
err := mh.write(continuationMessage)
|
||||
if err != nil {
|
||||
mh.cancel()
|
||||
break Loop
|
||||
}
|
||||
|
||||
case <-resetCh:
|
||||
continuationTicker.Reset(continuationInterval)
|
||||
|
||||
case <-stopCh:
|
||||
continuationTicker.Stop()
|
||||
|
||||
case <-progressTickerChan:
|
||||
var bytesScanned, bytesProcessed int64
|
||||
if mh.getProgress != nil {
|
||||
bytesScanned, bytesProcessed = mh.getProgress()
|
||||
}
|
||||
bytesReturned := atomic.LoadInt64(&mh.bytesReturned)
|
||||
err := mh.write(genProgressMessage(bytesScanned, bytesProcessed, bytesReturned))
|
||||
if err != nil {
|
||||
mh.cancel()
|
||||
break Loop
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SendRecord sends a single Records message
|
||||
func (mh *MessageHandler) SendRecord(payload []byte) error {
|
||||
if mh.ctx.Err() != nil {
|
||||
return mh.ctx.Err()
|
||||
}
|
||||
|
||||
if len(payload) > maxMessageSize {
|
||||
return fmt.Errorf("record max size exceeded")
|
||||
}
|
||||
|
||||
err := mh.write(genRecordsMessage(payload))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
atomic.AddInt64(&mh.bytesReturned, int64(len(payload)))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Finish terminates message stream with Stats and End message
|
||||
// generates stats and end message using function args based on:
|
||||
// BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed).
|
||||
// BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed).
|
||||
func (mh *MessageHandler) Finish(bytesScanned, bytesProcessed int64) error {
|
||||
if mh.ctx.Err() != nil {
|
||||
return mh.ctx.Err()
|
||||
}
|
||||
|
||||
bytesReturned := atomic.LoadInt64(&mh.bytesReturned)
|
||||
err := mh.write(genStatsMessage(bytesScanned, bytesProcessed, bytesReturned))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = mh.write(endMessage)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mh.cancel()
|
||||
return nil
|
||||
}
|
||||
|
||||
// FinishWithError terminates event stream with error
|
||||
func (mh *MessageHandler) FinishWithError(errorCode, errorMessage string) error {
|
||||
if mh.ctx.Err() != nil {
|
||||
return mh.ctx.Err()
|
||||
}
|
||||
err := mh.write(genErrorMessage(errorCode, errorMessage))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mh.cancel()
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user