feat: noop backend for testing use like /dev/null

fix: make s3 PutBucketAcl have receiver pointer like others for consi…

.gitignore

@@ -25,6 +25,9 @@ go.work
 # ignore IntelliJ directories
 .idea
 
+# ignore VS code directories
+.vscode
+
 # auto generated VERSION file
 VERSION
 

auth/iam.go

@@ -16,6 +16,7 @@ package auth
 
 import (
 	"errors"
+	"fmt"
 	"time"
 )
 
@@ -43,18 +44,25 @@ type IAMService interface {
 var ErrNoSuchUser = errors.New("user not found")
 
 type Opts struct {
-	Dir            string
-	LDAPServerURL  string
-	LDAPBindDN     string
-	LDAPPassword   string
-	LDAPQueryBase  string
-	LDAPObjClasses string
-	LDAPAccessAtr  string
-	LDAPSecretAtr  string
-	LDAPRoleAtr    string
-	CacheDisable   bool
-	CacheTTL       int
-	CachePrune     int
+	Dir                string
+	LDAPServerURL      string
+	LDAPBindDN         string
+	LDAPPassword       string
+	LDAPQueryBase      string
+	LDAPObjClasses     string
+	LDAPAccessAtr      string
+	LDAPSecretAtr      string
+	LDAPRoleAtr        string
+	S3Access           string
+	S3Secret           string
+	S3Region           string
+	S3Bucket           string
+	S3Endpoint         string
+	S3DisableSSlVerfiy bool
+	S3Debug            bool
+	CacheDisable       bool
+	CacheTTL           int
+	CachePrune         int
 }
 
 func New(o *Opts) (IAMService, error) {
@@ -64,12 +72,20 @@ func New(o *Opts) (IAMService, error) {
 	switch {
 	case o.Dir != "":
 		svc, err = NewInternal(o.Dir)
+		fmt.Printf("initializing internal IAM with %q\n", o.Dir)
 	case o.LDAPServerURL != "":
 		svc, err = NewLDAPService(o.LDAPServerURL, o.LDAPBindDN, o.LDAPPassword,
 			o.LDAPQueryBase, o.LDAPAccessAtr, o.LDAPSecretAtr, o.LDAPRoleAtr,
 			o.LDAPObjClasses)
+		fmt.Printf("initializing LDAP IAM with %q\n", o.LDAPServerURL)
+	case o.S3Endpoint != "":
+		svc, err = NewS3(o.S3Access, o.S3Secret, o.S3Region, o.S3Bucket,
+			o.S3Endpoint, o.S3DisableSSlVerfiy, o.S3Debug)
+		fmt.Printf("initializing S3 IAM with '%v/%v'\n",
+			o.S3Endpoint, o.S3Bucket)
 	default:
 		// if no iam options selected, default to the single user mode
+		fmt.Println("No IAM service configured, enabling single account mode")
 		return IAMServiceSingle{}, nil
 	}
 

auth/iam_s3_object.go

@@ -0,0 +1,263 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package auth
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"sort"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+)
+
+// IAMServiceS3 stores user accounts in an S3 object
+// The endpoint, credentials, bucket, and region are provided
+// from cli configuration.
+// The object format and name is the same as the internal IAM service:
+// coming from iAMConfig and iamFile in iam_internal.
+
+type IAMServiceS3 struct {
+	access        string
+	secret        string
+	region        string
+	bucket        string
+	endpoint      string
+	sslSkipVerify bool
+	debug         bool
+	client        *s3.Client
+}
+
+var _ IAMService = &IAMServiceS3{}
+
+func NewS3(access, secret, region, bucket, endpoint string, sslSkipVerify, debug bool) (*IAMServiceS3, error) {
+	if access == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service access key")
+	}
+	if secret == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service secret key")
+	}
+	if region == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service region")
+	}
+	if bucket == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service bucket")
+	}
+	if endpoint == "" {
+		return nil, fmt.Errorf("must provide s3 IAM service endpoint")
+	}
+
+	i := &IAMServiceS3{
+		access:        access,
+		secret:        secret,
+		region:        region,
+		bucket:        bucket,
+		endpoint:      endpoint,
+		sslSkipVerify: sslSkipVerify,
+		debug:         debug,
+	}
+
+	cfg, err := i.getConfig()
+	if err != nil {
+		return nil, fmt.Errorf("init s3 IAM: %v", err)
+	}
+
+	i.client = s3.NewFromConfig(cfg)
+	return i, nil
+}
+
+func (s *IAMServiceS3) CreateAccount(account Account) error {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	_, ok := conf.AccessAccounts[account.Access]
+	if ok {
+		return fmt.Errorf("account already exists")
+	}
+	conf.AccessAccounts[account.Access] = account
+
+	return s.storeAccts(conf)
+}
+
+func (s *IAMServiceS3) GetUserAccount(access string) (Account, error) {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return Account{}, err
+	}
+
+	acct, ok := conf.AccessAccounts[access]
+	if !ok {
+		return Account{}, ErrNoSuchUser
+	}
+
+	return acct, nil
+}
+
+func (s *IAMServiceS3) DeleteUserAccount(access string) error {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return err
+	}
+
+	_, ok := conf.AccessAccounts[access]
+	if !ok {
+		return fmt.Errorf("account does not exist")
+	}
+	delete(conf.AccessAccounts, access)
+
+	return s.storeAccts(conf)
+}
+
+func (s *IAMServiceS3) ListUserAccounts() ([]Account, error) {
+	conf, err := s.getAccounts()
+	if err != nil {
+		return nil, err
+	}
+
+	keys := make([]string, 0, len(conf.AccessAccounts))
+	for k := range conf.AccessAccounts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var accs []Account
+	for _, k := range keys {
+		accs = append(accs, Account{
+			Access:    k,
+			Secret:    conf.AccessAccounts[k].Secret,
+			Role:      conf.AccessAccounts[k].Role,
+			UserID:    conf.AccessAccounts[k].UserID,
+			GroupID:   conf.AccessAccounts[k].GroupID,
+			ProjectID: conf.AccessAccounts[k].ProjectID,
+		})
+	}
+
+	return accs, nil
+}
+
+// ResolveEndpoint is used for on prem or non-aws endpoints
+func (s *IAMServiceS3) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+	return aws.Endpoint{
+		PartitionID:       "aws",
+		URL:               s.endpoint,
+		SigningRegion:     s.region,
+		HostnameImmutable: true,
+	}, nil
+}
+
+func (s *IAMServiceS3) Shutdown() error {
+	return nil
+}
+
+func (s *IAMServiceS3) getConfig() (aws.Config, error) {
+	creds := credentials.NewStaticCredentialsProvider(s.access, s.secret, "")
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: s.sslSkipVerify},
+	}
+	client := &http.Client{Transport: tr}
+
+	opts := []func(*config.LoadOptions) error{
+		config.WithRegion(s.region),
+		config.WithCredentialsProvider(creds),
+		config.WithHTTPClient(client),
+	}
+
+	if s.endpoint != "" {
+		opts = append(opts,
+			config.WithEndpointResolverWithOptions(s))
+	}
+
+	if s.debug {
+		opts = append(opts,
+			config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))
+	}
+
+	return config.LoadDefaultConfig(context.Background(), opts...)
+}
+
+func (s *IAMServiceS3) getAccounts() (iAMConfig, error) {
+	obj := iamFile
+
+	out, err := s.client.GetObject(context.Background(), &s3.GetObjectInput{
+		Bucket: &s.bucket,
+		Key:    &obj,
+	})
+	if err != nil {
+		// if the error is object not exists,
+		// init empty accounts stuct and return that
+		var nsk *types.NoSuchKey
+		if errors.As(err, &nsk) {
+			return iAMConfig{}, nil
+		}
+		var apiErr smithy.APIError
+		if errors.As(err, &apiErr) {
+			if apiErr.ErrorCode() == "NotFound" {
+				return iAMConfig{}, nil
+			}
+		}
+
+		// all other errors, return the error
+		return iAMConfig{}, fmt.Errorf("get %v: %w", obj, err)
+	}
+
+	defer out.Body.Close()
+
+	b, err := io.ReadAll(out.Body)
+	if err != nil {
+		return iAMConfig{}, fmt.Errorf("read %v: %w", obj, err)
+	}
+
+	conf, err := parseIAM(b)
+	if err != nil {
+		return iAMConfig{}, fmt.Errorf("parse iam data: %w", err)
+	}
+
+	return conf, nil
+}
+
+func (s *IAMServiceS3) storeAccts(conf iAMConfig) error {
+	b, err := json.Marshal(conf)
+	if err != nil {
+		return fmt.Errorf("failed to serialize iam: %w", err)
+	}
+
+	obj := iamFile
+	uploader := manager.NewUploader(s.client)
+	upinfo := &s3.PutObjectInput{
+		Body:   bytes.NewReader(b),
+		Bucket: &s.bucket,
+		Key:    &obj,
+	}
+	_, err = uploader.Upload(context.Background(), upinfo)
+	if err != nil {
+		return fmt.Errorf("store accounts in %v: %w", iamFile, err)
+	}
+
+	return nil
+}

backend/noop/noop.go

@@ -0,0 +1,232 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package noop
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"io"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/versity/versitygw/backend"
+	"github.com/versity/versitygw/s3response"
+)
+
+type NoOp struct {
+	backend.BackendUnsupported
+	emptyETag string
+}
+
+func New() (*NoOp, error) {
+	sum := md5.Sum([]byte{})
+	etag := hex.EncodeToString(sum[:])
+
+	return &NoOp{emptyETag: etag}, nil
+}
+
+func (n *NoOp) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
+	return s3response.ListAllMyBucketsResult{
+		Owner: s3response.CanonicalUser{
+			ID: "fakeid",
+		},
+		Buckets: s3response.ListAllMyBucketsList{
+			Bucket: []s3response.ListAllMyBucketsEntry{},
+		},
+	}, nil
+}
+
+func (n *NoOp) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+	return &s3.HeadBucketOutput{}, nil
+}
+
+var (
+	fakeUploadID = "abcdefghijklmnopqrstuvwxyz"
+)
+
+func (n *NoOp) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	return &s3.CreateMultipartUploadOutput{
+		Bucket:   input.Bucket,
+		Key:      input.Key,
+		UploadId: &fakeUploadID,
+	}, nil
+}
+
+var (
+	fakeETag = ""
+)
+
+func (n *NoOp) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	return &s3.CompleteMultipartUploadOutput{
+		Bucket: input.Bucket,
+		ETag:   &n.emptyETag,
+		Key:    input.Key,
+	}, nil
+}
+
+func (n *NoOp) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
+	return nil
+}
+
+func (n *NoOp) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	return s3response.ListMultipartUploadsResult{
+		Bucket:         *input.Bucket,
+		Delimiter:      *input.Delimiter,
+		Prefix:         *input.Prefix,
+		MaxUploads:     int(*input.MaxUploads),
+		Uploads:        []s3response.Upload{},
+		CommonPrefixes: []s3response.CommonPrefix{},
+	}, nil
+}
+
+func (n *NoOp) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	return s3response.ListPartsResult{
+		Bucket:    *input.Bucket,
+		Key:       *input.Key,
+		UploadID:  *input.UploadId,
+		Initiator: s3response.Initiator{},
+		Owner:     s3response.Owner{},
+		MaxParts:  int(*input.MaxParts),
+		Parts:     []s3response.Part{},
+	}, nil
+}
+
+func (n *NoOp) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
+	b := make([]byte, 1048576)
+	for {
+		_, err := input.Body.Read(b)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+	}
+	return n.emptyETag, nil
+}
+
+func (n *NoOp) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	return s3response.CopyObjectResult{
+		ETag: n.emptyETag,
+	}, nil
+}
+
+func (n *NoOp) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
+	b := make([]byte, 1048576)
+	for {
+		_, err := input.Body.Read(b)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+	}
+	return n.emptyETag, nil
+}
+
+func (n *NoOp) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	return &s3.HeadObjectOutput{
+		ETag:         &n.emptyETag,
+		LastModified: backend.GetTimePtr(time.Now()),
+		Metadata:     map[string]string{},
+	}, nil
+}
+
+func (n *NoOp) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
+	return &s3.GetObjectOutput{
+		ETag:         &n.emptyETag,
+		LastModified: backend.GetTimePtr(time.Now()),
+		Metadata:     map[string]string{},
+	}, nil
+}
+
+func (n *NoOp) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+	return &s3.GetObjectAttributesOutput{
+		ETag:         &n.emptyETag,
+		LastModified: backend.GetTimePtr(time.Now()),
+		ObjectSize:   new(int64),
+	}, nil
+}
+
+func (n *NoOp) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	return &s3.CopyObjectOutput{
+		CopyObjectResult: &types.CopyObjectResult{
+			ETag: &n.emptyETag,
+		},
+	}, nil
+}
+
+func (n *NoOp) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	return &s3.ListObjectsOutput{
+		CommonPrefixes: []types.CommonPrefix{},
+		Contents:       []types.Object{},
+		Delimiter:      input.Delimiter,
+		IsTruncated:    new(bool),
+		Marker:         new(string),
+		MaxKeys:        input.MaxKeys,
+		Name:           input.Bucket,
+		NextMarker:     new(string),
+		Prefix:         input.Prefix,
+	}, nil
+}
+
+func (n *NoOp) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	return &s3.ListObjectsV2Output{
+		CommonPrefixes:        []types.CommonPrefix{},
+		Contents:              []types.Object{},
+		Delimiter:             input.Delimiter,
+		IsTruncated:           new(bool),
+		ContinuationToken:     new(string),
+		MaxKeys:               input.MaxKeys,
+		Name:                  input.Bucket,
+		NextContinuationToken: new(string),
+		Prefix:                input.Prefix,
+	}, nil
+}
+
+func (n *NoOp) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
+	return nil
+}
+
+func (n *NoOp) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	return s3response.DeleteObjectsResult{
+		Deleted: []types.DeletedObject{},
+		Error:   []types.Error{},
+	}, nil
+}
+
+func (n *NoOp) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
+	return []byte{}, nil
+}
+
+func (n *NoOp) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
+	return nil
+}
+
+func (n *NoOp) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
+	return nil
+}
+
+func (n *NoOp) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
+	tags := make(map[string]string)
+	return tags, nil
+}
+
+func (n *NoOp) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
+	return nil
+}

backend/s3proxy/client.go

@@ -27,7 +27,7 @@ import (
 	"github.com/aws/smithy-go/middleware"
 )
 
-func (s *S3be) getClientFromCtx(ctx context.Context) (*s3.Client, error) {
+func (s *S3Proxy) getClientWithCtx(ctx context.Context) (*s3.Client, error) {
 	cfg, err := s.getConfig(ctx, s.access, s.secret)
 	if err != nil {
 		return nil, err
@@ -36,7 +36,7 @@ func (s *S3be) getClientFromCtx(ctx context.Context) (*s3.Client, error) {
 	return s3.NewFromConfig(cfg), nil
 }
 
-func (s *S3be) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
+func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Config, error) {
 	creds := credentials.NewStaticCredentialsProvider(access, secret, "")
 
 	tr := &http.Transport{
@@ -69,7 +69,7 @@ func (s *S3be) getConfig(ctx context.Context, access, secret string) (aws.Config
 }
 
 // ResolveEndpoint is used for on prem or non-aws endpoints
-func (s *S3be) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
+func (s *S3Proxy) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) {
 	return aws.Endpoint{
 		PartitionID:       "aws",
 		URL:               s.endpoint,

backend/s3proxy/s3.go

@@ -16,12 +16,18 @@ package s3proxy
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"strconv"
+	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -32,9 +38,11 @@ import (
 	"github.com/versity/versitygw/s3response"
 )
 
-type S3be struct {
+type S3Proxy struct {
 	backend.BackendUnsupported
 
+	client *s3.Client
+
 	access          string
 	secret          string
 	endpoint        string
@@ -44,8 +52,8 @@ type S3be struct {
 	debug           bool
 }
 
-func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) *S3be {
-	return &S3be{
+func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) (*S3Proxy, error) {
+	s := &S3Proxy{
 		access:          access,
 		secret:          secret,
 		endpoint:        endpoint,
@@ -54,15 +62,16 @@ func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify
 		sslSkipVerify:   sslSkipVerify,
 		debug:           debug,
 	}
+	client, err := s.getClientWithCtx(context.Background())
+	if err != nil {
+		return nil, err
+	}
+	s.client = client
+	return s, nil
 }
 
-func (s *S3be) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return s3response.ListAllMyBucketsResult{}, err
-	}
-
-	output, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
+func (s *S3Proxy) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3response.ListAllMyBucketsResult, error) {
+	output, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{})
 	err = handleError(err)
 	if err != nil {
 		return s3response.ListAllMyBucketsResult{}, err
@@ -86,78 +95,43 @@ func (s *S3be) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3r
 	}, nil
 }
 
-func (s *S3be) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	out, err := client.HeadBucket(ctx, input)
+func (s *S3Proxy) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
+	out, err := s.client.HeadBucket(ctx, input)
 	err = handleError(err)
 
 	return out, err
 }
 
-func (s *S3be) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) error {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return err
-	}
-
-	_, err = client.CreateBucket(ctx, input)
+func (s *S3Proxy) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) error {
+	_, err := s.client.CreateBucket(ctx, input)
 	return handleError(err)
 }
 
-func (s *S3be) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return err
-	}
-
-	_, err = client.DeleteBucket(ctx, input)
+func (s *S3Proxy) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
+	_, err := s.client.DeleteBucket(ctx, input)
 	return handleError(err)
 }
 
-func (s *S3be) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	out, err := client.CreateMultipartUpload(ctx, input)
+func (s *S3Proxy) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
+	out, err := s.client.CreateMultipartUpload(ctx, input)
 	err = handleError(err)
 	return out, err
 }
 
-func (s *S3be) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	out, err := client.CompleteMultipartUpload(ctx, input)
+func (s *S3Proxy) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
+	out, err := s.client.CompleteMultipartUpload(ctx, input)
 	err = handleError(err)
 	return out, err
 }
 
-func (s *S3be) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return err
-	}
-
-	_, err = client.AbortMultipartUpload(ctx, input)
+func (s *S3Proxy) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
+	_, err := s.client.AbortMultipartUpload(ctx, input)
 	err = handleError(err)
 	return err
 }
 
-func (s *S3be) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return s3response.ListMultipartUploadsResult{}, err
-	}
-
-	output, err := client.ListMultipartUploads(ctx, input)
+func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
+	output, err := s.client.ListMultipartUploads(ctx, input)
 	err = handleError(err)
 	if err != nil {
 		return s3response.ListMultipartUploadsResult{}, err
@@ -204,13 +178,8 @@ func (s *S3be) ListMultipartUploads(ctx context.Context, input *s3.ListMultipart
 	}, nil
 }
 
-func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return s3response.ListPartsResult{}, err
-	}
-
-	output, err := client.ListParts(ctx, input)
+func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
+	output, err := s.client.ListParts(ctx, input)
 	err = handleError(err)
 	if err != nil {
 		return s3response.ListPartsResult{}, err
@@ -258,13 +227,12 @@ func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3respo
 	}, nil
 }
 
-func (s *S3be) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return "", err
-	}
-
-	output, err := client.UploadPart(ctx, input)
+func (s *S3Proxy) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) {
+	// streaming backend is not seekable,
+	// use unsigned payload for streaming ops
+	output, err := s.client.UploadPart(ctx, input, s3.WithAPIOptions(
+		v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
+	))
 	err = handleError(err)
 	if err != nil {
 		return "", err
@@ -273,13 +241,8 @@ func (s *S3be) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag
 	return *output.ETag, nil
 }
 
-func (s *S3be) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return s3response.CopyObjectResult{}, err
-	}
-
-	output, err := client.UploadPartCopy(ctx, input)
+func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
+	output, err := s.client.UploadPartCopy(ctx, input)
 	err = handleError(err)
 	if err != nil {
 		return s3response.CopyObjectResult{}, err
@@ -291,13 +254,12 @@ func (s *S3be) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput
 	}, nil
 }
 
-func (s *S3be) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return "", err
-	}
-
-	output, err := client.PutObject(ctx, input)
+func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (string, error) {
+	// streaming backend is not seekable,
+	// use unsigned payload for streaming ops
+	output, err := s.client.PutObject(ctx, input, s3.WithAPIOptions(
+		v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware,
+	))
 	err = handleError(err)
 	if err != nil {
 		return "", err
@@ -306,25 +268,15 @@ func (s *S3be) PutObject(ctx context.Context, input *s3.PutObjectInput) (string,
 	return *output.ETag, nil
 }
 
-func (s *S3be) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	out, err := client.HeadObject(ctx, input)
+func (s *S3Proxy) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	out, err := s.client.HeadObject(ctx, input)
 	err = handleError(err)
 
 	return out, err
 }
 
-func (s *S3be) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	output, err := client.GetObject(ctx, input)
+func (s *S3Proxy) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
+	output, err := s.client.GetObject(ctx, input)
 	err = handleError(err)
 	if err != nil {
 		return nil, err
@@ -339,71 +291,45 @@ func (s *S3be) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Wri
 	return output, nil
 }
 
-func (s *S3be) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	out, err := client.GetObjectAttributes(ctx, input)
+func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) {
+	out, err := s.client.GetObjectAttributes(ctx, input)
 	err = handleError(err)
 
 	return out, err
 }
 
-func (s *S3be) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	out, err := client.CopyObject(ctx, input)
+func (s *S3Proxy) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
+	out, err := s.client.CopyObject(ctx, input)
 	err = handleError(err)
 
 	return out, err
 }
 
-func (s *S3be) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	out, err := client.ListObjects(ctx, input)
+func (s *S3Proxy) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	out, err := s.client.ListObjects(ctx, input)
 	err = handleError(err)
 
 	return out, err
 }
 
-func (s *S3be) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	out, err := client.ListObjectsV2(ctx, input)
+func (s *S3Proxy) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	out, err := s.client.ListObjectsV2(ctx, input)
 	err = handleError(err)
 
 	return out, err
 }
 
-func (s *S3be) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return err
-	}
-
-	_, err = client.DeleteObject(ctx, input)
+func (s *S3Proxy) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
+	_, err := s.client.DeleteObject(ctx, input)
 	return handleError(err)
 }
 
-func (s *S3be) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return s3response.DeleteObjectsResult{}, err
+func (s *S3Proxy) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
+	if len(input.Delete.Objects) == 0 {
+		input.Delete.Objects = []types.ObjectIdentifier{}
 	}
 
-	output, err := client.DeleteObjects(ctx, input)
+	output, err := s.client.DeleteObjects(ctx, input)
 	err = handleError(err)
 	if err != nil {
 		return s3response.DeleteObjectsResult{}, err
@@ -415,13 +341,8 @@ func (s *S3be) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput)
 	}, nil
 }
 
-func (s *S3be) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	output, err := client.GetBucketAcl(ctx, input)
+func (s *S3Proxy) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
+	output, err := s.client.GetBucketAcl(ctx, input)
 	err = handleError(err)
 	if err != nil {
 		return nil, err
@@ -440,12 +361,7 @@ func (s *S3be) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([
 	return json.Marshal(acl)
 }
 
-func (s S3be) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return err
-	}
-
+func (s *S3Proxy) PutBucketAcl(ctx context.Context, bucket string, data []byte) error {
 	acl, err := auth.ParseACL(data)
 	if err != nil {
 		return err
@@ -462,25 +378,21 @@ func (s S3be) PutBucketAcl(ctx context.Context, bucket string, data []byte) erro
 	}
 
 	for _, el := range acl.Grantees {
+		acc := el.Access
 		input.AccessControlPolicy.Grants = append(input.AccessControlPolicy.Grants, types.Grant{
 			Permission: el.Permission,
 			Grantee: &types.Grantee{
-				ID:   &el.Access,
+				ID:   &acc,
 				Type: types.TypeCanonicalUser,
 			},
 		})
 	}
 
-	_, err = client.PutBucketAcl(ctx, input)
+	_, err = s.client.PutBucketAcl(ctx, input)
 	return handleError(err)
 }
 
-func (s *S3be) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return err
-	}
-
+func (s *S3Proxy) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
 	tagging := &types.Tagging{
 		TagSet: []types.Tag{},
 	}
@@ -491,7 +403,7 @@ func (s *S3be) PutObjectTagging(ctx context.Context, bucket, object string, tags
 		})
 	}
 
-	_, err = client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
+	_, err := s.client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
 		Bucket:  &bucket,
 		Key:     &object,
 		Tagging: tagging,
@@ -499,13 +411,8 @@ func (s *S3be) PutObjectTagging(ctx context.Context, bucket, object string, tags
 	return handleError(err)
 }
 
-func (s *S3be) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	output, err := client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
+func (s *S3Proxy) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
+	output, err := s.client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
 		Bucket: &bucket,
 		Key:    &object,
 	})
@@ -522,19 +429,90 @@ func (s *S3be) GetObjectTagging(ctx context.Context, bucket, object string) (map
 	return tags, nil
 }
 
-func (s *S3be) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
-	client, err := s.getClientFromCtx(ctx)
-	if err != nil {
-		return err
-	}
-
-	_, err = client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
+func (s *S3Proxy) DeleteObjectTagging(ctx context.Context, bucket, object string) error {
+	_, err := s.client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{
 		Bucket: &bucket,
 		Key:    &object,
 	})
 	return handleError(err)
 }
 
+func (s *S3Proxy) ChangeBucketOwner(ctx context.Context, bucket, newOwner string) error {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/change-bucket-owner/?bucket=%v&owner=%v", s.endpoint, bucket, newOwner), nil)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
+	if signErr != nil {
+		return fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	if resp.StatusCode > 300 {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		return fmt.Errorf(string(body))
+	}
+
+	return nil
+}
+
+func (s *S3Proxy) ListBucketsAndOwners(ctx context.Context) ([]s3response.Bucket, error) {
+	req, err := http.NewRequest(http.MethodPatch, fmt.Sprintf("%v/list-buckets", s.endpoint), nil)
+	if err != nil {
+		return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	hashedPayload := sha256.Sum256([]byte{})
+	hexPayload := hex.EncodeToString(hashedPayload[:])
+
+	req.Header.Set("X-Amz-Content-Sha256", hexPayload)
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{AccessKeyID: s.access, SecretAccessKey: s.secret}, req, hexPayload, "s3", s.awsRegion, time.Now())
+	if signErr != nil {
+		return []s3response.Bucket{}, fmt.Errorf("failed to sign the request: %w", err)
+	}
+
+	client := http.Client{}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return []s3response.Bucket{}, fmt.Errorf("failed to send the request: %w", err)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return []s3response.Bucket{}, err
+	}
+	defer resp.Body.Close()
+
+	var buckets []s3response.Bucket
+	if err := json.Unmarshal(body, &buckets); err != nil {
+		return []s3response.Bucket{}, err
+	}
+
+	return buckets, nil
+}
+
 func handleError(err error) error {
 	if err == nil {
 		return nil

cmd/versitygw/main.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"strconv"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/urfave/cli/v2"
@@ -44,10 +43,15 @@ var (
 	logWebhookURL                          string
 	accessLog                              string
 	debug                                  bool
+	quiet                                  bool
 	iamDir                                 string
 	ldapURL, ldapBindDN, ldapPassword      string
 	ldapQueryBase, ldapObjClasses          string
 	ldapAccessAtr, ldapSecAtr, ldapRoleAtr string
+	s3IamAccess, s3IamSecret               string
+	s3IamRegion, s3IamBucket               string
+	s3IamEndpoint                          string
+	s3IamSslNoVerify, s3IamDebug           bool
 	iamCacheDisable                        bool
 	iamCacheTTL                            int
 	iamCachePrune                          int
@@ -71,6 +75,7 @@ func main() {
 		posixCommand(),
 		scoutfsCommand(),
 		s3Command(),
+		noopCommand(),
 		adminCommand(),
 		testCommand(),
 	}
@@ -173,6 +178,12 @@ func initFlags() []cli.Flag {
 			Usage:       "enable debug output",
 			Destination: &debug,
 		},
+		&cli.BoolFlag{
+			Name:        "quiet",
+			Usage:       "silence stdout request logging output",
+			Destination: &quiet,
+			Aliases:     []string{"q"},
+		},
 		&cli.StringFlag{
 			Name:        "access-log",
 			Usage:       "enable server access logging to specified file",
@@ -260,6 +271,42 @@ func initFlags() []cli.Flag {
 			Usage:       "ldap server user role attribute name",
 			Destination: &ldapRoleAtr,
 		},
+		&cli.StringFlag{
+			Name:        "s3-iam-access",
+			Usage:       "s3 IAM access key",
+			Destination: &s3IamAccess,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-secret",
+			Usage:       "s3 IAM secret key",
+			Destination: &s3IamSecret,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-region",
+			Usage:       "s3 IAM region",
+			Destination: &s3IamRegion,
+			Value:       "us-east-1",
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-bucket",
+			Usage:       "s3 IAM bucket",
+			Destination: &s3IamBucket,
+		},
+		&cli.StringFlag{
+			Name:        "s3-iam-endpoint",
+			Usage:       "s3 IAM endpoint",
+			Destination: &s3IamEndpoint,
+		},
+		&cli.BoolFlag{
+			Name:        "s3-iam-noverify",
+			Usage:       "s3 IAM disable ssl verification",
+			Destination: &s3IamSslNoVerify,
+		},
+		&cli.BoolFlag{
+			Name:        "s3-iam-debug",
+			Usage:       "s3 IAM debug output",
+			Destination: &s3IamDebug,
+		},
 		&cli.BoolFlag{
 			Name:        "iam-cache-disable",
 			Usage:       "disable local iam cache",
@@ -281,17 +328,11 @@ func initFlags() []cli.Flag {
 }
 
 func runGateway(ctx context.Context, be backend.Backend) error {
-	// int32 max for 32 bit arch
-	blimit := int64(2*1024*1024*1024 - 1)
-	if strconv.IntSize > 32 {
-		// 5GB max for 64 bit arch
-		blimit = int64(5 * 1024 * 1024 * 1024)
-	}
-
 	app := fiber.New(fiber.Config{
-		AppName:      "versitygw",
-		ServerHeader: "VERSITYGW",
-		BodyLimit:    int(blimit),
+		AppName:           "versitygw",
+		ServerHeader:      "VERSITYGW",
+		StreamRequestBody: true,
+		DisableKeepalive:  true,
 	})
 
 	var opts []s3api.Option
@@ -316,6 +357,9 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	if admPort == "" {
 		opts = append(opts, s3api.WithAdminServer())
 	}
+	if quiet {
+		opts = append(opts, s3api.WithQuiet())
+	}
 
 	admApp := fiber.New(fiber.Config{
 		AppName:      "versitygw",
@@ -340,18 +384,25 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	}
 
 	iam, err := auth.New(&auth.Opts{
-		Dir:            iamDir,
-		LDAPServerURL:  ldapURL,
-		LDAPBindDN:     ldapBindDN,
-		LDAPPassword:   ldapPassword,
-		LDAPQueryBase:  ldapQueryBase,
-		LDAPObjClasses: ldapObjClasses,
-		LDAPAccessAtr:  ldapAccessAtr,
-		LDAPSecretAtr:  ldapSecAtr,
-		LDAPRoleAtr:    ldapRoleAtr,
-		CacheDisable:   iamCacheDisable,
-		CacheTTL:       iamCacheTTL,
-		CachePrune:     iamCachePrune,
+		Dir:                iamDir,
+		LDAPServerURL:      ldapURL,
+		LDAPBindDN:         ldapBindDN,
+		LDAPPassword:       ldapPassword,
+		LDAPQueryBase:      ldapQueryBase,
+		LDAPObjClasses:     ldapObjClasses,
+		LDAPAccessAtr:      ldapAccessAtr,
+		LDAPSecretAtr:      ldapSecAtr,
+		LDAPRoleAtr:        ldapRoleAtr,
+		S3Access:           s3IamAccess,
+		S3Secret:           s3IamSecret,
+		S3Region:           s3IamRegion,
+		S3Bucket:           s3IamBucket,
+		S3Endpoint:         s3IamEndpoint,
+		S3DisableSSlVerfiy: s3IamSslNoVerify,
+		S3Debug:            s3IamDebug,
+		CacheDisable:       iamCacheDisable,
+		CacheTTL:           iamCacheTTL,
+		CachePrune:         iamCachePrune,
 	})
 	if err != nil {
 		return fmt.Errorf("setup iam: %w", err)

cmd/versitygw/noop.go

@@ -0,0 +1,39 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+	"github.com/versity/versitygw/backend/noop"
+)
+
+func noopCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "noop",
+		Usage:       "noop storage backend",
+		Description: `This is a /dev/null backend for testing.`,
+		Action:      runNoOp,
+	}
+}
+
+func runNoOp(ctx *cli.Context) error {
+	be, err := noop.New()
+	if err != nil {
+		return fmt.Errorf("init noop backend: %w", err)
+	}
+	return runGateway(ctx.Context, be)
+}

cmd/versitygw/s3.go

@@ -15,6 +15,8 @@
 package main
 
 import (
+	"fmt"
+
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/backend/s3proxy"
 )
@@ -88,7 +90,10 @@ to an s3 storage backend service.`,
 }
 
 func runS3(ctx *cli.Context) error {
-	be := s3proxy.New(s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
+	be, err := s3proxy.New(s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
 		s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
+	if err != nil {
+		return fmt.Errorf("init s3 backend: %w", err)
+	}
 	return runGateway(ctx.Context, be)
 }

docker-compose.yml

@@ -20,4 +20,4 @@ services:
       - ./:/app
     ports:
       - "${PROXY_PORT}:${PROXY_PORT}"
-    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY -p :$PROXY_PORT s3 --endpoint http://posix:$POSIX_PORT"]
+    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$PROXY_PORT s3 -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --endpoint http://posix:$POSIX_PORT"]

go.mod

@@ -3,15 +3,15 @@ module github.com/versity/versitygw
 go 1.20
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.23.5
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.2
-	github.com/aws/smithy-go v1.18.1
+	github.com/aws/aws-sdk-go-v2 v1.24.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7
+	github.com/aws/smithy-go v1.19.0
 	github.com/go-ldap/ldap/v3 v3.4.6
 	github.com/gofiber/fiber/v2 v2.51.0
-	github.com/google/uuid v1.4.0
+	github.com/google/uuid v1.5.0
 	github.com/nats-io/nats.go v1.31.0
 	github.com/pkg/xattr v0.4.9
-	github.com/segmentio/kafka-go v0.4.46
+	github.com/segmentio/kafka-go v0.4.47
 	github.com/urfave/cli/v2 v2.26.0
 	github.com/valyala/fasthttp v1.51.0
 	github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
@@ -20,33 +20,33 @@ require (
 
 require (
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.18.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.26.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/nats-io/nkeys v0.4.6 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/stretchr/testify v1.8.1 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
 )
 
 require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.25.11
-	github.com/aws/aws-sdk-go-v2/credentials v1.16.9
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.4
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.26.2
+	github.com/aws/aws-sdk-go-v2/credentials v1.16.13
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.9
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect

go.sum

@@ -4,44 +4,44 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3Uu
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/aws/aws-sdk-go-v2 v1.23.5 h1:xK6C4udTyDMd82RFvNkDQxtAd00xlzFUtX4fF2nMZyg=
-github.com/aws/aws-sdk-go-v2 v1.23.5/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 h1:Zx9+31KyB8wQna6SXFWOewlgoY5uGdDAu6PTOEU3OQI=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3/go.mod h1:zxbEJhRdKTH1nqS2qu6UJ7zGe25xaHxZXaC2CvuQFnA=
-github.com/aws/aws-sdk-go-v2/config v1.25.11 h1:RWzp7jhPRliIcACefGkKp03L0Yofmd2p8M25kbiyvno=
-github.com/aws/aws-sdk-go-v2/config v1.25.11/go.mod h1:BVUs0chMdygHsQtvaMyEOpW2GIW+ubrxJLgIz/JU29s=
-github.com/aws/aws-sdk-go-v2/credentials v1.16.9 h1:LQo3MUIOzod9JdUK+wxmSdgzLVYUbII3jXn3S/HJZU0=
-github.com/aws/aws-sdk-go-v2/credentials v1.16.9/go.mod h1:R7mDuIJoCjH6TxGUc/cylE7Lp/o0bhKVoxdBThsjqCM=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 h1:FZVFahMyZle6WcogZCOxo6D/lkDA2lqKIn4/ueUmVXw=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9/go.mod h1:kjq7REMIkxdtcEC9/4BVXjOsNY5isz6jQbEgk6osRTU=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.4 h1:TUCNKBd4/JEefsZDxo5deRmrRRPZHqGyBYiUAeBKOWU=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.4/go.mod h1:egDkcl+zsgFqS6VO142bKboip5Pe1sNMwN55Xy38QsM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 h1:8GVZIR0y6JRIUNSYI1xAMF4HDfV8H/bOsZ/8AD/uY5Q=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8/go.mod h1:rwBfu0SoUkBUZndVgPZKAD9Y2JigaZtRP68unRiYToQ=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 h1:ZE2ds/qeBkhk3yqYvS3CDCFNvd9ir5hMjlVStLZWrvM=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8/go.mod h1:/lAPPymDYL023+TS6DJmjuL42nxix2AvEvfjqOBRODk=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 h1:abKT+RuM1sdCNZIGIfZpLkvxEX3Rpsto019XG/rkYG8=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8/go.mod h1:Owc4ysUE71JSruVTTa3h4f2pp3E4hlcAtmeNXxDmjj4=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 h1:e3PCNeEaev/ZF01cQyNZgmYE9oYYePIMJs2mWSKG514=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3/go.mod h1:gIeeNyaL8tIEqZrzAnTeyhHcE0yysCtcaP+N9kxLZ+E=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 h1:xyfOAYV/ujzZOo01H9+OnyeiRKmTEp6EsITTsmq332Q=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8/go.mod h1:coLeQEoKzW9ViTL2bn0YUlU7K0RYjivKudG74gtd+sI=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 h1:EamsKe+ZjkOQjDdHd86/JCEucjFKQ9T0atWKO4s2Lgs=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8/go.mod h1:Q0vV3/csTpbkfKLI5Sb56cJQTCTtJ0ixdb7P+Wedqiw=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 h1:ip5ia3JOXl4OAsqeTdrOOmqKgoWiu+t9XSOnRzBwmRs=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8/go.mod h1:kE+aERnK9VQIw1vrk7ElAvhCsgLNzGyCPNg2Qe4Eq4c=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.47.2 h1:DLSAG8zpJV2pYsU+UPkj1IEZghyBnnUsvIRs6UuXSDU=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.47.2/go.mod h1:thjZng67jGsvMyVZnSxlcqKyLwB0XTG8bHIRZPTJ+Bs=
-github.com/aws/aws-sdk-go-v2/service/sso v1.18.2 h1:xJPydhNm0Hiqct5TVKEuHG7weC0+sOs4MUnd7A5n5F4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.18.2/go.mod h1:zxk6y1X2KXThESWMS5CrKRvISD8mbIMab6nZrCGxDG0=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.2 h1:8dU9zqA77C5egbU6yd4hFLaiIdPv3rU+6cp7sz5FjCU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.2/go.mod h1:7Lt5mjQ8x5rVdKqg+sKKDeuwoszDJIIPmkd8BVsEdS0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.26.2 h1:fFrLsy08wEbAisqW3KDl/cPHrF43GmV79zXB9EwJiZw=
-github.com/aws/aws-sdk-go-v2/service/sts v1.26.2/go.mod h1:7Ld9eTqocTvJqqJ5K/orbSDwmGcpRdlDiLjz2DO+SL8=
-github.com/aws/smithy-go v1.18.1 h1:pOdBTUfXNazOlxLrgeYalVnuTpKreACHtc62xLwIB3c=
-github.com/aws/smithy-go v1.18.1/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
+github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk=
+github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo=
+github.com/aws/aws-sdk-go-v2/config v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc=
+github.com/aws/aws-sdk-go-v2/config v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.13 h1:WLABQ4Cp4vXtXfOWOS3MEZKr6AAYUpMczLhgKtAjQ/8=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.13/go.mod h1:Qg6x82FXwW0sJHzYruxGiuApNo31UEtJvXVSZAXeWiw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.9 h1:5zA8qVCXMPGt6YneFnll5B157SfdK2SewU85PH9/yM0=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.9/go.mod h1:t4gy210hPxkbtYM8xOzrWdxVq1PyekR76OOKXy3s0Vs=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9/go.mod h1:YD0aYBWCrPENpHolhKw2XDlTIWae2GKXT1T4o6N6hiM=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7 h1:o0ASbVwUAIrfp/WcCac+6jioZt4Hd8k/1X8u7GJ/QeM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.47.7/go.mod h1:vADO6Jn+Rq4nDtfwNjhgR84qkZwiC6FqCaXdw/kYwjA=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 h1:HJeiuZ2fldpd0WqngyMR6KW7ofkXNLyOaHwEIGm39Cs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.6/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU=
+github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
+github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -55,8 +55,8 @@ github.com/gofiber/fiber/v2 v2.51.0 h1:JNACcZy5e2tGApWB2QrRpenTWn0fq0hkFm6k0C86g
 github.com/gofiber/fiber/v2 v2.51.0/go.mod h1:xaQRZQJGqnKOQnbQw+ltvku3/h8QxvNi8o6JiJ7Ll0U=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
-github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -89,8 +89,8 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/segmentio/kafka-go v0.4.46 h1:Sx8/kvtY+/G8nM0roTNnFezSJj3bT2sW0Xy/YY3CgBI=
-github.com/segmentio/kafka-go v0.4.46/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
+github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
+github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -120,8 +120,9 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -160,8 +161,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

integration/group-tests.go

@@ -51,6 +51,7 @@ func TestPutObject(s *S3Conf) {
 	PutObject_special_chars(s)
 	PutObject_invalid_long_tags(s)
 	PutObject_success(s)
+	PutObject_invalid_credentials(s)
 }
 
 func TestHeadObject(s *S3Conf) {
@@ -262,6 +263,7 @@ func GetIntTests() IntTests {
 		"PutObject_special_chars":                             PutObject_special_chars,
 		"PutObject_invalid_long_tags":                         PutObject_invalid_long_tags,
 		"PutObject_success":                                   PutObject_success,
+		"PutObject_invalid_credentials":                       PutObject_invalid_credentials,
 		"HeadObject_non_existing_object":                      HeadObject_non_existing_object,
 		"HeadObject_success":                                  HeadObject_success,
 		"GetObject_non_existing_key":                          GetObject_non_existing_key,

integration/tests.go

@@ -447,7 +447,7 @@ func Authentication_invalid_signed_headers(s *S3Conf) error {
 			return err
 		}
 		defer resp.Body.Close()
-		if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrCredMalformed)); err != nil {
+		if err := checkAuthErr(resp, s3err.GetAPIError(s3err.ErrInvalidQueryParams)); err != nil {
 			return err
 		}
 
@@ -1088,6 +1088,17 @@ func PutObject_success(s *S3Conf) error {
 	})
 }
 
+func PutObject_invalid_credentials(s *S3Conf) error {
+	testName := "PutObject_invalid_credentials"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		newconf := *s
+		newconf.awsSecret = newconf.awsSecret + "badpassword"
+		client := s3.NewFromConfig(newconf.Config())
+		err := putObjects(client, []string{"my-obj"}, bucket)
+		return checkApiErr(err, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch))
+	})
+}
+
 func HeadObject_non_existing_object(s *S3Conf) error {
 	testName := "HeadObject_non_existing_object"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {

runtests.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 # make temp dirs
+rm -rf /tmp/gw
 mkdir /tmp/gw
 rm -rf /tmp/covdata
 mkdir /tmp/covdata

s3api/controllers/base.go

@@ -17,6 +17,7 @@ package controllers
 import (
 	"bytes"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -63,7 +64,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	key := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	uploadId := ctx.Query("uploadId")
-	maxParts := int32(ctx.QueryInt("max-parts", 0))
+	maxParts := int32(ctx.QueryInt("max-parts", -1))
 	partNumberMarker := ctx.Query("part-number-marker")
 	acceptRange := ctx.Get("Range")
 	acct := ctx.Locals("account").(auth.Account)
@@ -92,7 +93,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if uploadId != "" {
-		if maxParts < 0 || (maxParts == 0 && ctx.Query("max-parts") != "") {
+		if maxParts < 0 && ctx.Request().URI().QueryArgs().Has("max-parts") {
 			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts), &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
 		}
 		if partNumberMarker != "" {
@@ -105,13 +106,17 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 		if err := auth.VerifyACL(parsedAcl, acct.Access, "READ", isRoot); err != nil {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
 		}
+		var mxParts *int32
+		if ctx.Request().URI().QueryArgs().Has("max-parts") {
+			mxParts = &maxParts
+		}
 
 		res, err := c.be.ListParts(ctx.Context(), &s3.ListPartsInput{
 			Bucket:           &bucket,
 			Key:              &key,
 			UploadId:         &uploadId,
 			PartNumberMarker: &partNumberMarker,
-			MaxParts:         &maxParts,
+			MaxParts:         mxParts,
 		})
 		return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
 	}
@@ -512,7 +517,14 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
 		}
 
-		body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body())))
+		var body io.Reader
+		bodyi := ctx.Locals("body-reader")
+		if bodyi != nil {
+			body = bodyi.(io.Reader)
+		} else {
+			body = bytes.NewReader([]byte{})
+		}
+
 		ctx.Locals("logReqBody", false)
 		etag, err := c.be.UploadPart(ctx.Context(), &s3.UploadPartInput{
 			Bucket:        &bucket,
@@ -591,20 +603,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
 		}
 
-		var mtime time.Time
-		var err error
+		var mtime *time.Time
+		var umtime *time.Time
 		if copySrcModifSince != "" {
-			mtime, err = time.Parse(iso8601Format, copySrcModifSince)
+			tm, err := time.Parse(iso8601Format, copySrcModifSince)
 			if err != nil {
 				return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidCopySource), &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
 			}
+			mtime = &tm
 		}
-		var umtime time.Time
-		if copySrcModifSince != "" {
-			mtime, err = time.Parse(iso8601Format, copySrcUnmodifSince)
+		if copySrcUnmodifSince != "" {
+			tm, err := time.Parse(iso8601Format, copySrcUnmodifSince)
 			if err != nil {
 				return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidCopySource), &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner})
 			}
+			umtime = &tm
 		}
 
 		metadata := utils.GetUserMetaData(&ctx.Request().Header)
@@ -615,8 +628,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			CopySource:                  &copySource,
 			CopySourceIfMatch:           &copySrcIfMatch,
 			CopySourceIfNoneMatch:       &copySrcIfNoneMatch,
-			CopySourceIfModifiedSince:   &mtime,
-			CopySourceIfUnmodifiedSince: &umtime,
+			CopySourceIfModifiedSince:   mtime,
+			CopySourceIfUnmodifiedSince: umtime,
 			ExpectedBucketOwner:         &acct.Access,
 			Metadata:                    metadata,
 		})
@@ -650,13 +663,21 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 		return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutObject", BucketOwner: parsedAcl.Owner})
 	}
 
+	var body io.Reader
+	bodyi := ctx.Locals("body-reader")
+	if bodyi != nil {
+		body = bodyi.(io.Reader)
+	} else {
+		body = bytes.NewReader([]byte{})
+	}
+
 	ctx.Locals("logReqBody", false)
 	etag, err := c.be.PutObject(ctx.Context(), &s3.PutObjectInput{
 		Bucket:        &bucket,
 		Key:           &keyStart,
 		ContentLength: &contentLength,
 		Metadata:      metadata,
-		Body:          bytes.NewReader(ctx.Request().Body()),
+		Body:          body,
 		Tagging:       &tagging,
 	})
 	ctx.Response().Header.Set("ETag", etag)
@@ -996,10 +1017,10 @@ func SendResponse(ctx *fiber.Ctx, err error, l *MetaOpts) error {
 		})
 	}
 	if err != nil {
-		serr, ok := err.(s3err.APIError)
-		if ok {
-			ctx.Status(serr.HTTPStatusCode)
-			return ctx.Send(s3err.GetAPIErrorResponse(serr, "", "", ""))
+		var apierr s3err.APIError
+		if errors.As(err, &apierr) {
+			ctx.Status(apierr.HTTPStatusCode)
+			return ctx.Send(s3err.GetAPIErrorResponse(apierr, "", "", ""))
 		}
 
 		log.Printf("Internal Error, %v", err)

s3api/middlewares/authentication.go

@@ -18,15 +18,11 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
-	"math"
+	"io"
 	"net/http"
-	"os"
-	"strings"
+	"strconv"
 	"time"
 
-	"github.com/aws/aws-sdk-go-v2/aws"
-	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
-	"github.com/aws/smithy-go/logging"
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3api/controllers"
@@ -37,7 +33,6 @@ import (
 
 const (
 	iso8601Format = "20060102T150405Z"
-	YYYYMMDD      = "20060102"
 )
 
 type RootUserConfig struct {
@@ -53,137 +48,93 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, logger s3log.Au
 		ctx.Locals("startTime", time.Now())
 		authorization := ctx.Get("Authorization")
 		if authorization == "" {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), &controllers.MetaOpts{Logger: logger})
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrAuthHeaderEmpty), logger)
 		}
 
-		// Check the signature version
-		authParts := strings.Split(authorization, ",")
-		for i, el := range authParts {
-			authParts[i] = strings.TrimSpace(el)
+		authData, err := utils.ParseAuthorization(authorization)
+		if err != nil {
+			return sendResponse(ctx, err, logger)
 		}
 
-		if len(authParts) != 3 {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
+		if authData.Algorithm != "AWS4-HMAC-SHA256" {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), logger)
 		}
 
-		startParts := strings.Split(authParts[0], " ")
-
-		if startParts[0] != "AWS4-HMAC-SHA256" {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureVersionNotSupported), &controllers.MetaOpts{Logger: logger})
-		}
-
-		credKv := strings.Split(startParts[1], "=")
-		if len(credKv) != 2 {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
-		}
-		// Credential variables validation
-		creds := strings.Split(credKv[1], "/")
-		if len(creds) != 5 {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
-		}
-		if creds[4] != "aws4_request" {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureTerminationStr), &controllers.MetaOpts{Logger: logger})
-		}
-		if creds[3] != "s3" {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureIncorrService), &controllers.MetaOpts{Logger: logger})
-		}
-		if creds[2] != region {
-			return controllers.SendResponse(ctx, s3err.APIError{
+		if authData.Region != region {
+			return sendResponse(ctx, s3err.APIError{
 				Code:           "SignatureDoesNotMatch",
-				Description:    fmt.Sprintf("Credential should be scoped to a valid Region, not %v", creds[2]),
+				Description:    fmt.Sprintf("Credential should be scoped to a valid Region, not %v", authData.Region),
 				HTTPStatusCode: http.StatusForbidden,
-			}, &controllers.MetaOpts{Logger: logger})
+			}, logger)
 		}
 
-		ctx.Locals("isRoot", creds[0] == root.Access)
+		ctx.Locals("isRoot", authData.Access == root.Access)
 
-		_, err := time.Parse(YYYYMMDD, creds[1])
-		if err != nil {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
-		}
-
-		signHdrKv := strings.Split(authParts[1], "=")
-		if len(signHdrKv) != 2 {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrCredMalformed), &controllers.MetaOpts{Logger: logger})
-		}
-		signedHdrs := strings.Split(signHdrKv[1], ";")
-
-		account, err := acct.getAccount(creds[0])
+		account, err := acct.getAccount(authData.Access)
 		if err == auth.ErrNoSuchUser {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), &controllers.MetaOpts{Logger: logger})
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidAccessKeyID), logger)
 		}
 		if err != nil {
-			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+			return sendResponse(ctx, err, logger)
 		}
 		ctx.Locals("account", account)
 
 		// Check X-Amz-Date header
 		date := ctx.Get("X-Amz-Date")
 		if date == "" {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), &controllers.MetaOpts{Logger: logger})
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingDateHeader), logger)
 		}
 
 		// Parse the date and check the date validity
 		tdate, err := time.Parse(iso8601Format, date)
 		if err != nil {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), &controllers.MetaOpts{Logger: logger})
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrMalformedDate), logger)
 		}
 
-		if date[:8] != creds[1] {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), &controllers.MetaOpts{Logger: logger})
+		if date[:8] != authData.Date {
+			return sendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch), logger)
 		}
 
 		// Validate the dates difference
 		err = validateDate(tdate)
 		if err != nil {
-			return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+			return sendResponse(ctx, err, logger)
 		}
 
-		hashPayloadHeader := ctx.Get("X-Amz-Content-Sha256")
-		ok := isSpecialPayload(hashPayloadHeader)
+		if utils.IsBigDataAction(ctx) {
+			// for streaming PUT actions, authorization is deferred
+			// until end of stream due to need to get length and
+			// checksum of the stream to validate authorization
+			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+				return utils.NewAuthReader(ctx, r, authData, account.Secret, debug)
+			})
+			return ctx.Next()
+		}
 
-		if !ok {
+		hashPayload := ctx.Get("X-Amz-Content-Sha256")
+		if !utils.IsSpecialPayload(hashPayload) {
 			// Calculate the hash of the request payload
 			hashedPayload := sha256.Sum256(ctx.Body())
 			hexPayload := hex.EncodeToString(hashedPayload[:])
 
 			// Compare the calculated hash with the hash provided
-			if hashPayloadHeader != hexPayload {
-				return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), &controllers.MetaOpts{Logger: logger})
+			if hashPayload != hexPayload {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch), logger)
 			}
 		}
 
-		// Create a new http request instance from fasthttp request
-		req, err := utils.CreateHttpRequestFromCtx(ctx, signedHdrs)
+		var contentLength int64
+		contentLengthStr := ctx.Get("Content-Length")
+		if contentLengthStr != "" {
+			contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64)
+			if err != nil {
+				return sendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), logger)
+			}
+		}
+
+		err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, debug)
 		if err != nil {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
-		}
-
-		signer := v4.NewSigner()
-
-		signErr := signer.SignHTTP(req.Context(), aws.Credentials{
-			AccessKeyID:     creds[0],
-			SecretAccessKey: account.Secret,
-		}, req, hashPayloadHeader, creds[3], region, tdate, func(options *v4.SignerOptions) {
-			options.DisableURIPathEscaping = true
-			if debug {
-				options.LogSigning = true
-				options.Logger = logging.NewStandardLogger(os.Stderr)
-			}
-		})
-		if signErr != nil {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInternalError), &controllers.MetaOpts{Logger: logger})
-		}
-
-		parts := strings.Split(req.Header.Get("Authorization"), " ")
-		if len(parts) < 4 {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrMissingFields), &controllers.MetaOpts{Logger: logger})
-		}
-		calculatedSign := strings.Split(parts[3], "=")[1]
-		expectedSign := strings.Split(authParts[2], "=")[1]
-
-		if expectedSign != calculatedSign {
-			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch), &controllers.MetaOpts{Logger: logger})
+			return sendResponse(ctx, err, logger)
 		}
 
 		return ctx.Next()
@@ -207,39 +158,29 @@ func (a accounts) getAccount(access string) (auth.Account, error) {
 	return a.iam.GetUserAccount(access)
 }
 
-func isSpecialPayload(str string) bool {
-	specialValues := map[string]bool{
-		"UNSIGNED-PAYLOAD":                                 true,
-		"STREAMING-UNSIGNED-PAYLOAD-TRAILER":               true,
-		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD":               true,
-		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER":       true,
-		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD":         true,
-		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
-	}
-
-	return specialValues[str]
-}
-
 func validateDate(date time.Time) error {
 	now := time.Now().UTC()
 	diff := date.Unix() - now.Unix()
 
 	// Checks the dates difference to be less than a minute
-	if math.Abs(float64(diff)) > 60 {
-		if diff > 0 {
-			return s3err.APIError{
-				Code:           "SignatureDoesNotMatch",
-				Description:    fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
-				HTTPStatusCode: http.StatusForbidden,
-			}
-		} else {
-			return s3err.APIError{
-				Code:           "SignatureDoesNotMatch",
-				Description:    fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
-				HTTPStatusCode: http.StatusForbidden,
-			}
+	if diff > 60 {
+		return s3err.APIError{
+			Code:           "SignatureDoesNotMatch",
+			Description:    fmt.Sprintf("Signature not yet current: %s is still later than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+			HTTPStatusCode: http.StatusForbidden,
+		}
+	}
+	if diff < -60 {
+		return s3err.APIError{
+			Code:           "SignatureDoesNotMatch",
+			Description:    fmt.Sprintf("Signature expired: %s is now earlier than %s", date.Format(iso8601Format), now.Format(iso8601Format)),
+			HTTPStatusCode: http.StatusForbidden,
 		}
 	}
 
 	return nil
 }
+
+func sendResponse(ctx *fiber.Ctx, err error, logger s3log.AuditLogger) error {
+	return controllers.SendResponse(ctx, err, &controllers.MetaOpts{Logger: logger})
+}

s3api/middlewares/body-reader.go

@@ -0,0 +1,31 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"io"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+func wrapBodyReader(ctx *fiber.Ctx, wr func(io.Reader) io.Reader) {
+	r, ok := ctx.Locals("body-reader").(io.Reader)
+	if !ok {
+		r = ctx.Request().BodyStream()
+	}
+
+	r = wr(r)
+	ctx.Locals("body-reader", r)
+}

s3api/middlewares/md5.go

@@ -16,10 +16,11 @@ package middlewares
 
 import (
 	"crypto/md5"
-	"encoding/base64"
+	"io"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/s3api/controllers"
+	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3log"
 )
@@ -31,8 +32,16 @@ func VerifyMD5Body(logger s3log.AuditLogger) fiber.Handler {
 			return ctx.Next()
 		}
 
+		if utils.IsBigDataAction(ctx) {
+			wrapBodyReader(ctx, func(r io.Reader) io.Reader {
+				r, _ = utils.NewHashReader(r, incomingSum, utils.HashTypeMd5)
+				return r
+			})
+			return ctx.Next()
+		}
+
 		sum := md5.Sum(ctx.Body())
-		calculatedSum := base64.StdEncoding.EncodeToString(sum[:])
+		calculatedSum := utils.Md5SumString(sum[:])
 
 		if incomingSum != calculatedSum {
 			return controllers.SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidDigest), &controllers.MetaOpts{Logger: logger})

s3api/server.go

@@ -32,6 +32,7 @@ type S3ApiServer struct {
 	router  *S3ApiRouter
 	port    string
 	cert    *tls.Certificate
+	quiet   bool
 	debug   bool
 }
 
@@ -48,7 +49,9 @@ func New(app *fiber.App, be backend.Backend, root middlewares.RootUserConfig, po
 	}
 
 	// Logging middlewares
-	app.Use(logger.New())
+	if !server.quiet {
+		app.Use(logger.New())
+	}
 	app.Use(middlewares.DecodeURL(l))
 	app.Use(middlewares.RequestLogger(server.debug))
 
@@ -80,6 +83,11 @@ func WithDebug() Option {
 	return func(s *S3ApiServer) { s.debug = true }
 }
 
+// WithQuiet silences default logging output
+func WithQuiet() Option {
+	return func(s *S3ApiServer) { s.quiet = true }
+}
+
 func (sa *S3ApiServer) Serve() (err error) {
 	if sa.cert != nil {
 		return sa.app.ListenTLSWithCertificate(sa.port, *sa.cert)

s3api/utils/auth-reader.go

@@ -0,0 +1,261 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/smithy-go/logging"
+	"github.com/gofiber/fiber/v2"
+	"github.com/versity/versitygw/s3err"
+)
+
+const (
+	iso8601Format = "20060102T150405Z"
+	yyyymmdd      = "20060102"
+)
+
+// AuthReader is an io.Reader that validates the request authorization
+// once the underlying reader returns io.EOF.  This is needed for streaming
+// data requests where the data size and checksum are not known until
+// the data is completely read.
+type AuthReader struct {
+	ctx    *fiber.Ctx
+	auth   AuthData
+	secret string
+	size   int
+	r      *HashReader
+	debug  bool
+}
+
+// NewAuthReader initializes an io.Reader that will verify the request
+// v4 auth when the underlying reader returns io.EOF. This postpones the
+// authorization check until the reader is consumed. So it is important that
+// the consumer of this reader checks for the auth errors while reading.
+func NewAuthReader(ctx *fiber.Ctx, r io.Reader, auth AuthData, secret string, debug bool) *AuthReader {
+	var hr *HashReader
+	hashPayload := ctx.Get("X-Amz-Content-Sha256")
+	if !IsSpecialPayload(hashPayload) {
+		hr, _ = NewHashReader(r, "", HashTypeSha256)
+	} else {
+		hr, _ = NewHashReader(r, "", HashTypeNone)
+	}
+
+	return &AuthReader{
+		ctx:    ctx,
+		r:      hr,
+		auth:   auth,
+		secret: secret,
+		debug:  debug,
+	}
+}
+
+// Read allows *AuthReader to be used as an io.Reader
+func (ar *AuthReader) Read(p []byte) (int, error) {
+	n, err := ar.r.Read(p)
+	ar.size += n
+
+	if errors.Is(err, io.EOF) {
+		verr := ar.validateSignature()
+		if verr != nil {
+			return n, verr
+		}
+	}
+
+	return n, err
+}
+
+func (ar *AuthReader) validateSignature() error {
+	date := ar.ctx.Get("X-Amz-Date")
+	if date == "" {
+		return s3err.GetAPIError(s3err.ErrMissingDateHeader)
+	}
+
+	hashPayload := ar.ctx.Get("X-Amz-Content-Sha256")
+	if !IsSpecialPayload(hashPayload) {
+		hexPayload := ar.r.Sum()
+
+		// Compare the calculated hash with the hash provided
+		if hashPayload != hexPayload {
+			return s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
+		}
+	}
+
+	// Parse the date and check the date validity
+	tdate, err := time.Parse(iso8601Format, date)
+	if err != nil {
+		return s3err.GetAPIError(s3err.ErrMalformedDate)
+	}
+
+	return CheckValidSignature(ar.ctx, ar.auth, ar.secret, hashPayload, tdate, int64(ar.size), ar.debug)
+}
+
+const (
+	service = "s3"
+)
+
+// CheckValidSignature validates the ctx v4 auth signature
+func CheckValidSignature(ctx *fiber.Ctx, auth AuthData, secret, checksum string, tdate time.Time, contentLen int64, debug bool) error {
+	signedHdrs := strings.Split(auth.SignedHeaders, ";")
+
+	// Create a new http request instance from fasthttp request
+	req, err := createHttpRequestFromCtx(ctx, signedHdrs, contentLen)
+	if err != nil {
+		return fmt.Errorf("create http request from context: %w", err)
+	}
+
+	signer := v4.NewSigner()
+
+	signErr := signer.SignHTTP(req.Context(), aws.Credentials{
+		AccessKeyID:     auth.Access,
+		SecretAccessKey: secret,
+	}, req, checksum, service, auth.Region, tdate, func(options *v4.SignerOptions) {
+		options.DisableURIPathEscaping = true
+		if debug {
+			options.LogSigning = true
+			options.Logger = logging.NewStandardLogger(os.Stderr)
+		}
+	})
+	if signErr != nil {
+		return fmt.Errorf("sign generated http request: %w", err)
+	}
+
+	genAuth, err := ParseAuthorization(req.Header.Get("Authorization"))
+	if err != nil {
+		return err
+	}
+
+	if auth.Signature != genAuth.Signature {
+		return s3err.GetAPIError(s3err.ErrSignatureDoesNotMatch)
+	}
+
+	return nil
+}
+
+// AuthData is the parsed authorization data from the header
+type AuthData struct {
+	Algorithm     string
+	Access        string
+	Region        string
+	SignedHeaders string
+	Signature     string
+	Date          string
+}
+
+// ParseAuthorization returns the parsed fields for the aws v4 auth header
+// example authorization string from aws docs:
+// Authorization: AWS4-HMAC-SHA256
+// Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,
+// SignedHeaders=host;range;x-amz-date,
+// Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024
+func ParseAuthorization(authorization string) (AuthData, error) {
+	a := AuthData{}
+
+	// authorization must start with:
+	// Authorization: <ALGORITHM>
+	// followed by key=value pairs separated by ","
+	authParts := strings.Fields(authorization)
+	for i, el := range authParts {
+		authParts[i] = strings.TrimSpace(el)
+	}
+
+	if len(authParts) < 3 {
+		return a, s3err.GetAPIError(s3err.ErrMissingFields)
+	}
+
+	algo := authParts[0]
+
+	kvData := strings.Join(authParts[1:], "")
+	kvPairs := strings.Split(kvData, ",")
+	// we are expecting at least Credential, SignedHeaders, and Signature
+	// key value pairs here
+	if len(kvPairs) < 3 {
+		return a, s3err.GetAPIError(s3err.ErrMissingFields)
+	}
+
+	var access, region, signedHeaders, signature, date string
+
+	for _, kv := range kvPairs {
+		keyValue := strings.Split(kv, "=")
+		if len(keyValue) != 2 {
+			switch {
+			case strings.HasPrefix(kv, "Credential"):
+				return a, s3err.GetAPIError(s3err.ErrCredMalformed)
+			case strings.HasPrefix(kv, "SignedHeaders"):
+				return a, s3err.GetAPIError(s3err.ErrInvalidQueryParams)
+			}
+			return a, s3err.GetAPIError(s3err.ErrMissingFields)
+		}
+		key := strings.TrimSpace(keyValue[0])
+		value := strings.TrimSpace(keyValue[1])
+
+		switch key {
+		case "Credential":
+			creds := strings.Split(value, "/")
+			if len(creds) != 5 {
+				return a, s3err.GetAPIError(s3err.ErrCredMalformed)
+			}
+			if creds[3] != "s3" {
+				return a, s3err.GetAPIError(s3err.ErrSignatureIncorrService)
+			}
+			if creds[4] != "aws4_request" {
+				return a, s3err.GetAPIError(s3err.ErrSignatureTerminationStr)
+			}
+			_, err := time.Parse(yyyymmdd, creds[1])
+			if err != nil {
+				return a, s3err.GetAPIError(s3err.ErrSignatureDateDoesNotMatch)
+			}
+			access = creds[0]
+			date = creds[1]
+			region = creds[2]
+		case "SignedHeaders":
+			signedHeaders = value
+		case "Signature":
+			signature = value
+		}
+	}
+
+	return AuthData{
+		Algorithm:     algo,
+		Access:        access,
+		Region:        region,
+		SignedHeaders: signedHeaders,
+		Signature:     signature,
+		Date:          date,
+	}, nil
+}
+
+var (
+	specialValues = map[string]bool{
+		"UNSIGNED-PAYLOAD":                                 true,
+		"STREAMING-UNSIGNED-PAYLOAD-TRAILER":               true,
+		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD":               true,
+		"STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER":       true,
+		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD":         true,
+		"STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER": true,
+	}
+)
+
+// IsSpecialPayload checks for streaming/unsigned authorization types
+func IsSpecialPayload(str string) bool {
+	return specialValues[str]
+}

s3api/utils/csum-reader.go

@@ -0,0 +1,130 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+	"crypto/md5"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"hash"
+	"io"
+
+	"github.com/versity/versitygw/s3err"
+)
+
+// HashType identifies the checksum algorithm to be used
+type HashType string
+
+const (
+	// HashTypeMd5 generates MD5 checksum for the data stream
+	HashTypeMd5 = "md5"
+	// HashTypeSha256 generates SHA256 checksum for the data stream
+	HashTypeSha256 = "sha256"
+	// HashTypeNone is a no-op checksum for the data stream
+	HashTypeNone = "none"
+)
+
+// HashReader is an io.Reader that calculates the checksum
+// as the data is read
+type HashReader struct {
+	hashType HashType
+	hash     hash.Hash
+	r        io.Reader
+	sum      string
+}
+
+var (
+	errInvalidHashType = errors.New("unsupported or invalid checksum type")
+)
+
+// NewHashReader intializes an io.Reader from an underlying io.Reader that
+// calculates the checksum while the reader is being read from. If the
+// sum provided is not "", the reader will return an error when the underlying
+// reader returns io.EOF if the checksum does not match the provided expected
+// checksum.  If the provided sum is "", then the Sum() method can still
+// be used to get the current checksum for the data read so far.
+func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, error) {
+	var hash hash.Hash
+	switch ht {
+	case HashTypeMd5:
+		hash = md5.New()
+	case HashTypeSha256:
+		hash = sha256.New()
+	case HashTypeNone:
+		hash = noop{}
+	default:
+		return nil, errInvalidHashType
+	}
+
+	return &HashReader{
+		hash:     hash,
+		r:        r,
+		sum:      expectedSum,
+		hashType: ht,
+	}, nil
+}
+
+// Read allows *HashReader to be used as an io.Reader
+func (hr *HashReader) Read(p []byte) (int, error) {
+	n, readerr := hr.r.Read(p)
+	_, err := hr.hash.Write(p[:n])
+	if err != nil {
+		return n, err
+	}
+	if errors.Is(readerr, io.EOF) && hr.sum != "" {
+		switch hr.hashType {
+		case HashTypeMd5:
+			sum := base64.StdEncoding.EncodeToString(hr.hash.Sum(nil))
+			if sum != hr.sum {
+				return n, s3err.GetAPIError(s3err.ErrInvalidDigest)
+			}
+		case HashTypeSha256:
+			sum := hex.EncodeToString(hr.hash.Sum(nil))
+			if sum != hr.sum {
+				return n, s3err.GetAPIError(s3err.ErrContentSHA256Mismatch)
+			}
+		default:
+			return n, errInvalidHashType
+		}
+	}
+	return n, readerr
+}
+
+// Sum returns the checksum hash of the data read so far
+func (hr *HashReader) Sum() string {
+	switch hr.hashType {
+	case HashTypeMd5:
+		return Md5SumString(hr.hash.Sum(nil))
+	case HashTypeSha256:
+		return hex.EncodeToString(hr.hash.Sum(nil))
+	default:
+		return ""
+	}
+}
+
+// Md5SumString converts the hash bytes to the string checksum value
+func Md5SumString(b []byte) string {
+	return base64.StdEncoding.EncodeToString(b)
+}
+
+type noop struct{}
+
+func (n noop) Write(p []byte) (int, error) { return 0, nil }
+func (n noop) Sum(b []byte) []byte         { return []byte{} }
+func (n noop) Reset()                      {}
+func (n noop) Size() int                   { return 0 }
+func (n noop) BlockSize() int              { return 1 }

s3api/utils/utils.go

@@ -18,6 +18,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -49,10 +50,16 @@ func GetUserMetaData(headers *fasthttp.RequestHeader) (metadata map[string]strin
 	return
 }
 
-func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Request, error) {
+func createHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string, contentLength int64) (*http.Request, error) {
 	req := ctx.Request()
+	var body io.Reader
+	if IsBigDataAction(ctx) {
+		body = req.BodyStream()
+	} else {
+		body = bytes.NewReader(req.Body())
+	}
 
-	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), bytes.NewReader(req.Body()))
+	httpReq, err := http.NewRequest(string(req.Header.Method()), string(ctx.Context().RequestURI()), body)
 	if err != nil {
 		return nil, errors.New("error in creating an http request")
 	}
@@ -69,6 +76,8 @@ func CreateHttpRequestFromCtx(ctx *fiber.Ctx, signedHdrs []string) (*http.Reques
 	// If content length is non 0, then the header will be included
 	if !includeHeader("Content-Length", signedHdrs) {
 		httpReq.ContentLength = 0
+	} else {
+		httpReq.ContentLength = contentLength
 	}
 
 	// Set the Host header
@@ -131,3 +140,12 @@ func includeHeader(hdr string, signedHdrs []string) bool {
 	}
 	return false
 }
+
+func IsBigDataAction(ctx *fiber.Ctx) bool {
+	if ctx.Method() == http.MethodPut && len(strings.Split(ctx.Path(), "/")) >= 3 {
+		if !ctx.Request().URI().QueryArgs().Has("tagging") && ctx.Get("X-Amz-Copy-Source") == "" && !ctx.Request().URI().QueryArgs().Has("acl") {
+			return true
+		}
+	}
+	return false
+}

s3api/utils/utils_test.go

@@ -55,7 +55,7 @@ func TestCreateHttpRequestFromCtx(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := CreateHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"})
+			got, err := createHttpRequestFromCtx(tt.args.ctx, []string{"X-Amz-Mfa"}, 0)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("CreateHttpRequestFromCtx() error = %v, wantErr %v", err, tt.wantErr)
 				return