feat: Added s3 proxy comparison to upload, download and throughput benchmark tests

.dockerignore

@@ -1,46 +0,0 @@
-# Binaries for programs and plugins
-*.exe
-*.exe~
-*.dll
-*.so
-*.dylib
-cmd/versitygw/versitygw
-/versitygw
-
-# Test binary, built with `go test -c`
-*.test
-
-# Output of the go coverage tool, specifically when used with LiteIDE
-*.out
-
-# Go workspace file
-go.work
-
-# ignore IntelliJ directories
-.idea
-
-# auto generated VERSION file
-VERSION
-
-# build output
-/versitygw.spec
-/versitygw.spec.in
-*.tar
-*.tar.gz
-**/rand.data
-/profile.txt
-
-dist/
-
-# Release config files
-/.github
-
-# Docker configuration files
-*Dockerfile
-/docker-compose.yml
-
-# read files
-/LICENSE
-/NOTICE
-/CODE_OF_CONDUCT.md
-/README.md

.env.dev

@@ -1,6 +0,0 @@
-POSIX_PORT=
-PROXY_PORT=
-ACCESS_KEY_ID=
-SECRET_ACCESS_KEY=
-IAM_DIR=
-SETUP_DIR=

.goreleaser.yaml

@@ -9,10 +9,6 @@ builds:
       # windows is untested, we can start doing windows releases
       # if someone is interested in taking on testing
       # - windows
-    env:
-      # disable cgo to fix glibc issues: https://github.com/golang/go/issues/58550
-      # once we need to enable this, we will need to do per distro releases
-      - CGO_ENABLED=0
     main: ./cmd/versitygw
     binary: ./cmd/versitygw
     id: versitygw

Dockerfile

@@ -1,24 +0,0 @@
-FROM golang:1.20-alpine
-
-WORKDIR /app
-
-COPY go.mod ./
-RUN go mod download
-
-COPY ./ ./
-
-WORKDIR /app/cmd/versitygw
-RUN go build -o versitygw
-
-FROM alpine:latest
-
-# These arguments can be overriden when building the image
-ARG IAM_DIR=/tmp/vgw
-ARG SETUP_DIR=/tmp/vgw
-
-RUN mkdir -p $IAM_DIR
-RUN mkdir -p $SETUP_DIR
-
-COPY --from=0 /app/cmd/versitygw/versitygw /app/versitygw
-
-ENTRYPOINT [ "/app/versitygw" ]

Dockerfile.dev

@@ -1,17 +0,0 @@
-FROM golang:1.20
-
-WORKDIR /app
-
-COPY go.mod ./
-RUN go mod download
-
-COPY ./ ./
-
-ARG IAM_DIR=/tmp/vgw
-ARG SETUP_DIR=/tmp/vgw
-
-RUN mkdir -p $IAM_DIR
-RUN mkdir -p $SETUP_DIR
-
-RUN go get github.com/githubnemo/CompileDaemon
-RUN go install github.com/githubnemo/CompileDaemon

Makefile

@@ -74,18 +74,3 @@ dist: $(BIN).spec
 	rm -f VERSION
 	rm -f $(BIN).spec
 	gzip -f $(TARFILE)
-
-# Creates and runs S3 gateway instance in a docker container
-.PHONY: up-posix
-up-posix:
-	docker compose --env-file .env.dev up posix
-
-# Creates and runs S3 gateway proxy instance in a docker container
-.PHONY: up-proxy
-up-proxy:
-	docker compose --env-file .env.dev up proxy
-
-# Creates and runs both S3 gateway and proxy server instances in docker containers
-.PHONY: up-app
-up-app:
-	docker compose --env-file .env.dev up

backend/backend.go

@@ -15,7 +15,6 @@
 package backend
 
 import (
-	"bufio"
 	"context"
 	"fmt"
 	"io"
@@ -23,7 +22,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
-	"github.com/versity/versitygw/s3select"
 )
 
 //go:generate moq -out ../s3api/controllers/backend_moq_test.go -pkg controllers . Backend
@@ -63,7 +61,7 @@ type Backend interface {
 
 	// special case object operations
 	RestoreObject(context.Context, *s3.RestoreObjectInput) error
-	SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
+	SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
 
 	// object tags operations
 	GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error)
@@ -164,19 +162,8 @@ func (BackendUnsupported) PutObjectAcl(context.Context, *s3.PutObjectAclInput) e
 func (BackendUnsupported) RestoreObject(context.Context, *s3.RestoreObjectInput) error {
 	return s3err.GetAPIError(s3err.ErrNotImplemented)
 }
-func (BackendUnsupported) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
-	return func(w *bufio.Writer) {
-		var getProgress s3select.GetProgress
-		progress := input.RequestProgress
-		if progress != nil && *progress.Enabled {
-			getProgress = func() (bytesScanned int64, bytesProcessed int64) {
-				return -1, -1
-			}
-		}
-		mh := s3select.NewMessageHandler(ctx, w, getProgress)
-		apiErr := s3err.GetAPIError(s3err.ErrNotImplemented)
-		mh.FinishWithError(apiErr.Code, apiErr.Description)
-	}
+func (BackendUnsupported) SelectObjectContent(context.Context, *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
+	return s3response.SelectObjectContentResult{}, s3err.GetAPIError(s3err.ErrNotImplemented)
 }
 
 func (BackendUnsupported) GetObjectTagging(_ context.Context, bucket, object string) (map[string]string, error) {

backend/posix/posix.go

@@ -146,10 +146,6 @@ func (p *Posix) ListBuckets(_ context.Context, owner string, isAdmin bool) (s3re
 }
 
 func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) {
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-
 	_, err := os.Lstat(*input.Bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -162,10 +158,6 @@ func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.He
 }
 
 func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput) error {
-	if input.Bucket == nil {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-
 	bucket := *input.Bucket
 	owner := string(input.ObjectOwnership)
 
@@ -191,10 +183,6 @@ func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput) err
 }
 
 func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) error {
-	if input.Bucket == nil {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-
 	names, err := os.ReadDir(*input.Bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -224,13 +212,6 @@ func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) err
 }
 
 func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
-	if mpu.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-	if mpu.Key == nil {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-
 	bucket := *mpu.Bucket
 	object := *mpu.Key
 
@@ -288,19 +269,6 @@ func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipart
 }
 
 func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-	if input.Key == nil {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-	if input.UploadId == nil {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload)
-	}
-	if input.MultipartUpload == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
-	}
-
 	bucket := *input.Bucket
 	object := *input.Key
 	uploadID := *input.UploadId
@@ -326,7 +294,7 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
 	partsize := int64(0)
 	var totalsize int64
 	for i, p := range parts {
-		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))
+		partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))
 		fi, err := os.Lstat(partPath)
 		if err != nil {
 			return nil, s3err.GetAPIError(s3err.ErrInvalidPart)
@@ -358,7 +326,7 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul
 	defer f.cleanup()
 
 	for _, p := range parts {
-		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)))
+		pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)))
 		if err != nil {
 			return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err)
 		}
@@ -544,16 +512,6 @@ func mkdirAll(path string, perm os.FileMode, bucket, object string) error {
 }
 
 func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUploadInput) error {
-	if mpu.Bucket == nil {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-	if mpu.Key == nil {
-		return s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-	if mpu.UploadId == nil {
-		return s3err.GetAPIError(s3err.ErrNoSuchUpload)
-	}
-
 	bucket := *mpu.Bucket
 	object := *mpu.Key
 	uploadID := *mpu.UploadId
@@ -584,12 +542,6 @@ func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUp
 }
 
 func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
-	var lmu s3response.ListMultipartUploadsResult
-
-	if mpu.Bucket == nil {
-		return lmu, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-
 	bucket := *mpu.Bucket
 	var delimiter string
 	if mpu.Delimiter != nil {
@@ -600,6 +552,8 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
 		prefix = *mpu.Prefix
 	}
 
+	var lmu s3response.ListMultipartUploadsResult
+
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return lmu, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -672,16 +626,12 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
 		}
 	}
 
-	maxUploads := 0
-	if mpu.MaxUploads != nil {
-		maxUploads = int(*mpu.MaxUploads)
-	}
 	if (uploadIDMarker != "" && !uploadIdMarkerFound) || (keyMarker != "" && keyMarkerInd == -1) {
 		return s3response.ListMultipartUploadsResult{
 			Bucket:         bucket,
 			Delimiter:      delimiter,
 			KeyMarker:      keyMarker,
-			MaxUploads:     maxUploads,
+			MaxUploads:     int(mpu.MaxUploads),
 			Prefix:         prefix,
 			UploadIDMarker: uploadIDMarker,
 			Uploads:        []s3response.Upload{},
@@ -693,18 +643,18 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
 	})
 
 	for i := keyMarkerInd + 1; i < len(uploads); i++ {
-		if maxUploads == 0 {
+		if mpu.MaxUploads == 0 {
 			break
 		}
 		if keyMarker != "" && uploadIDMarker != "" && uploads[i].UploadID < uploadIDMarker {
 			continue
 		}
-		if i != len(uploads)-1 && len(resultUpds) == maxUploads {
+		if i != len(uploads)-1 && len(resultUpds) == int(mpu.MaxUploads) {
 			return s3response.ListMultipartUploadsResult{
 				Bucket:             bucket,
 				Delimiter:          delimiter,
 				KeyMarker:          keyMarker,
-				MaxUploads:         maxUploads,
+				MaxUploads:         int(mpu.MaxUploads),
 				NextKeyMarker:      resultUpds[i-1].Key,
 				NextUploadIDMarker: resultUpds[i-1].UploadID,
 				IsTruncated:        true,
@@ -721,7 +671,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
 		Bucket:         bucket,
 		Delimiter:      delimiter,
 		KeyMarker:      keyMarker,
-		MaxUploads:     maxUploads,
+		MaxUploads:     int(mpu.MaxUploads),
 		Prefix:         prefix,
 		UploadIDMarker: uploadIDMarker,
 		Uploads:        resultUpds,
@@ -729,29 +679,13 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl
 }
 
 func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) {
-	var lpr s3response.ListPartsResult
-
-	if input.Bucket == nil {
-		return lpr, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-	if input.Key == nil {
-		return lpr, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-	if input.UploadId == nil {
-		return lpr, s3err.GetAPIError(s3err.ErrNoSuchUpload)
-	}
-
 	bucket := *input.Bucket
 	object := *input.Key
 	uploadID := *input.UploadId
-	stringMarker := ""
-	if input.PartNumberMarker != nil {
-		stringMarker = *input.PartNumberMarker
-	}
-	maxParts := 0
-	if input.MaxParts != nil {
-		maxParts = int(*input.MaxParts)
-	}
+	stringMarker := *input.PartNumberMarker
+	maxParts := int(input.MaxParts)
+
+	var lpr s3response.ListPartsResult
 
 	var partNumberMarker int
 	if stringMarker != "" {
@@ -843,21 +777,11 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon
 }
 
 func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string, error) {
-	if input.Bucket == nil {
-		return "", s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-	if input.Key == nil {
-		return "", s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-
 	bucket := *input.Bucket
 	object := *input.Key
 	uploadID := *input.UploadId
 	part := input.PartNumber
-	length := int64(0)
-	if input.ContentLength != nil {
-		length = *input.ContentLength
-	}
+	length := input.ContentLength
 	r := input.Body
 
 	_, err := os.Stat(bucket)
@@ -879,7 +803,7 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
 		return "", fmt.Errorf("stat uploadid: %w", err)
 	}
 
-	partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part))
+	partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", part))
 
 	f, err := openTmpFile(filepath.Join(bucket, objdir),
 		bucket, partPath, length)
@@ -909,13 +833,6 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string
 }
 
 func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) {
-	if upi.Bucket == nil {
-		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-	if upi.Key == nil {
-		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-
 	_, err := os.Stat(*upi.Bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket)
@@ -935,7 +852,7 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
 		return s3response.CopyObjectResult{}, fmt.Errorf("stat uploadid: %w", err)
 	}
 
-	partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", *upi.PartNumber))
+	partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", upi.PartNumber))
 
 	substrs := strings.SplitN(*upi.CopySource, "/", 2)
 	if len(substrs) != 2 {
@@ -1021,13 +938,6 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (
 }
 
 func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) {
-	if po.Bucket == nil {
-		return "", s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-	if po.Key == nil {
-		return "", s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-
 	tagsStr := getString(po.Tagging)
 	tags := make(map[string]string)
 	_, err := os.Stat(*po.Bucket)
@@ -1054,13 +964,9 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 
 	name := filepath.Join(*po.Bucket, *po.Key)
 
-	contentLength := int64(0)
-	if po.ContentLength != nil {
-		contentLength = *po.ContentLength
-	}
 	if strings.HasSuffix(*po.Key, "/") {
 		// object is directory
-		if contentLength != 0 {
+		if po.ContentLength != 0 {
 			// posix directories can't contain data, send error
 			// if reuests has a data payload associated with a
 			// directory object
@@ -1089,7 +995,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 	}
 
 	f, err := openTmpFile(filepath.Join(*po.Bucket, metaTmpDir),
-		*po.Bucket, *po.Key, contentLength)
+		*po.Bucket, *po.Key, po.ContentLength)
 	if err != nil {
 		return "", fmt.Errorf("open temp file: %w", err)
 	}
@@ -1133,13 +1039,6 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e
 }
 
 func (p *Posix) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) error {
-	if input.Bucket == nil {
-		return s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-	if input.Key == nil {
-		return s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-
 	bucket := *input.Bucket
 	object := *input.Key
 
@@ -1232,16 +1131,6 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput)
 }
 
 func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) {
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-	if input.Key == nil {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
-	if input.Range == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRange)
-	}
-
 	bucket := *input.Bucket
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1303,17 +1192,15 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 			return nil, fmt.Errorf("get object tags: %w", err)
 		}
 
-		tagCount := int32(len(tags))
-
 		return &s3.GetObjectOutput{
 			AcceptRanges:    &acceptRange,
-			ContentLength:   &length,
+			ContentLength:   length,
 			ContentEncoding: &contentEncoding,
 			ContentType:     &contentType,
 			ETag:            &etag,
 			LastModified:    backend.GetTimePtr(fi.ModTime()),
 			Metadata:        userMetaData,
-			TagCount:        &tagCount,
+			TagCount:        int32(len(tags)),
 			ContentRange:    &contentRange,
 		}, nil
 	}
@@ -1348,28 +1235,20 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io
 		return nil, fmt.Errorf("get object tags: %w", err)
 	}
 
-	tagCount := int32(len(tags))
-
 	return &s3.GetObjectOutput{
 		AcceptRanges:    &acceptRange,
-		ContentLength:   &length,
+		ContentLength:   length,
 		ContentEncoding: &contentEncoding,
 		ContentType:     &contentType,
 		ETag:            &etag,
 		LastModified:    backend.GetTimePtr(fi.ModTime()),
 		Metadata:        userMetaData,
-		TagCount:        &tagCount,
+		TagCount:        int32(len(tags)),
 		ContentRange:    &contentRange,
 	}, nil
 }
 
 func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-	if input.Key == nil {
-		return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
-	}
 	bucket := *input.Bucket
 	object := *input.Key
 
@@ -1399,10 +1278,8 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He
 		etag = ""
 	}
 
-	size := fi.Size()
-
 	return &s3.HeadObjectOutput{
-		ContentLength:   &size,
+		ContentLength:   fi.Size(),
 		ContentType:     &contentType,
 		ContentEncoding: &contentEncoding,
 		ETag:            &etag,
@@ -1412,18 +1289,6 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He
 }
 
 func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
-	if input.Key == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest)
-	}
-	if input.CopySource == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource)
-	}
-	if input.ExpectedBucketOwner == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidRequest)
-	}
 	srcBucket, srcObject, ok := strings.Cut(*input.CopySource, "/")
 	if !ok {
 		return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource)
@@ -1496,16 +1361,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 		}
 	}
 
-	contentLength := fInfo.Size()
-
-	etag, err := p.PutObject(ctx,
-		&s3.PutObjectInput{
-			Bucket:        &dstBucket,
-			Key:           &dstObject,
-			Body:          f,
-			ContentLength: &contentLength,
-			Metadata:      meta,
-		})
+	etag, err := p.PutObject(ctx, &s3.PutObjectInput{Bucket: &dstBucket, Key: &dstObject, Body: f, ContentLength: fInfo.Size(), Metadata: meta})
 	if err != nil {
 		return nil, err
 	}
@@ -1524,26 +1380,11 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
 }
 
 func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
 	bucket := *input.Bucket
-	prefix := ""
-	if input.Prefix != nil {
-		prefix = *input.Prefix
-	}
-	marker := ""
-	if input.Marker != nil {
-		marker = *input.Marker
-	}
-	delim := ""
-	if input.Delimiter != nil {
-		delim = *input.Delimiter
-	}
-	maxkeys := int32(0)
-	if input.MaxKeys != nil {
-		maxkeys = *input.MaxKeys
-	}
+	prefix := *input.Prefix
+	marker := *input.Marker
+	delim := *input.Delimiter
+	maxkeys := input.MaxKeys
 
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1564,9 +1405,9 @@ func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.
 		CommonPrefixes: results.CommonPrefixes,
 		Contents:       results.Objects,
 		Delimiter:      &delim,
-		IsTruncated:    &results.Truncated,
+		IsTruncated:    results.Truncated,
 		Marker:         &marker,
-		MaxKeys:        &maxkeys,
+		MaxKeys:        maxkeys,
 		Name:           &bucket,
 		NextMarker:     &results.NextMarker,
 		Prefix:         &prefix,
@@ -1625,38 +1466,21 @@ func fileToObj(bucket string) backend.GetObjFunc {
 			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
 		}
 
-		size := fi.Size()
-
 		return types.Object{
 			ETag:         &etag,
 			Key:          &path,
 			LastModified: backend.GetTimePtr(fi.ModTime()),
-			Size:         &size,
+			Size:         fi.Size(),
 		}, nil
 	}
 }
 
 func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
 	bucket := *input.Bucket
-	prefix := ""
-	if input.Prefix != nil {
-		prefix = *input.Prefix
-	}
-	marker := ""
-	if input.ContinuationToken != nil {
-		marker = *input.ContinuationToken
-	}
-	delim := ""
-	if input.Delimiter != nil {
-		delim = *input.Delimiter
-	}
-	maxkeys := int32(0)
-	if input.MaxKeys != nil {
-		maxkeys = *input.MaxKeys
-	}
+	prefix := *input.Prefix
+	marker := *input.ContinuationToken
+	delim := *input.Delimiter
+	maxkeys := input.MaxKeys
 
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1667,25 +1491,23 @@ func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (
 	}
 
 	fileSystem := os.DirFS(bucket)
-	results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
+	results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys),
 		fileToObj(bucket), []string{metaTmpDir})
 	if err != nil {
 		return nil, fmt.Errorf("walk %v: %w", bucket, err)
 	}
 
-	count := int32(len(results.Objects))
-
 	return &s3.ListObjectsV2Output{
 		CommonPrefixes:        results.CommonPrefixes,
 		Contents:              results.Objects,
 		Delimiter:             &delim,
-		IsTruncated:           &results.Truncated,
+		IsTruncated:           results.Truncated,
 		ContinuationToken:     &marker,
-		MaxKeys:               &maxkeys,
+		MaxKeys:               int32(maxkeys),
 		Name:                  &bucket,
 		NextContinuationToken: &results.NextMarker,
 		Prefix:                &prefix,
-		KeyCount:              &count,
+		KeyCount:              int32(len(results.Objects)),
 	}, nil
 }
 
@@ -1706,9 +1528,6 @@ func (p *Posix) PutBucketAcl(_ context.Context, bucket string, data []byte) erro
 }
 
 func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]byte, error) {
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
 	_, err := os.Stat(*input.Bucket)
 	if errors.Is(err, fs.ErrNotExist) {
 		return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket)

backend/s3proxy/client.go

@@ -17,6 +17,7 @@ package s3proxy
 import (
 	"context"
 	"crypto/tls"
+	"fmt"
 	"net/http"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -25,10 +26,16 @@ import (
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/smithy-go/middleware"
+	"github.com/versity/versitygw/auth"
 )
 
 func (s *S3be) getClientFromCtx(ctx context.Context) (*s3.Client, error) {
-	cfg, err := s.getConfig(ctx, s.access, s.secret)
+	acct, ok := ctx.Value("account").(auth.Account)
+	if !ok {
+		return nil, fmt.Errorf("invalid account in context")
+	}
+
+	cfg, err := s.getConfig(ctx, acct.Access, acct.Secret)
 	if err != nil {
 		return nil, err
 	}

backend/s3proxy/s3.go

@@ -17,26 +17,20 @@ package s3proxy
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"strconv"
 
-	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/aws/smithy-go"
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/backend"
-	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3response"
 )
 
 type S3be struct {
 	backend.BackendUnsupported
 
-	access          string
-	secret          string
 	endpoint        string
 	awsRegion       string
 	disableChecksum bool
@@ -44,10 +38,8 @@ type S3be struct {
 	debug           bool
 }
 
-func New(access, secret, endpoint, region string, disableChecksum, sslSkipVerify, debug bool) *S3be {
+func New(endpoint, region string, disableChecksum, sslSkipVerify, debug bool) *S3be {
 	return &S3be{
-		access:          access,
-		secret:          secret,
 		endpoint:        endpoint,
 		awsRegion:       region,
 		disableChecksum: disableChecksum,
@@ -63,7 +55,6 @@ func (s *S3be) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3r
 	}
 
 	output, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
-	err = handleError(err)
 	if err != nil {
 		return s3response.ListAllMyBucketsResult{}, err
 	}
@@ -78,7 +69,8 @@ func (s *S3be) ListBuckets(ctx context.Context, owner string, isAdmin bool) (s3r
 
 	return s3response.ListAllMyBucketsResult{
 		Owner: s3response.CanonicalUser{
-			ID: *output.Owner.ID,
+			ID:          *output.Owner.ID,
+			DisplayName: *output.Owner.DisplayName,
 		},
 		Buckets: s3response.ListAllMyBucketsList{
 			Bucket: buckets,
@@ -92,10 +84,7 @@ func (s *S3be) HeadBucket(ctx context.Context, input *s3.HeadBucketInput) (*s3.H
 		return nil, err
 	}
 
-	out, err := client.HeadBucket(ctx, input)
-	err = handleError(err)
-
-	return out, err
+	return client.HeadBucket(ctx, input)
 }
 
 func (s *S3be) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) error {
@@ -105,7 +94,7 @@ func (s *S3be) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) er
 	}
 
 	_, err = client.CreateBucket(ctx, input)
-	return handleError(err)
+	return err
 }
 
 func (s *S3be) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) error {
@@ -115,7 +104,7 @@ func (s *S3be) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInput) er
 	}
 
 	_, err = client.DeleteBucket(ctx, input)
-	return handleError(err)
+	return err
 }
 
 func (s *S3be) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
@@ -124,9 +113,7 @@ func (s *S3be) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultip
 		return nil, err
 	}
 
-	out, err := client.CreateMultipartUpload(ctx, input)
-	err = handleError(err)
-	return out, err
+	return client.CreateMultipartUpload(ctx, input)
 }
 
 func (s *S3be) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
@@ -135,9 +122,7 @@ func (s *S3be) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMu
 		return nil, err
 	}
 
-	out, err := client.CompleteMultipartUpload(ctx, input)
-	err = handleError(err)
-	return out, err
+	return client.CompleteMultipartUpload(ctx, input)
 }
 
 func (s *S3be) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error {
@@ -147,10 +132,13 @@ func (s *S3be) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipar
 	}
 
 	_, err = client.AbortMultipartUpload(ctx, input)
-	err = handleError(err)
 	return err
 }
 
+const (
+	iso8601Format = "20060102T150405Z"
+)
+
 func (s *S3be) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) {
 	client, err := s.getClientFromCtx(ctx)
 	if err != nil {
@@ -158,7 +146,6 @@ func (s *S3be) ListMultipartUploads(ctx context.Context, input *s3.ListMultipart
 	}
 
 	output, err := client.ListMultipartUploads(ctx, input)
-	err = handleError(err)
 	if err != nil {
 		return s3response.ListMultipartUploadsResult{}, err
 	}
@@ -177,7 +164,7 @@ func (s *S3be) ListMultipartUploads(ctx context.Context, input *s3.ListMultipart
 				DisplayName: *u.Owner.DisplayName,
 			},
 			StorageClass: string(u.StorageClass),
-			Initiated:    u.Initiated.Format(backend.RFC3339TimeFormat),
+			Initiated:    u.Initiated.Format(iso8601Format),
 		})
 	}
 
@@ -197,8 +184,8 @@ func (s *S3be) ListMultipartUploads(ctx context.Context, input *s3.ListMultipart
 		Delimiter:          *output.Delimiter,
 		Prefix:             *output.Prefix,
 		EncodingType:       string(output.EncodingType),
-		MaxUploads:         int(*output.MaxUploads),
-		IsTruncated:        *output.IsTruncated,
+		MaxUploads:         int(output.MaxUploads),
+		IsTruncated:        output.IsTruncated,
 		Uploads:            uploads,
 		CommonPrefixes:     cps,
 	}, nil
@@ -211,7 +198,6 @@ func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3respo
 	}
 
 	output, err := client.ListParts(ctx, input)
-	err = handleError(err)
 	if err != nil {
 		return s3response.ListPartsResult{}, err
 	}
@@ -219,10 +205,10 @@ func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3respo
 	var parts []s3response.Part
 	for _, p := range output.Parts {
 		parts = append(parts, s3response.Part{
-			PartNumber:   int(*p.PartNumber),
-			LastModified: p.LastModified.Format(backend.RFC3339TimeFormat),
+			PartNumber:   int(p.PartNumber),
+			LastModified: p.LastModified.Format(iso8601Format),
 			ETag:         *p.ETag,
-			Size:         *p.Size,
+			Size:         p.Size,
 		})
 	}
 	pnm, err := strconv.Atoi(*output.PartNumberMarker)
@@ -252,8 +238,8 @@ func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3respo
 		StorageClass:         string(output.StorageClass),
 		PartNumberMarker:     pnm,
 		NextPartNumberMarker: npmn,
-		MaxParts:             int(*output.MaxParts),
-		IsTruncated:          *output.IsTruncated,
+		MaxParts:             int(output.MaxParts),
+		IsTruncated:          output.IsTruncated,
 		Parts:                parts,
 	}, nil
 }
@@ -265,7 +251,6 @@ func (s *S3be) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag
 	}
 
 	output, err := client.UploadPart(ctx, input)
-	err = handleError(err)
 	if err != nil {
 		return "", err
 	}
@@ -280,7 +265,6 @@ func (s *S3be) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput
 	}
 
 	output, err := client.UploadPartCopy(ctx, input)
-	err = handleError(err)
 	if err != nil {
 		return s3response.CopyObjectResult{}, err
 	}
@@ -298,7 +282,6 @@ func (s *S3be) PutObject(ctx context.Context, input *s3.PutObjectInput) (string,
 	}
 
 	output, err := client.PutObject(ctx, input)
-	err = handleError(err)
 	if err != nil {
 		return "", err
 	}
@@ -312,10 +295,7 @@ func (s *S3be) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.H
 		return nil, err
 	}
 
-	out, err := client.HeadObject(ctx, input)
-	err = handleError(err)
-
-	return out, err
+	return client.HeadObject(ctx, input)
 }
 
 func (s *S3be) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Writer) (*s3.GetObjectOutput, error) {
@@ -325,7 +305,6 @@ func (s *S3be) GetObject(ctx context.Context, input *s3.GetObjectInput, w io.Wri
 	}
 
 	output, err := client.GetObject(ctx, input)
-	err = handleError(err)
 	if err != nil {
 		return nil, err
 	}
@@ -345,10 +324,7 @@ func (s *S3be) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttri
 		return nil, err
 	}
 
-	out, err := client.GetObjectAttributes(ctx, input)
-	err = handleError(err)
-
-	return out, err
+	return client.GetObjectAttributes(ctx, input)
 }
 
 func (s *S3be) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) {
@@ -357,10 +333,7 @@ func (s *S3be) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.C
 		return nil, err
 	}
 
-	out, err := client.CopyObject(ctx, input)
-	err = handleError(err)
-
-	return out, err
+	return client.CopyObject(ctx, input)
 }
 
 func (s *S3be) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
@@ -369,10 +342,7 @@ func (s *S3be) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (*s3
 		return nil, err
 	}
 
-	out, err := client.ListObjects(ctx, input)
-	err = handleError(err)
-
-	return out, err
+	return client.ListObjects(ctx, input)
 }
 
 func (s *S3be) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
@@ -381,10 +351,7 @@ func (s *S3be) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input)
 		return nil, err
 	}
 
-	out, err := client.ListObjectsV2(ctx, input)
-	err = handleError(err)
-
-	return out, err
+	return client.ListObjectsV2(ctx, input)
 }
 
 func (s *S3be) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) error {
@@ -394,7 +361,7 @@ func (s *S3be) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) er
 	}
 
 	_, err = client.DeleteObject(ctx, input)
-	return handleError(err)
+	return err
 }
 
 func (s *S3be) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) (s3response.DeleteObjectsResult, error) {
@@ -404,7 +371,6 @@ func (s *S3be) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput)
 	}
 
 	output, err := client.DeleteObjects(ctx, input)
-	err = handleError(err)
 	if err != nil {
 		return s3response.DeleteObjectsResult{}, err
 	}
@@ -422,7 +388,6 @@ func (s *S3be) GetBucketAcl(ctx context.Context, input *s3.GetBucketAclInput) ([
 	}
 
 	output, err := client.GetBucketAcl(ctx, input)
-	err = handleError(err)
 	if err != nil {
 		return nil, err
 	}
@@ -472,7 +437,7 @@ func (s S3be) PutBucketAcl(ctx context.Context, bucket string, data []byte) erro
 	}
 
 	_, err = client.PutBucketAcl(ctx, input)
-	return handleError(err)
+	return err
 }
 
 func (s *S3be) PutObjectTagging(ctx context.Context, bucket, object string, tags map[string]string) error {
@@ -496,7 +461,7 @@ func (s *S3be) PutObjectTagging(ctx context.Context, bucket, object string, tags
 		Key:     &object,
 		Tagging: tagging,
 	})
-	return handleError(err)
+	return err
 }
 
 func (s *S3be) GetObjectTagging(ctx context.Context, bucket, object string) (map[string]string, error) {
@@ -509,7 +474,6 @@ func (s *S3be) GetObjectTagging(ctx context.Context, bucket, object string) (map
 		Bucket: &bucket,
 		Key:    &object,
 	})
-	err = handleError(err)
 	if err != nil {
 		return nil, err
 	}
@@ -532,25 +496,5 @@ func (s *S3be) DeleteObjectTagging(ctx context.Context, bucket, object string) e
 		Bucket: &bucket,
 		Key:    &object,
 	})
-	return handleError(err)
-}
-
-func handleError(err error) error {
-	if err == nil {
-		return nil
-	}
-
-	var ae smithy.APIError
-	if errors.As(err, &ae) {
-		apiErr := s3err.APIError{
-			Code:        ae.ErrorCode(),
-			Description: ae.ErrorMessage(),
-		}
-		var re *awshttp.ResponseError
-		if errors.As(err, &re) {
-			apiErr.HTTPStatusCode = re.Response.StatusCode
-		}
-		return apiErr
-	}
 	return err
 }

backend/scoutfs/scoutfs.go

@@ -415,10 +415,8 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.
 		}
 	}
 
-	contentLength := fi.Size()
-
 	return &s3.HeadObjectOutput{
-		ContentLength:   &contentLength,
+		ContentLength:   fi.Size(),
 		ContentType:     &contentType,
 		ContentEncoding: &contentEncoding,
 		ETag:            &etag,
@@ -509,17 +507,15 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer
 		return nil, fmt.Errorf("get object tags: %w", err)
 	}
 
-	tagCount := int32(len(tags))
-
 	return &s3.GetObjectOutput{
 		AcceptRanges:    &acceptRange,
-		ContentLength:   &length,
+		ContentLength:   length,
 		ContentEncoding: &contentEncoding,
 		ContentType:     &contentType,
 		ETag:            &etag,
 		LastModified:    backend.GetTimePtr(fi.ModTime()),
 		Metadata:        userMetaData,
-		TagCount:        &tagCount,
+		TagCount:        int32(len(tags)),
 		StorageClass:    types.StorageClassStandard,
 	}, nil
 }
@@ -546,26 +542,11 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error)
 }
 
 func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
 	bucket := *input.Bucket
-	prefix := ""
-	if input.Prefix != nil {
-		prefix = *input.Prefix
-	}
-	marker := ""
-	if input.Marker != nil {
-		marker = *input.Marker
-	}
-	delim := ""
-	if input.Delimiter != nil {
-		delim = *input.Delimiter
-	}
-	maxkeys := int32(0)
-	if input.MaxKeys != nil {
-		maxkeys = *input.MaxKeys
-	}
+	prefix := *input.Prefix
+	marker := *input.Marker
+	delim := *input.Delimiter
+	maxkeys := input.MaxKeys
 
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -586,9 +567,9 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s
 		CommonPrefixes: results.CommonPrefixes,
 		Contents:       results.Objects,
 		Delimiter:      &delim,
-		IsTruncated:    &results.Truncated,
+		IsTruncated:    results.Truncated,
 		Marker:         &marker,
-		MaxKeys:        &maxkeys,
+		MaxKeys:        maxkeys,
 		Name:           &bucket,
 		NextMarker:     &results.NextMarker,
 		Prefix:         &prefix,
@@ -596,26 +577,11 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s
 }
 
 func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
-	if input.Bucket == nil {
-		return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName)
-	}
 	bucket := *input.Bucket
-	prefix := ""
-	if input.Prefix != nil {
-		prefix = *input.Prefix
-	}
-	marker := ""
-	if input.ContinuationToken != nil {
-		marker = *input.ContinuationToken
-	}
-	delim := ""
-	if input.Delimiter != nil {
-		delim = *input.Delimiter
-	}
-	maxkeys := int32(0)
-	if input.MaxKeys != nil {
-		maxkeys = *input.MaxKeys
-	}
+	prefix := *input.Prefix
+	marker := *input.ContinuationToken
+	delim := *input.Delimiter
+	maxkeys := input.MaxKeys
 
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -636,9 +602,9 @@ func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input)
 		CommonPrefixes:        results.CommonPrefixes,
 		Contents:              results.Objects,
 		Delimiter:             &delim,
-		IsTruncated:           &results.Truncated,
+		IsTruncated:           results.Truncated,
 		ContinuationToken:     &marker,
-		MaxKeys:               &maxkeys,
+		MaxKeys:               int32(maxkeys),
 		Name:                  &bucket,
 		NextContinuationToken: &results.NextMarker,
 		Prefix:                &prefix,
@@ -711,13 +677,11 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc {
 			}
 		}
 
-		size := fi.Size()
-
 		return types.Object{
 			ETag:         &etag,
 			Key:          &path,
 			LastModified: backend.GetTimePtr(fi.ModTime()),
-			Size:         &size,
+			Size:         fi.Size(),
 			StorageClass: sc,
 		}, nil
 	}

backend/walk_test.go

@@ -55,13 +55,11 @@ func getObj(path string, d fs.DirEntry) (types.Object, error) {
 		return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
 	}
 
-	size := fi.Size()
-
 	return types.Object{
 		ETag:         &etag,
 		Key:          &path,
 		LastModified: backend.GetTimePtr(fi.ModTime()),
-		Size:         &size,
+		Size:         fi.Size(),
 	}, nil
 }
 

cmd/versitygw/gateway_test.go

@@ -1,104 +0,0 @@
-package main
-
-import (
-	"context"
-	"log"
-	"os"
-	"path/filepath"
-	"sync"
-	"testing"
-
-	"github.com/versity/versitygw/backend/posix"
-	"github.com/versity/versitygw/integration"
-)
-
-const (
-	tdir = "tempdir"
-)
-
-var (
-	wg sync.WaitGroup
-)
-
-func initEnv(dir string) {
-	// both
-	debug = true
-	region = "us-east-1"
-
-	// server
-	rootUserAccess = "user"
-	rootUserSecret = "pass"
-	iamDir = dir
-	port = "127.0.0.1:7070"
-
-	// client
-	awsID = "user"
-	awsSecret = "pass"
-	endpoint = "http://127.0.0.1:7070"
-}
-
-func initPosix(ctx context.Context) {
-	path, err := os.Getwd()
-	if err != nil {
-		log.Fatalf("get current directory: %v", err)
-	}
-
-	tempdir := filepath.Join(path, tdir)
-	initEnv(tempdir)
-
-	err = os.RemoveAll(tempdir)
-	if err != nil {
-		log.Fatalf("remove temp directory: %v", err)
-	}
-
-	err = os.Mkdir(tempdir, 0755)
-	if err != nil {
-		log.Fatalf("make temp directory: %v", err)
-	}
-
-	be, err := posix.New(tempdir)
-	if err != nil {
-		log.Fatalf("init posix: %v", err)
-	}
-
-	wg.Add(1)
-	go func() {
-		err = runGateway(ctx, be)
-		if err != nil && err != context.Canceled {
-			log.Fatalf("run gateway: %v", err)
-		}
-
-		err := os.RemoveAll(tempdir)
-		if err != nil {
-			log.Fatalf("remove temp directory: %v", err)
-		}
-		wg.Done()
-	}()
-}
-
-func TestIntegration(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-
-	initPosix(ctx)
-
-	opts := []integration.Option{
-		integration.WithAccess(awsID),
-		integration.WithSecret(awsSecret),
-		integration.WithRegion(region),
-		integration.WithEndpoint(endpoint),
-	}
-	if debug {
-		opts = append(opts, integration.WithDebug())
-	}
-
-	s := integration.NewS3Conf(opts...)
-
-	// replace below with desired test
-	err := integration.HeadBucket_non_existing_bucket(s)
-	if err != nil {
-		t.Error(err)
-	}
-
-	cancel()
-	wg.Wait()
-}

cmd/versitygw/main.go

@@ -280,7 +280,7 @@ func initFlags() []cli.Flag {
 	}
 }
 
-func runGateway(ctx context.Context, be backend.Backend) error {
+func runGateway(ctx *cli.Context, be backend.Backend) error {
 	// int32 max for 32 bit arch
 	blimit := int64(2*1024*1024*1024 - 1)
 	if strconv.IntSize > 32 {

cmd/versitygw/posix.go

@@ -49,5 +49,5 @@ func runPosix(ctx *cli.Context) error {
 		return fmt.Errorf("init posix: %v", err)
 	}
 
-	return runGateway(ctx.Context, be)
+	return runGateway(ctx, be)
 }

cmd/versitygw/s3.go

@@ -20,8 +20,6 @@ import (
 )
 
 var (
-	s3proxyAccess          string
-	s3proxySecret          string
 	s3proxyEndpoint        string
 	s3proxyRegion          string
 	s3proxyDisableChecksum bool
@@ -37,22 +35,6 @@ func s3Command() *cli.Command {
 to an s3 storage backend service.`,
 		Action: runS3,
 		Flags: []cli.Flag{
-			&cli.StringFlag{
-				Name:        "access",
-				Usage:       "s3 proxy server access key id",
-				Value:       "",
-				Required:    true,
-				Destination: &s3proxyAccess,
-				Aliases:     []string{"a"},
-			},
-			&cli.StringFlag{
-				Name:        "secret",
-				Usage:       "s3 proxy server secret access key",
-				Value:       "",
-				Required:    true,
-				Destination: &s3proxySecret,
-				Aliases:     []string{"s"},
-			},
 			&cli.StringFlag{
 				Name:        "endpoint",
 				Usage:       "s3 service endpoint, default AWS if not specified",
@@ -88,7 +70,7 @@ to an s3 storage backend service.`,
 }
 
 func runS3(ctx *cli.Context) error {
-	be := s3proxy.New(s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
+	be := s3proxy.New(s3proxyEndpoint, s3proxyRegion,
 		s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyDebug)
-	return runGateway(ctx.Context, be)
+	return runGateway(ctx, be)
 }

cmd/versitygw/scoutfs.go

@@ -69,5 +69,5 @@ func runScoutfs(ctx *cli.Context) error {
 		return fmt.Errorf("init scoutfs: %v", err)
 	}
 
-	return runGateway(ctx.Context, be)
+	return runGateway(ctx, be)
 }

cmd/versitygw/test.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"fmt"
+	"math"
+	"os"
+	"text/tabwriter"
 
 	"github.com/urfave/cli/v2"
 	"github.com/versity/versitygw/integration"
@@ -13,6 +16,7 @@ var (
 	endpoint        string
 	prefix          string
 	dstBucket       string
+	proxyURL        string
 	partSize        int64
 	objSize         int64
 	concurrency     int
@@ -67,7 +71,7 @@ func initTestFlags() []cli.Flag {
 }
 
 func initTestCommands() []*cli.Command {
-	return append([]*cli.Command{
+	return []*cli.Command{
 		{
 			Name:        "full-flow",
 			Usage:       "Tests the full flow of gateway.",
@@ -118,6 +122,7 @@ func initTestCommands() []*cli.Command {
 					Name:        "bucket",
 					Usage:       "Destination bucket name to read/write data",
 					Destination: &dstBucket,
+					Required:    true,
 				},
 				&cli.Int64Flag{
 					Name:        "partSize",
@@ -143,6 +148,11 @@ func initTestCommands() []*cli.Command {
 					Value:       false,
 					Destination: &checksumDisable,
 				},
+				&cli.StringFlag{
+					Name:        "proxy-url",
+					Usage:       "S3 proxy server url to compare",
+					Destination: &proxyURL,
+				},
 			},
 			Action: func(ctx *cli.Context) error {
 				if upload && download {
@@ -152,10 +162,6 @@ func initTestCommands() []*cli.Command {
 					return fmt.Errorf("must specify one of upload or download")
 				}
 
-				if dstBucket == "" {
-					return fmt.Errorf("must specify bucket")
-				}
-
 				opts := []integration.Option{
 					integration.WithAccess(awsID),
 					integration.WithSecret(awsSecret),
@@ -177,9 +183,47 @@ func initTestCommands() []*cli.Command {
 				s3conf := integration.NewS3Conf(opts...)
 
 				if upload {
-					return integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+					if proxyURL == "" {
+						integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+						return nil
+					} else {
+						size, elapsed, err := integration.TestUpload(s3conf, files, objSize, dstBucket, prefix)
+						opts = append(opts, integration.WithEndpoint(proxyURL))
+						proxyS3Conf := integration.NewS3Conf(opts...)
+						proxySize, proxyElapsed, proxyErr := integration.TestUpload(proxyS3Conf, files, objSize, dstBucket, prefix)
+						if err != nil || proxyErr != nil {
+							return nil
+						}
+
+						printProxyResultsTable([][4]string{
+							{"    #    ", "Total Size", "Time Taken", "Speed(MB/S)"},
+							{"---------", "----------", "----------", "-----------"},
+							{"S3 Server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
+							{"S3 Proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
+						})
+						return nil
+					}
 				} else {
-					return integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+					if proxyURL == "" {
+						integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+						return nil
+					} else {
+						size, elapsed, err := integration.TestDownload(s3conf, files, objSize, dstBucket, prefix)
+						opts = append(opts, integration.WithEndpoint(proxyURL))
+						proxyS3Conf := integration.NewS3Conf(opts...)
+						proxySize, proxyElapsed, proxyErr := integration.TestDownload(proxyS3Conf, files, objSize, dstBucket, prefix)
+						if err != nil || proxyErr != nil {
+							return nil
+						}
+
+						printProxyResultsTable([][4]string{
+							{"    #    ", "Total Size", "Time Taken", "Speed(MB/S)"},
+							{"---------", "----------", "----------", "-----------"},
+							{"S3 server", fmt.Sprint(size), fmt.Sprintf("%v", elapsed), fmt.Sprint(int(math.Ceil(float64(size)/elapsed.Seconds()) / 1048576))},
+							{"S3 proxy", fmt.Sprint(proxySize), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(int(math.Ceil(float64(proxySize)/proxyElapsed.Seconds()) / 1048576))},
+						})
+						return nil
+					}
 				}
 			},
 		},
@@ -211,12 +255,13 @@ func initTestCommands() []*cli.Command {
 					Value:       false,
 					Destination: &checksumDisable,
 				},
+				&cli.StringFlag{
+					Name:        "proxy-url",
+					Usage:       "S3 proxy server url to compare",
+					Destination: &proxyURL,
+				},
 			},
 			Action: func(ctx *cli.Context) error {
-				if dstBucket == "" {
-					return fmt.Errorf("must specify the destination bucket")
-				}
-
 				opts := []integration.Option{
 					integration.WithAccess(awsID),
 					integration.WithSecret(awsSecret),
@@ -233,10 +278,30 @@ func initTestCommands() []*cli.Command {
 
 				s3conf := integration.NewS3Conf(opts...)
 
-				return integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+				if proxyURL == "" {
+					_, _, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+					return err
+				} else {
+					elapsed, rps, err := integration.TestReqPerSec(s3conf, totalReqs, dstBucket)
+					opts = append(opts, integration.WithEndpoint(proxyURL))
+					s3proxy := integration.NewS3Conf(opts...)
+					proxyElapsed, proxyRPS, proxyErr := integration.TestReqPerSec(s3proxy, totalReqs, dstBucket)
+					if err != nil || proxyErr != nil {
+						return nil
+					}
+
+					printProxyResultsTable([][4]string{
+						{"    #    ", "Total Requests", "Time Taken", "Requests Per Second(Req/Sec)"},
+						{"---------", "--------------", "----------", "----------------------------"},
+						{"S3 Server", fmt.Sprint(totalReqs), fmt.Sprintf("%v", elapsed), fmt.Sprint(rps)},
+						{"S3 Proxy", fmt.Sprint(totalReqs), fmt.Sprintf("%v", proxyElapsed), fmt.Sprint(proxyRPS)},
+					})
+
+					return nil
+				}
 			},
 		},
-	}, extractIntTests()...)
+	}
 }
 
 type testFunc func(*integration.S3Conf)
@@ -265,30 +330,12 @@ func getAction(tf testFunc) func(*cli.Context) error {
 	}
 }
 
-func extractIntTests() (commands []*cli.Command) {
-	tests := integration.GetIntTests()
-	for key, val := range tests {
-		testKey := key
-		testFunc := val
-		commands = append(commands, &cli.Command{
-			Name:  testKey,
-			Usage: fmt.Sprintf("Runs %v integration test", testKey),
-			Action: func(ctx *cli.Context) error {
-				opts := []integration.Option{
-					integration.WithAccess(awsID),
-					integration.WithSecret(awsSecret),
-					integration.WithRegion(region),
-					integration.WithEndpoint(endpoint),
-				}
-				if debug {
-					opts = append(opts, integration.WithDebug())
-				}
-
-				s := integration.NewS3Conf(opts...)
-				err := testFunc(s)
-				return err
-			},
-		})
+func printProxyResultsTable(stats [][4]string) {
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, minwidth, tabwidth, padding, padchar, flags)
+	for _, elem := range stats {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", elem[0], elem[1], elem[2], elem[3])
 	}
-	return
+	fmt.Fprintln(w)
+	w.Flush()
 }

docker-compose.yml

@@ -1,23 +0,0 @@
-version: "3"
-services:
-  posix:
-    build: 
-      context: .
-      dockerfile: ./Dockerfile.dev
-      args:
-        - IAM_DIR=${IAM_DIR}
-        - SETUP_DIR=${SETUP_DIR}
-    volumes:
-      - ./:/app
-    ports:
-      - "${POSIX_PORT}:${POSIX_PORT}"
-    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -p :$POSIX_PORT -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY --iam-dir $IAM_DIR posix $SETUP_DIR"]
-  proxy:
-    build: 
-      context: .
-      dockerfile: ./Dockerfile.dev
-    volumes:
-      - ./:/app
-    ports:
-      - "${PROXY_PORT}:${PROXY_PORT}"
-    command: ["sh", "-c", CompileDaemon -build="go build -C ./cmd/versitygw -o versitygw" -command="./cmd/versitygw/versitygw -a $ACCESS_KEY_ID -s $SECRET_ACCESS_KEY -p :$PROXY_PORT s3 --endpoint http://posix:$POSIX_PORT"]

go.mod

@@ -3,28 +3,28 @@ module github.com/versity/versitygw
 go 1.20
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.23.5
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.2
-	github.com/aws/smithy-go v1.18.1
+	github.com/aws/aws-sdk-go-v2 v1.22.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1
+	github.com/aws/smithy-go v1.16.0
 	github.com/go-ldap/ldap/v3 v3.4.6
-	github.com/gofiber/fiber/v2 v2.51.0
+	github.com/gofiber/fiber/v2 v2.50.0
 	github.com/google/uuid v1.4.0
 	github.com/nats-io/nats.go v1.31.0
 	github.com/pkg/xattr v0.4.9
-	github.com/segmentio/kafka-go v0.4.46
-	github.com/urfave/cli/v2 v2.26.0
-	github.com/valyala/fasthttp v1.51.0
+	github.com/segmentio/kafka-go v0.4.44
+	github.com/urfave/cli/v2 v2.25.7
+	github.com/valyala/fasthttp v1.50.0
 	github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9
-	golang.org/x/sys v0.15.0
+	golang.org/x/sys v0.14.0
 )
 
 require (
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.18.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.26.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/nats-io/nkeys v0.4.6 // indirect
@@ -36,21 +36,21 @@ require (
 
 require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.25.11
-	github.com/aws/aws-sdk-go-v2/credentials v1.16.9
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.4
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.24.0
+	github.com/aws/aws-sdk-go-v2/credentials v1.15.2
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect

go.sum

@@ -4,44 +4,44 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3Uu
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/aws/aws-sdk-go-v2 v1.23.5 h1:xK6C4udTyDMd82RFvNkDQxtAd00xlzFUtX4fF2nMZyg=
-github.com/aws/aws-sdk-go-v2 v1.23.5/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 h1:Zx9+31KyB8wQna6SXFWOewlgoY5uGdDAu6PTOEU3OQI=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3/go.mod h1:zxbEJhRdKTH1nqS2qu6UJ7zGe25xaHxZXaC2CvuQFnA=
-github.com/aws/aws-sdk-go-v2/config v1.25.11 h1:RWzp7jhPRliIcACefGkKp03L0Yofmd2p8M25kbiyvno=
-github.com/aws/aws-sdk-go-v2/config v1.25.11/go.mod h1:BVUs0chMdygHsQtvaMyEOpW2GIW+ubrxJLgIz/JU29s=
-github.com/aws/aws-sdk-go-v2/credentials v1.16.9 h1:LQo3MUIOzod9JdUK+wxmSdgzLVYUbII3jXn3S/HJZU0=
-github.com/aws/aws-sdk-go-v2/credentials v1.16.9/go.mod h1:R7mDuIJoCjH6TxGUc/cylE7Lp/o0bhKVoxdBThsjqCM=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 h1:FZVFahMyZle6WcogZCOxo6D/lkDA2lqKIn4/ueUmVXw=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9/go.mod h1:kjq7REMIkxdtcEC9/4BVXjOsNY5isz6jQbEgk6osRTU=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.4 h1:TUCNKBd4/JEefsZDxo5deRmrRRPZHqGyBYiUAeBKOWU=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.4/go.mod h1:egDkcl+zsgFqS6VO142bKboip5Pe1sNMwN55Xy38QsM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 h1:8GVZIR0y6JRIUNSYI1xAMF4HDfV8H/bOsZ/8AD/uY5Q=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8/go.mod h1:rwBfu0SoUkBUZndVgPZKAD9Y2JigaZtRP68unRiYToQ=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 h1:ZE2ds/qeBkhk3yqYvS3CDCFNvd9ir5hMjlVStLZWrvM=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8/go.mod h1:/lAPPymDYL023+TS6DJmjuL42nxix2AvEvfjqOBRODk=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 h1:abKT+RuM1sdCNZIGIfZpLkvxEX3Rpsto019XG/rkYG8=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8/go.mod h1:Owc4ysUE71JSruVTTa3h4f2pp3E4hlcAtmeNXxDmjj4=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 h1:e3PCNeEaev/ZF01cQyNZgmYE9oYYePIMJs2mWSKG514=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3/go.mod h1:gIeeNyaL8tIEqZrzAnTeyhHcE0yysCtcaP+N9kxLZ+E=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 h1:xyfOAYV/ujzZOo01H9+OnyeiRKmTEp6EsITTsmq332Q=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8/go.mod h1:coLeQEoKzW9ViTL2bn0YUlU7K0RYjivKudG74gtd+sI=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 h1:EamsKe+ZjkOQjDdHd86/JCEucjFKQ9T0atWKO4s2Lgs=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8/go.mod h1:Q0vV3/csTpbkfKLI5Sb56cJQTCTtJ0ixdb7P+Wedqiw=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 h1:ip5ia3JOXl4OAsqeTdrOOmqKgoWiu+t9XSOnRzBwmRs=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8/go.mod h1:kE+aERnK9VQIw1vrk7ElAvhCsgLNzGyCPNg2Qe4Eq4c=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.47.2 h1:DLSAG8zpJV2pYsU+UPkj1IEZghyBnnUsvIRs6UuXSDU=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.47.2/go.mod h1:thjZng67jGsvMyVZnSxlcqKyLwB0XTG8bHIRZPTJ+Bs=
-github.com/aws/aws-sdk-go-v2/service/sso v1.18.2 h1:xJPydhNm0Hiqct5TVKEuHG7weC0+sOs4MUnd7A5n5F4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.18.2/go.mod h1:zxk6y1X2KXThESWMS5CrKRvISD8mbIMab6nZrCGxDG0=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.2 h1:8dU9zqA77C5egbU6yd4hFLaiIdPv3rU+6cp7sz5FjCU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.2/go.mod h1:7Lt5mjQ8x5rVdKqg+sKKDeuwoszDJIIPmkd8BVsEdS0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.26.2 h1:fFrLsy08wEbAisqW3KDl/cPHrF43GmV79zXB9EwJiZw=
-github.com/aws/aws-sdk-go-v2/service/sts v1.26.2/go.mod h1:7Ld9eTqocTvJqqJ5K/orbSDwmGcpRdlDiLjz2DO+SL8=
-github.com/aws/smithy-go v1.18.1 h1:pOdBTUfXNazOlxLrgeYalVnuTpKreACHtc62xLwIB3c=
-github.com/aws/smithy-go v1.18.1/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
+github.com/aws/aws-sdk-go-v2 v1.22.2 h1:lV0U8fnhAnPz8YcdmZVV60+tr6CakHzqA6P8T46ExJI=
+github.com/aws/aws-sdk-go-v2 v1.22.2/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 h1:hHgLiIrTRtddC0AKcJr5s7i/hLgcpTt+q/FKxf1Zayk=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0/go.mod h1:w4I/v3NOWgD+qvs1NPEwhd++1h3XPHFaVxasfY6HlYQ=
+github.com/aws/aws-sdk-go-v2/config v1.24.0 h1:4LEk29JO3w+y9dEo/5Tq5QTP7uIEw+KQrKiHOs4xlu4=
+github.com/aws/aws-sdk-go-v2/config v1.24.0/go.mod h1:11nNDAuK86kOUHeuEQo8f3CkcV5xuUxvPwFjTZE/PnQ=
+github.com/aws/aws-sdk-go-v2/credentials v1.15.2 h1:rKH7khRMxPdD0u3dHecd0Q7NOVw3EUe7AqdkUOkiOGI=
+github.com/aws/aws-sdk-go-v2/credentials v1.15.2/go.mod h1:tXM8wmaeAhfC7nZoCxb0FzM/aRaB1m1WQ7x0qlBLq80=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 h1:G5KawTAkyHH6WyKQCdHiW4h3PmAXNJpOgwKg3H7sDRE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3/go.mod h1:hugKmSFnZB+HgNI1sYGT14BUPZkO6alC/e0AWu+0IAQ=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6 h1:IpQbitxCZeC64C1ALz9QZu6AHHWundnU2evQ9xbp5k8=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.6/go.mod h1:27jIVQK+al9s0yTo3pkMdahRinbscqSC6zNGfNWXPZc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 h1:AaQsr5vvGR7rmeSWBtTCcw16tT9r51mWijuCQhzLnq8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2/go.mod h1:o1IiRn7CWocIFTXJjGKJDOwxv1ibL53NpcvcqGWyRBA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 h1:UZx8SXZ0YtzRiALzYAWcjb9Y9hZUR7MBKaBQ5ouOjPs=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2/go.mod h1:ipuRpcSaklmxR6C39G187TpBAO132gUfleTGccUPs8c=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0 h1:usgqiJtamuGIBj+OvYmMq89+Z1hIKkMJToz1WpoeNUY=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.0/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 h1:pyVrNAf7Hwz0u39dLKN5t+n0+K/3rMYKuiOoIum3AsU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2/go.mod h1:mydrfOb9uiOYCxuCPR8YHQNQyGQwUQ7gPMZGBKbH8NY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 h1:CJxo7ZBbaIzmXfV3hjcx36n9V87gJsIUPJflwqEHl3Q=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0/go.mod h1:yjVfjuY4nD1EW9i387Kau+I6V5cBA5YnC/mWNopjZrI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 h1:f2LhPofnjcdOQKRtumKjMvIHkfSQ8aH/rwKUDEQ/SB4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2/go.mod h1:q+xX0H4OfuWDuBy7y/LDi4v8IBOWuF+vtp8Z6ex+lw4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 h1:h7j73yuAVVjic8pqswh+L/7r2IHP43QwRyOu6zcCDDE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2/go.mod h1:H07AHdK5LSy8F7EJUQhoxyiCNkePoHj2D8P2yGTWafo=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 h1:gbIaOzpXixUpoPK+js/bCBK1QBDXM22SigsnzGZio0U=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2/go.mod h1:p+S7RNbdGN8qgHDSg2SCQJ9FeMAmvcETQiVpeGhYnNM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 h1:o6MCcX1rJW8Y3g+hvg2xpjF6JR6DftuYhfl3Nc1WV9Q=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1/go.mod h1:UDtxEWbREX6y4KREapT+jjtjoH0TiVSS6f5nfaY1UaM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 h1:km+ZNjtLtpXYf42RdaDZnNHm9s7SYAuDGTafy6nd89A=
+github.com/aws/aws-sdk-go-v2/service/sso v1.17.1/go.mod h1:aHBr3pvBSD5MbzOvQtYutyPLLRPbl/y9x86XyJJnUXQ=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 h1:iRFNqZH4a67IqPvK8xxtyQYnyrlsvwmpHOe9r55ggBA=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1/go.mod h1:pTy5WM+6sNv2tB24JNKFtn6EvciQ5k40ZJ0pq/Iaxj0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 h1:txgVXIXWPXyqdiVn92BV6a/rgtpX31HYdsOYj0sVQQQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.25.1/go.mod h1:VAiJiNaoP1L89STFlEMgmHX1bKixY+FaP+TpRFrmyZ4=
+github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik=
+github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -51,8 +51,8 @@ github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
 github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
-github.com/gofiber/fiber/v2 v2.51.0 h1:JNACcZy5e2tGApWB2QrRpenTWn0fq0hkFm6k0C86gKQ=
-github.com/gofiber/fiber/v2 v2.51.0/go.mod h1:xaQRZQJGqnKOQnbQw+ltvku3/h8QxvNi8o6JiJ7Ll0U=
+github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw=
+github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
@@ -67,8 +67,8 @@ github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
@@ -89,8 +89,8 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/segmentio/kafka-go v0.4.46 h1:Sx8/kvtY+/G8nM0roTNnFezSJj3bT2sW0Xy/YY3CgBI=
-github.com/segmentio/kafka-go v0.4.46/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
+github.com/segmentio/kafka-go v0.4.44 h1:Vjjksniy0WSTZ7CuVJrz1k04UoZeTc77UV6Yyk6tLY4=
+github.com/segmentio/kafka-go v0.4.44/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -98,12 +98,12 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI=
-github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
+github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
+github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
-github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
+github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
+github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/versity/scoutfs-go v0.0.0-20230606232754-0474b14343b9 h1:ZfmQR01Kk6/kQh6+zlqfBYszVY02fzf9xYrchOY4NFM=
@@ -146,8 +146,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=

integration/bench.go

@@ -17,16 +17,16 @@ type prefResult struct {
 	err     error
 }
 
-func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
+func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
 	var sg sync.WaitGroup
 	results := make([]prefResult, files)
 	start := time.Now()
 	if objSize == 0 {
-		return fmt.Errorf("must specify object size for upload")
+		return 0, time.Since(start), fmt.Errorf("must specify object size for upload")
 	}
 
 	if objSize > (int64(10000) * s.PartSize) {
-		return fmt.Errorf("object size can not exceed 10000 * chunksize")
+		return 0, time.Since(start), fmt.Errorf("object size can not exceed 10000 * chunksize")
 	}
 
 	runF("performance test: upload objects")
@@ -45,13 +45,13 @@ func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) erro
 		}(i)
 	}
 	sg.Wait()
-	elapsed := time.Since(start)
+	elapsed = time.Since(start)
 
 	var tot int64
 	for i, res := range results {
 		if res.err != nil {
 			failF("%v: %v\n", i, res.err)
-			break
+			return 0, time.Since(start), res.err
 		}
 		tot += res.size
 		fmt.Printf("%v: %v in %v (%v MB/s)\n",
@@ -63,10 +63,10 @@ func TestUpload(s *S3Conf, files int, objSize int64, bucket, prefix string) erro
 	passF("run upload: %v in %v (%v MB/s)\n",
 		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
 
-	return nil
+	return tot, time.Since(start), nil
 }
 
-func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) error {
+func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) (size int64, elapsed time.Duration, err error) {
 	var sg sync.WaitGroup
 	results := make([]prefResult, files)
 	start := time.Now()
@@ -86,13 +86,13 @@ func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) er
 		}(i)
 	}
 	sg.Wait()
-	elapsed := time.Since(start)
+	elapsed = time.Since(start)
 
 	var tot int64
 	for i, res := range results {
 		if res.err != nil {
 			failF("%v: %v\n", i, res.err)
-			break
+			return 0, elapsed, err
 		}
 		tot += res.size
 		fmt.Printf("%v: %v in %v (%v MB/s)\n",
@@ -104,10 +104,10 @@ func TestDownload(s *S3Conf, files int, objSize int64, bucket, prefix string) er
 	passF("run download: %v in %v (%v MB/s)\n",
 		tot, elapsed, int(math.Ceil(float64(tot)/elapsed.Seconds())/1048576))
 
-	return nil
+	return tot, elapsed, nil
 }
 
-func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) error {
+func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) (time.Duration, int, error) {
 	client := s3.NewFromConfig(s.Config())
 	var wg sync.WaitGroup
 	var resErr error
@@ -132,11 +132,11 @@ func TestReqPerSec(s *S3Conf, totalReqs int, bucket string) error {
 	wg.Wait()
 	if resErr != nil {
 		failF("performance test failed with error: %w", resErr)
-		return nil
+		return time.Since(startTime), 0, resErr
 	}
 	elapsedTime := time.Since(startTime)
 	rps := int(float64(totalReqs) / elapsedTime.Seconds())
 
 	passF("Success\nTotal Requests: %d,\nConcurrency Level: %d,\nTime Taken: %s,\nRequests Per Second: %dreq/sec", totalReqs, s.Concurrency, elapsedTime, rps)
-	return nil
+	return elapsedTime, rps, nil
 }

integration/group-tests.go

@@ -222,132 +222,3 @@ func TestPosix(s *S3Conf) {
 	PutObject_dir_obj_with_data(s)
 	CreateMultipartUpload_dir_obj(s)
 }
-
-type IntTests map[string]func(s *S3Conf) error
-
-func GetIntTests() IntTests {
-	return IntTests{
-		"Authentication_empty_auth_header":                    Authentication_empty_auth_header,
-		"Authentication_invalid_auth_header":                  Authentication_invalid_auth_header,
-		"Authentication_unsupported_signature_version":        Authentication_unsupported_signature_version,
-		"Authentication_malformed_credentials":                Authentication_malformed_credentials,
-		"Authentication_malformed_credentials_invalid_parts":  Authentication_malformed_credentials_invalid_parts,
-		"Authentication_credentials_terminated_string":        Authentication_credentials_terminated_string,
-		"Authentication_credentials_incorrect_service":        Authentication_credentials_incorrect_service,
-		"Authentication_credentials_incorrect_region":         Authentication_credentials_incorrect_region,
-		"Authentication_credentials_invalid_date":             Authentication_credentials_invalid_date,
-		"Authentication_credentials_future_date":              Authentication_credentials_future_date,
-		"Authentication_credentials_past_date":                Authentication_credentials_past_date,
-		"Authentication_credentials_non_existing_access_key":  Authentication_credentials_non_existing_access_key,
-		"Authentication_invalid_signed_headers":               Authentication_invalid_signed_headers,
-		"Authentication_missing_date_header":                  Authentication_missing_date_header,
-		"Authentication_invalid_date_header":                  Authentication_invalid_date_header,
-		"Authentication_date_mismatch":                        Authentication_date_mismatch,
-		"Authentication_incorrect_payload_hash":               Authentication_incorrect_payload_hash,
-		"Authentication_incorrect_md5":                        Authentication_incorrect_md5,
-		"Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key,
-		"CreateBucket_invalid_bucket_name":                    CreateBucket_invalid_bucket_name,
-		"CreateBucket_existing_bucket":                        CreateBucket_existing_bucket,
-		"CreateBucket_as_user":                                CreateBucket_as_user,
-		"CreateDeleteBucket_success":                          CreateDeleteBucket_success,
-		"HeadBucket_non_existing_bucket":                      HeadBucket_non_existing_bucket,
-		"HeadBucket_success":                                  HeadBucket_success,
-		"ListBuckets_as_user":                                 ListBuckets_as_user,
-		"ListBuckets_as_admin":                                ListBuckets_as_admin,
-		"ListBuckets_success":                                 ListBuckets_success,
-		"DeleteBucket_non_existing_bucket":                    DeleteBucket_non_existing_bucket,
-		"DeleteBucket_non_empty_bucket":                       DeleteBucket_non_empty_bucket,
-		"DeleteBucket_success_status_code":                    DeleteBucket_success_status_code,
-		"PutObject_non_existing_bucket":                       PutObject_non_existing_bucket,
-		"PutObject_special_chars":                             PutObject_special_chars,
-		"PutObject_invalid_long_tags":                         PutObject_invalid_long_tags,
-		"PutObject_success":                                   PutObject_success,
-		"HeadObject_non_existing_object":                      HeadObject_non_existing_object,
-		"HeadObject_success":                                  HeadObject_success,
-		"GetObject_non_existing_key":                          GetObject_non_existing_key,
-		"GetObject_invalid_ranges":                            GetObject_invalid_ranges,
-		"GetObject_with_meta":                                 GetObject_with_meta,
-		"GetObject_success":                                   GetObject_success,
-		"GetObject_by_range_success":                          GetObject_by_range_success,
-		"ListObjects_non_existing_bucket":                     ListObjects_non_existing_bucket,
-		"ListObjects_with_prefix":                             ListObjects_with_prefix,
-		"ListObject_truncated":                                ListObject_truncated,
-		"ListObjects_invalid_max_keys":                        ListObjects_invalid_max_keys,
-		"ListObjects_max_keys_0":                              ListObjects_max_keys_0,
-		"ListObjects_delimiter":                               ListObjects_delimiter,
-		"ListObjects_max_keys_none":                           ListObjects_max_keys_none,
-		"ListObjects_marker_not_from_obj_list":                ListObjects_marker_not_from_obj_list,
-		"DeleteObject_non_existing_object":                    DeleteObject_non_existing_object,
-		"DeleteObject_success":                                DeleteObject_success,
-		"DeleteObject_success_status_code":                    DeleteObject_success_status_code,
-		"DeleteObjects_empty_input":                           DeleteObjects_empty_input,
-		"DeleteObjects_non_existing_objects":                  DeleteObjects_non_existing_objects,
-		"DeleteObjects_success":                               DeleteObjects_success,
-		"CopyObject_non_existing_dst_bucket":                  CopyObject_non_existing_dst_bucket,
-		"CopyObject_not_owned_source_bucket":                  CopyObject_not_owned_source_bucket,
-		"CopyObject_copy_to_itself":                           CopyObject_copy_to_itself,
-		"CopyObject_to_itself_with_new_metadata":              CopyObject_to_itself_with_new_metadata,
-		"CopyObject_success":                                  CopyObject_success,
-		"PutObjectTagging_non_existing_object":                PutObjectTagging_non_existing_object,
-		"PutObjectTagging_long_tags":                          PutObjectTagging_long_tags,
-		"PutObjectTagging_success":                            PutObjectTagging_success,
-		"GetObjectTagging_non_existing_object":                GetObjectTagging_non_existing_object,
-		"GetObjectTagging_success":                            GetObjectTagging_success,
-		"DeleteObjectTagging_non_existing_object":             DeleteObjectTagging_non_existing_object,
-		"DeleteObjectTagging_success_status":                  DeleteObjectTagging_success_status,
-		"DeleteObjectTagging_success":                         DeleteObjectTagging_success,
-		"CreateMultipartUpload_non_existing_bucket":           CreateMultipartUpload_non_existing_bucket,
-		"CreateMultipartUpload_success":                       CreateMultipartUpload_success,
-		"UploadPart_non_existing_bucket":                      UploadPart_non_existing_bucket,
-		"UploadPart_invalid_part_number":                      UploadPart_invalid_part_number,
-		"UploadPart_non_existing_key":                         UploadPart_non_existing_key,
-		"UploadPart_non_existing_mp_upload":                   UploadPart_non_existing_mp_upload,
-		"UploadPart_success":                                  UploadPart_success,
-		"UploadPartCopy_non_existing_bucket":                  UploadPartCopy_non_existing_bucket,
-		"UploadPartCopy_incorrect_uploadId":                   UploadPartCopy_incorrect_uploadId,
-		"UploadPartCopy_incorrect_object_key":                 UploadPartCopy_incorrect_object_key,
-		"UploadPartCopy_invalid_part_number":                  UploadPartCopy_invalid_part_number,
-		"UploadPartCopy_invalid_copy_source":                  UploadPartCopy_invalid_copy_source,
-		"UploadPartCopy_non_existing_source_bucket":           UploadPartCopy_non_existing_source_bucket,
-		"UploadPartCopy_non_existing_source_object_key":       UploadPartCopy_non_existing_source_object_key,
-		"UploadPartCopy_success":                              UploadPartCopy_success,
-		"UploadPartCopy_by_range_invalid_range":               UploadPartCopy_by_range_invalid_range,
-		"UploadPartCopy_greater_range_than_obj_size":          UploadPartCopy_greater_range_than_obj_size,
-		"UploadPartCopy_by_range_success":                     UploadPartCopy_by_range_success,
-		"ListParts_incorrect_uploadId":                        ListParts_incorrect_uploadId,
-		"ListParts_incorrect_object_key":                      ListParts_incorrect_object_key,
-		"ListParts_success":                                   ListParts_success,
-		"ListMultipartUploads_non_existing_bucket":            ListMultipartUploads_non_existing_bucket,
-		"ListMultipartUploads_empty_result":                   ListMultipartUploads_empty_result,
-		"ListMultipartUploads_invalid_max_uploads":            ListMultipartUploads_invalid_max_uploads,
-		"ListMultipartUploads_max_uploads":                    ListMultipartUploads_max_uploads,
-		"ListMultipartUploads_incorrect_next_key_marker":      ListMultipartUploads_incorrect_next_key_marker,
-		"ListMultipartUploads_ignore_upload_id_marker":        ListMultipartUploads_ignore_upload_id_marker,
-		"ListMultipartUploads_success":                        ListMultipartUploads_success,
-		"AbortMultipartUpload_non_existing_bucket":            AbortMultipartUpload_non_existing_bucket,
-		"AbortMultipartUpload_incorrect_uploadId":             AbortMultipartUpload_incorrect_uploadId,
-		"AbortMultipartUpload_incorrect_object_key":           AbortMultipartUpload_incorrect_object_key,
-		"AbortMultipartUpload_success":                        AbortMultipartUpload_success,
-		"AbortMultipartUpload_success_status_code":            AbortMultipartUpload_success_status_code,
-		"CompletedMultipartUpload_non_existing_bucket":        CompletedMultipartUpload_non_existing_bucket,
-		"CompleteMultipartUpload_invalid_part_number":         CompleteMultipartUpload_invalid_part_number,
-		"CompleteMultipartUpload_invalid_ETag":                CompleteMultipartUpload_invalid_ETag,
-		"CompleteMultipartUpload_success":                     CompleteMultipartUpload_success,
-		"PutBucketAcl_non_existing_bucket":                    PutBucketAcl_non_existing_bucket,
-		"PutBucketAcl_invalid_acl_canned_and_acp":             PutBucketAcl_invalid_acl_canned_and_acp,
-		"PutBucketAcl_invalid_acl_canned_and_grants":          PutBucketAcl_invalid_acl_canned_and_grants,
-		"PutBucketAcl_invalid_acl_acp_and_grants":             PutBucketAcl_invalid_acl_acp_and_grants,
-		"PutBucketAcl_invalid_owner":                          PutBucketAcl_invalid_owner,
-		"PutBucketAcl_success_access_denied":                  PutBucketAcl_success_access_denied,
-		"PutBucketAcl_success_grants":                         PutBucketAcl_success_grants,
-		"PutBucketAcl_success_canned_acl":                     PutBucketAcl_success_canned_acl,
-		"PutBucketAcl_success_acp":                            PutBucketAcl_success_acp,
-		"GetBucketAcl_non_existing_bucket":                    GetBucketAcl_non_existing_bucket,
-		"GetBucketAcl_access_denied":                          GetBucketAcl_access_denied,
-		"GetBucketAcl_success":                                GetBucketAcl_success,
-		"PutObject_overwrite_dir_obj":                         PutObject_overwrite_dir_obj,
-		"PutObject_overwrite_file_obj":                        PutObject_overwrite_file_obj,
-		"PutObject_dir_obj_with_data":                         PutObject_dir_obj_with_data,
-		"CreateMultipartUpload_dir_obj":                       CreateMultipartUpload_dir_obj,
-	}
-}

integration/utils.go

@@ -60,7 +60,7 @@ func teardown(s *S3Conf, bucket string) error {
 		})
 		cancel()
 		if err != nil {
-			return fmt.Errorf("failed to delete object %v: %w", *key, err)
+			return fmt.Errorf("failed to delete object %v: %v", *key, err)
 		}
 		return nil
 	}
@@ -71,7 +71,7 @@ func teardown(s *S3Conf, bucket string) error {
 		out, err := s3client.ListObjectsV2(ctx, in)
 		cancel()
 		if err != nil {
-			return fmt.Errorf("failed to list objects: %w", err)
+			return fmt.Errorf("failed to list objects: %v", err)
 		}
 
 		for _, item := range out.Contents {
@@ -81,7 +81,7 @@ func teardown(s *S3Conf, bucket string) error {
 			}
 		}
 
-		if *out.IsTruncated {
+		if out.IsTruncated {
 			in.ContinuationToken = out.ContinuationToken
 		} else {
 			break
@@ -96,32 +96,31 @@ func teardown(s *S3Conf, bucket string) error {
 	return err
 }
 
-func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) error {
+func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) {
 	runF(testName)
 	bucketName := getBucketName()
 	err := setup(s, bucketName)
 	if err != nil {
-		failF("%v: failed to create a bucket: %v", testName, err)
-		return fmt.Errorf("%v: failed to create a bucket: %w", testName, err)
+		failF("%v: failed to create a bucket: %v", testName, err.Error())
+		return
 	}
 	client := s3.NewFromConfig(s.Config())
 	handlerErr := handler(client, bucketName)
 	if handlerErr != nil {
-		failF("%v: %v", testName, handlerErr)
+		failF("%v: %v", testName, handlerErr.Error())
 	}
 
 	err = teardown(s, bucketName)
 	if err != nil {
-		fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err)
 		if handlerErr == nil {
-			return fmt.Errorf("%v: failed to delete the bucket: %w", testName, err)
+			failF("%v: failed to delete the bucket: %v", testName, err.Error())
+		} else {
+			fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err.Error())
 		}
 	}
 	if handlerErr == nil {
 		passF(testName)
 	}
-
-	return handlerErr
 }
 
 type authConfig struct {
@@ -133,21 +132,20 @@ type authConfig struct {
 	date     time.Time
 }
 
-func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) error {
+func authHandler(s *S3Conf, cfg *authConfig, handler func(req *http.Request) error) {
 	runF(cfg.testName)
 	req, err := createSignedReq(cfg.method, s.endpoint, cfg.path, s.awsID, s.awsSecret, cfg.service, s.awsRegion, cfg.body, cfg.date)
 	if err != nil {
-		failF("%v: %v", cfg.testName, err)
-		return fmt.Errorf("%v: %w", cfg.testName, err)
+		failF("%v: %v", cfg.testName, err.Error())
+		return
 	}
 
 	err = handler(req)
 	if err != nil {
-		failF("%v: %v", cfg.testName, err)
-		return fmt.Errorf("%v: %w", cfg.testName, err)
+		failF("%v: %v", cfg.testName, err.Error())
+		return
 	}
 	passF(cfg.testName)
-	return nil
 }
 
 func createSignedReq(method, endpoint, path, access, secret, service, region string, body []byte, date time.Time) (*http.Request, error) {
@@ -207,8 +205,9 @@ func checkApiErr(err error, apiErr s3err.APIError) error {
 		}
 
 		return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode())
+	} else {
+		return fmt.Errorf("expected aws api error, instead got: %v", err.Error())
 	}
-	return fmt.Errorf("expected aws api error, instead got: %w", err)
 }
 
 func checkSdkApiErr(err error, code string) error {
@@ -294,7 +293,7 @@ func compareParts(parts1, parts2 []types.Part) bool {
 	}
 
 	for i, prt := range parts1 {
-		if *prt.PartNumber != *parts2[i].PartNumber {
+		if prt.PartNumber != parts2[i].PartNumber {
 			return false
 		}
 		if *prt.ETag != *parts2[i].ETag {
@@ -485,23 +484,20 @@ func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId s
 			return parts, err
 		}
 		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
-		pn := int32(partNumber)
 		out, err := client.UploadPart(ctx, &s3.UploadPartInput{
 			Bucket:     &bucket,
 			Key:        &key,
 			UploadId:   &uploadId,
 			Body:       bytes.NewReader(partBuffer),
-			PartNumber: &pn,
+			PartNumber: int32(partNumber),
 		})
 		cancel()
 		if err != nil {
 			return parts, err
+		} else {
+			parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)})
+			offset += partSize
 		}
-		parts = append(parts, types.Part{
-			ETag:       out.ETag,
-			PartNumber: &pn,
-		})
-		offset += partSize
 	}
 
 	return parts, err

s3api/controllers/backend_moq_test.go

@@ -4,7 +4,6 @@
 package controllers
 
 import (
-	"bufio"
 	"context"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/versity/versitygw/backend"
@@ -107,7 +106,7 @@ var _ backend.Backend = &BackendMock{}
 //			RestoreObjectFunc: func(contextMoqParam context.Context, restoreObjectInput *s3.RestoreObjectInput) error {
 //				panic("mock out the RestoreObject method")
 //			},
-//			SelectObjectContentFunc: func(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
+//			SelectObjectContentFunc: func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
 //				panic("mock out the SelectObjectContent method")
 //			},
 //			ShutdownFunc: func()  {
@@ -214,7 +213,7 @@ type BackendMock struct {
 	RestoreObjectFunc func(contextMoqParam context.Context, restoreObjectInput *s3.RestoreObjectInput) error
 
 	// SelectObjectContentFunc mocks the SelectObjectContent method.
-	SelectObjectContentFunc func(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer)
+	SelectObjectContentFunc func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error)
 
 	// ShutdownFunc mocks the Shutdown method.
 	ShutdownFunc func()
@@ -442,10 +441,10 @@ type BackendMock struct {
 		}
 		// SelectObjectContent holds details about calls to the SelectObjectContent method.
 		SelectObjectContent []struct {
-			// Ctx is the ctx argument value.
-			Ctx context.Context
-			// Input is the input argument value.
-			Input *s3.SelectObjectContentInput
+			// ContextMoqParam is the contextMoqParam argument value.
+			ContextMoqParam context.Context
+			// SelectObjectContentInput is the selectObjectContentInput argument value.
+			SelectObjectContentInput *s3.SelectObjectContentInput
 		}
 		// Shutdown holds details about calls to the Shutdown method.
 		Shutdown []struct {
@@ -1540,21 +1539,21 @@ func (mock *BackendMock) RestoreObjectCalls() []struct {
 }
 
 // SelectObjectContent calls SelectObjectContentFunc.
-func (mock *BackendMock) SelectObjectContent(ctx context.Context, input *s3.SelectObjectContentInput) func(w *bufio.Writer) {
+func (mock *BackendMock) SelectObjectContent(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
 	if mock.SelectObjectContentFunc == nil {
 		panic("BackendMock.SelectObjectContentFunc: method is nil but Backend.SelectObjectContent was just called")
 	}
 	callInfo := struct {
-		Ctx   context.Context
-		Input *s3.SelectObjectContentInput
+		ContextMoqParam          context.Context
+		SelectObjectContentInput *s3.SelectObjectContentInput
 	}{
-		Ctx:   ctx,
-		Input: input,
+		ContextMoqParam:          contextMoqParam,
+		SelectObjectContentInput: selectObjectContentInput,
 	}
 	mock.lockSelectObjectContent.Lock()
 	mock.calls.SelectObjectContent = append(mock.calls.SelectObjectContent, callInfo)
 	mock.lockSelectObjectContent.Unlock()
-	return mock.SelectObjectContentFunc(ctx, input)
+	return mock.SelectObjectContentFunc(contextMoqParam, selectObjectContentInput)
 }
 
 // SelectObjectContentCalls gets all the calls that were made to SelectObjectContent.
@@ -1562,12 +1561,12 @@ func (mock *BackendMock) SelectObjectContent(ctx context.Context, input *s3.Sele
 //
 //	len(mockedBackend.SelectObjectContentCalls())
 func (mock *BackendMock) SelectObjectContentCalls() []struct {
-	Ctx   context.Context
-	Input *s3.SelectObjectContentInput
+	ContextMoqParam          context.Context
+	SelectObjectContentInput *s3.SelectObjectContentInput
 } {
 	var calls []struct {
-		Ctx   context.Context
-		Input *s3.SelectObjectContentInput
+		ContextMoqParam          context.Context
+		SelectObjectContentInput *s3.SelectObjectContentInput
 	}
 	mock.lockSelectObjectContent.RLock()
 	calls = mock.calls.SelectObjectContent

s3api/controllers/base.go

@@ -63,7 +63,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	key := ctx.Params("key")
 	keyEnd := ctx.Params("*1")
 	uploadId := ctx.Query("uploadId")
-	maxParts := int32(ctx.QueryInt("max-parts", 0))
+	maxParts := ctx.QueryInt("max-parts", 0)
 	partNumberMarker := ctx.Query("part-number-marker")
 	acceptRange := ctx.Get("Range")
 	acct := ctx.Locals("account").(auth.Account)
@@ -111,7 +111,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 			Key:              &key,
 			UploadId:         &uploadId,
 			PartNumberMarker: &partNumberMarker,
-			MaxParts:         &maxParts,
+			MaxParts:         int32(maxParts),
 		})
 		return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner})
 	}
@@ -169,7 +169,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	utils.SetResponseHeaders(ctx, []utils.CustomHeader{
 		{
 			Key:   "Content-Length",
-			Value: fmt.Sprint(getint64(res.ContentLength)),
+			Value: fmt.Sprint(res.ContentLength),
 		},
 		{
 			Key:   "Content-Type",
@@ -199,17 +199,11 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 			Key:   "accept-ranges",
 			Value: getstring(res.AcceptRanges),
 		},
+		{
+			Key:   "x-amz-tagging-count",
+			Value: fmt.Sprint(res.TagCount),
+		},
 	})
-
-	if res.TagCount != nil {
-		utils.SetResponseHeaders(ctx, []utils.CustomHeader{
-			{
-				Key:   "x-amz-tagging-count",
-				Value: fmt.Sprint(*res.TagCount),
-			},
-		})
-	}
-
 	return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner})
 }
 
@@ -220,13 +214,6 @@ func getstring(s *string) string {
 	return *s
 }
 
-func getint64(i *int64) int64 {
-	if i == nil {
-		return 0
-	}
-	return *i
-}
-
 func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 	bucket := ctx.Params("bucket")
 	prefix := ctx.Query("prefix")
@@ -272,7 +259,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 			Delimiter:      &delimiter,
 			Prefix:         &prefix,
 			UploadIdMarker: &uploadIdMarker,
-			MaxUploads:     &maxUploads,
+			MaxUploads:     maxUploads,
 			KeyMarker:      &keyMarker,
 		})
 		return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner})
@@ -295,7 +282,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 			Prefix:            &prefix,
 			ContinuationToken: &cToken,
 			Delimiter:         &delimiter,
-			MaxKeys:           &maxkeys,
+			MaxKeys:           maxkeys,
 		})
 		return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner})
 	}
@@ -318,7 +305,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error {
 		Prefix:    &prefix,
 		Marker:    &marker,
 		Delimiter: &delimiter,
-		MaxKeys:   &maxkeys,
+		MaxKeys:   maxkeys,
 	})
 	return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner})
 }
@@ -480,7 +467,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") && copySource != "" {
-		partNumber := int32(ctx.QueryInt("partNumber", -1))
+		partNumber := ctx.QueryInt("partNumber", -1)
 		if partNumber < 1 || partNumber > 10000 {
 			return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPartCopy", BucketOwner: parsedAcl.Owner})
 		}
@@ -489,7 +476,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			Bucket:              &bucket,
 			Key:                 &keyStart,
 			CopySource:          &copySource,
-			PartNumber:          &partNumber,
+			PartNumber:          int32(partNumber),
 			UploadId:            &uploadId,
 			ExpectedBucketOwner: &bucketOwner,
 			CopySourceRange:     &copySrcRange,
@@ -498,7 +485,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	}
 
 	if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") {
-		partNumber := int32(ctx.QueryInt("partNumber", -1))
+		partNumber := ctx.QueryInt("partNumber", -1)
 		if partNumber < 1 || partNumber > 10000 {
 			return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner})
 		}
@@ -518,8 +505,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 			Bucket:        &bucket,
 			Key:           &keyStart,
 			UploadId:      &uploadId,
-			PartNumber:    &partNumber,
-			ContentLength: &contentLength,
+			PartNumber:    int32(partNumber),
+			ContentLength: contentLength,
 			Body:          body,
 		})
 		ctx.Response().Header.Set("Etag", etag)
@@ -654,7 +641,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
 	etag, err := c.be.PutObject(ctx.Context(), &s3.PutObjectInput{
 		Bucket:        &bucket,
 		Key:           &keyStart,
-		ContentLength: &contentLength,
+		ContentLength: contentLength,
 		Metadata:      metadata,
 		Body:          bytes.NewReader(ctx.Request().Body()),
 		Tagging:       &tagging,
@@ -818,7 +805,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
 	utils.SetResponseHeaders(ctx, []utils.CustomHeader{
 		{
 			Key:   "Content-Length",
-			Value: fmt.Sprint(getint64(res.ContentLength)),
+			Value: fmt.Sprint(res.ContentLength),
 		},
 		{
 			Key:   "Content-Type",
@@ -906,7 +893,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "SelectObjectContent", BucketOwner: parsedAcl.Owner})
 		}
 
-		sw := c.be.SelectObjectContent(ctx.Context(), &s3.SelectObjectContentInput{
+		res, err := c.be.SelectObjectContent(ctx.Context(), &s3.SelectObjectContentInput{
 			Bucket:              &bucket,
 			Key:                 &key,
 			Expression:          payload.Expression,
@@ -916,10 +903,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
 			RequestProgress:     payload.RequestProgress,
 			ScanRange:           payload.ScanRange,
 		})
-
-		ctx.Context().SetBodyStreamWriter(sw)
-
-		return nil
+		return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "SelectObjectContent", BucketOwner: parsedAcl.Owner})
 	}
 
 	if uploadId != "" {

s3api/controllers/base_test.go

@@ -15,7 +15,6 @@
 package controllers
 
 import (
-	"bufio"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -175,7 +174,6 @@ func TestS3ApiController_GetActions(t *testing.T) {
 	now := time.Now()
 
 	app := fiber.New()
-	contentLength := int64(1000)
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
 			GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) {
@@ -196,7 +194,7 @@ func TestS3ApiController_GetActions(t *testing.T) {
 					ContentType:     getPtr("application/xml"),
 					ContentEncoding: getPtr("gzip"),
 					ETag:            getPtr("98sda7f97sa9df798sd79f8as9df"),
-					ContentLength:   &contentLength,
+					ContentLength:   1000,
 					LastModified:    &now,
 					StorageClass:    "storage class",
 				}, nil
@@ -1200,7 +1198,6 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 	contentType := "application/xml"
 	eTag := "Valid etag"
 	lastModifie := time.Now()
-	contentLength := int64(64)
 
 	s3ApiController := S3ApiController{
 		be: &BackendMock{
@@ -1210,7 +1207,7 @@ func TestS3ApiController_HeadObject(t *testing.T) {
 			HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
 				return &s3.HeadObjectOutput{
 					ContentEncoding: &contentEncoding,
-					ContentLength:   &contentLength,
+					ContentLength:   64,
 					ContentType:     &contentType,
 					LastModified:    &lastModifie,
 					ETag:            &eTag,
@@ -1309,8 +1306,8 @@ func TestS3ApiController_CreateActions(t *testing.T) {
 			CreateMultipartUploadFunc: func(context.Context, *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
 				return &s3.CreateMultipartUploadOutput{}, nil
 			},
-			SelectObjectContentFunc: func(context.Context, *s3.SelectObjectContentInput) func(w *bufio.Writer) {
-				return func(w *bufio.Writer) {}
+			SelectObjectContentFunc: func(contextMoqParam context.Context, selectObjectContentInput *s3.SelectObjectContentInput) (s3response.SelectObjectContentResult, error) {
+				return s3response.SelectObjectContentResult{}, nil
 			},
 		},
 	}

s3select/message-handler.go

@@ -1,358 +0,0 @@
-// Copyright 2023 Versity Software
-// This file is licensed under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package s3select
-
-import (
-	"bufio"
-	"context"
-	"encoding/binary"
-	"encoding/xml"
-	"fmt"
-	"hash/crc32"
-	"sync"
-	"sync/atomic"
-	"time"
-)
-
-// Protocol definition for messages can be found here:
-// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTSelectObjectAppendix.html
-
-var (
-	// From ptotocol def:
-	// Enum indicating the header value type.
-	// For Amazon S3 Select, this is always 7.
-	headerValueType = byte(7)
-)
-
-func intToTwoBytes(i int) []byte {
-	return []byte{byte(i >> 8), byte(i)}
-}
-
-func generateHeader(messages ...string) []byte {
-	var header []byte
-
-	for i, message := range messages {
-		if i%2 == 1 {
-			header = append(header, headerValueType)
-			header = append(header, intToTwoBytes(len(message))...)
-		} else {
-			header = append(header, byte(len(message)))
-		}
-		header = append(header, message...)
-	}
-
-	return header
-}
-
-func generateOctetHeader(message string) []byte {
-	return generateHeader(
-		":message-type",
-		"event",
-		":content-type",
-		"application/octet-stream",
-		":event-type",
-		message)
-}
-
-func generateTextHeader(message string) []byte {
-	return generateHeader(
-		":message-type",
-		"event",
-		":content-type",
-		"text/xml",
-		":event-type",
-		message)
-}
-
-func generateNoContentHeader(message string) []byte {
-	return generateHeader(
-		":message-type",
-		"event",
-		":event-type",
-		message)
-}
-
-const (
-	// 4 bytes total byte len +
-	// 4 bytes headers bytes len +
-	// 4 bytes prelude CRC
-	preludeLen = 12
-	// CRC is uint32
-	msgCrcLen = 4
-)
-
-var (
-	recordsHeader       = generateOctetHeader("Records")
-	continuationHeader  = generateNoContentHeader("Cont")
-	continuationMessage = genMessage(continuationHeader, []byte{})
-	progressHeader      = generateTextHeader("Progress")
-	statsHeader         = generateTextHeader("Stats")
-	endHeader           = generateNoContentHeader("End")
-	endMessage          = genMessage(endHeader, []byte{})
-)
-
-func uintToBytes(n uint32) []byte {
-	b := make([]byte, 4)
-	binary.BigEndian.PutUint32(b, n)
-	return b
-}
-
-func generatePrelude(msgLen int, headerLen int) []byte {
-	prelude := make([]byte, 0, preludeLen)
-
-	// 4 bytes total byte len
-	prelude = append(prelude, uintToBytes(uint32(msgLen+headerLen+preludeLen+msgCrcLen))...)
-	// 4 bytes headers bytes len
-	prelude = append(prelude, uintToBytes(uint32(headerLen))...)
-	// 4 bytes prelude CRC
-	prelude = append(prelude, uintToBytes(crc32.ChecksumIEEE(prelude))...)
-
-	return prelude
-}
-
-const (
-	maxHeaderSize  = 1024 * 1024
-	maxMessageSize = 5 * 1024 * 1024 * 1024
-)
-
-func genMessage(header, payload []byte) []byte {
-	var msg []byte
-	// below is always true since the size is validated
-	// in the send record
-	if len(header) <= maxHeaderSize && len(payload) <= maxMessageSize {
-		msglen := preludeLen + len(header) + len(payload) + msgCrcLen
-		msg = make([]byte, 0, msglen)
-	}
-
-	msg = append(msg, generatePrelude(len(payload), len(header))...)
-	msg = append(msg, header...)
-	msg = append(msg, payload...)
-	msg = append(msg, uintToBytes(crc32.ChecksumIEEE(msg))...)
-
-	return msg
-}
-
-func genRecordsMessage(payload []byte) []byte {
-	return genMessage(recordsHeader, payload)
-}
-
-type progress struct {
-	XMLName        xml.Name `xml:"Progress"`
-	BytesScanned   int64    `xml:"BytesScanned"`
-	BytesProcessed int64    `xml:"BytesProcessed"`
-	BytesReturned  int64    `xml:"BytesReturned"`
-}
-
-func genProgressMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte {
-	progress := progress{
-		BytesScanned:   bytesScanned,
-		BytesProcessed: bytesProcessed,
-		BytesReturned:  bytesReturned,
-	}
-
-	xmlData, _ := xml.MarshalIndent(progress, "", "    ")
-	payload := []byte(xml.Header + string(xmlData))
-	return genMessage(progressHeader, payload)
-}
-
-type stats struct {
-	XMLName        xml.Name `xml:"Stats"`
-	BytesScanned   int64    `xml:"BytesScanned"`
-	BytesProcessed int64    `xml:"BytesProcessed"`
-	BytesReturned  int64    `xml:"BytesReturned"`
-}
-
-func genStatsMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte {
-	stats := stats{
-		BytesScanned:   bytesScanned,
-		BytesProcessed: bytesProcessed,
-		BytesReturned:  bytesReturned,
-	}
-
-	xmlData, _ := xml.MarshalIndent(stats, "", "    ")
-	payload := []byte(xml.Header + string(xmlData))
-	return genMessage(statsHeader, payload)
-}
-
-func genErrorMessage(errorCode, errorMessage string) []byte {
-	return genMessage(generateHeader(
-		":error-code",
-		errorCode,
-		":error-message",
-		errorMessage,
-		":message-type",
-		"error",
-	), []byte{})
-}
-
-// GetProgress is a callback function that periodically retrieves the current
-// values for the following if not nil.  This is used to send Progress
-// messages back to client.
-// BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed).
-// BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed).
-type GetProgress func() (bytesScanned int64, bytesProcessed int64)
-
-type MessageHandler struct {
-	sync.Mutex
-	ctx           context.Context
-	cancel        context.CancelFunc
-	writer        *bufio.Writer
-	data          chan []byte
-	getProgress   GetProgress
-	stopCh        chan bool
-	resetCh       chan bool
-	bytesReturned int64
-}
-
-// NewMessageHandler creates a new MessageHandler instance and starts the event streaming
-func NewMessageHandler(ctx context.Context, w *bufio.Writer, getProgressFunc GetProgress) *MessageHandler {
-	ctx, cancel := context.WithCancel(ctx)
-
-	mh := &MessageHandler{
-		ctx:         ctx,
-		cancel:      cancel,
-		writer:      w,
-		data:        make(chan []byte),
-		getProgress: getProgressFunc,
-		resetCh:     make(chan bool),
-		stopCh:      make(chan bool),
-	}
-
-	go mh.sendBackgroundMessages(mh.resetCh, mh.stopCh)
-	return mh
-}
-
-func (mh *MessageHandler) write(data []byte) error {
-	mh.Lock()
-	defer mh.Unlock()
-
-	mh.stopCh <- true
-	defer func() { mh.resetCh <- true }()
-
-	_, err := mh.writer.Write(data)
-	if err != nil {
-		return err
-	}
-
-	return mh.writer.Flush()
-}
-
-const (
-	continuationInterval = time.Second
-	progressInterval     = time.Minute
-)
-
-func (mh *MessageHandler) sendBackgroundMessages(resetCh, stopCh <-chan bool) {
-	continuationTicker := time.NewTicker(continuationInterval)
-	defer continuationTicker.Stop()
-
-	var progressTicker *time.Ticker
-	var progressTickerChan <-chan time.Time
-	if mh.getProgress != nil {
-		progressTicker = time.NewTicker(progressInterval)
-		progressTickerChan = progressTicker.C
-		defer progressTicker.Stop()
-	}
-
-Loop:
-	for {
-		select {
-		case <-mh.ctx.Done():
-			break Loop
-
-		case <-continuationTicker.C:
-			err := mh.write(continuationMessage)
-			if err != nil {
-				mh.cancel()
-				break Loop
-			}
-
-		case <-resetCh:
-			continuationTicker.Reset(continuationInterval)
-
-		case <-stopCh:
-			continuationTicker.Stop()
-
-		case <-progressTickerChan:
-			var bytesScanned, bytesProcessed int64
-			if mh.getProgress != nil {
-				bytesScanned, bytesProcessed = mh.getProgress()
-			}
-			bytesReturned := atomic.LoadInt64(&mh.bytesReturned)
-			err := mh.write(genProgressMessage(bytesScanned, bytesProcessed, bytesReturned))
-			if err != nil {
-				mh.cancel()
-				break Loop
-			}
-		}
-	}
-}
-
-// SendRecord sends a single Records message
-func (mh *MessageHandler) SendRecord(payload []byte) error {
-	if mh.ctx.Err() != nil {
-		return mh.ctx.Err()
-	}
-
-	if len(payload) > maxMessageSize {
-		return fmt.Errorf("record max size exceeded")
-	}
-
-	err := mh.write(genRecordsMessage(payload))
-	if err != nil {
-		return err
-	}
-
-	atomic.AddInt64(&mh.bytesReturned, int64(len(payload)))
-	return nil
-}
-
-// Finish terminates message stream with Stats and End message
-// generates stats and end message using function args based on:
-// BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed).
-// BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed).
-func (mh *MessageHandler) Finish(bytesScanned, bytesProcessed int64) error {
-	if mh.ctx.Err() != nil {
-		return mh.ctx.Err()
-	}
-
-	bytesReturned := atomic.LoadInt64(&mh.bytesReturned)
-	err := mh.write(genStatsMessage(bytesScanned, bytesProcessed, bytesReturned))
-	if err != nil {
-		return err
-	}
-
-	err = mh.write(endMessage)
-	if err != nil {
-		return err
-	}
-
-	mh.cancel()
-	return nil
-}
-
-// FinishWithError terminates event stream with error
-func (mh *MessageHandler) FinishWithError(errorCode, errorMessage string) error {
-	if mh.ctx.Err() != nil {
-		return mh.ctx.Err()
-	}
-	err := mh.write(genErrorMessage(errorCode, errorMessage))
-	if err != nil {
-		return err
-	}
-
-	mh.cancel()
-	return nil
-}