From cd8ad7d4823e9ec4d54de2bda4aed0615e135ce9 Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Tue, 28 Nov 2023 09:45:28 -0800 Subject: [PATCH] fix: breaking changes with aws sdk updates --- backend/posix/posix.go | 259 ++++++++++++++++++++++++++++----- backend/s3proxy/s3.go | 12 +- backend/scoutfs/scoutfs.go | 68 +++++++-- backend/walk_test.go | 4 +- integration/tests.go | 113 ++++++++------ integration/utils.go | 12 +- s3api/controllers/base.go | 47 +++--- s3api/controllers/base_test.go | 6 +- 8 files changed, 394 insertions(+), 127 deletions(-) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 02d6867..a5c88da 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -146,6 +146,10 @@ func (p *Posix) ListBuckets(_ context.Context, owner string, isAdmin bool) (s3re } func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + _, err := os.Lstat(*input.Bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -158,6 +162,10 @@ func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.He } func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput) error { + if input.Bucket == nil { + return s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + bucket := *input.Bucket owner := string(input.ObjectOwnership) @@ -183,6 +191,10 @@ func (p *Posix) CreateBucket(_ context.Context, input *s3.CreateBucketInput) err } func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) error { + if input.Bucket == nil { + return s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + names, err := os.ReadDir(*input.Bucket) if errors.Is(err, fs.ErrNotExist) { return s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -212,6 +224,13 @@ func (p *Posix) DeleteBucket(_ context.Context, input *s3.DeleteBucketInput) err } func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) { + if mpu.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if mpu.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + bucket := *mpu.Bucket object := *mpu.Key @@ -269,6 +288,19 @@ func (p *Posix) CreateMultipartUpload(_ context.Context, mpu *s3.CreateMultipart } func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if input.UploadId == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + if input.MultipartUpload == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + bucket := *input.Bucket object := *input.Key uploadID := *input.UploadId @@ -294,7 +326,7 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul partsize := int64(0) var totalsize int64 for i, p := range parts { - partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber)) + partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber)) fi, err := os.Lstat(partPath) if err != nil { return nil, s3err.GetAPIError(s3err.ErrInvalidPart) @@ -326,7 +358,7 @@ func (p *Posix) CompleteMultipartUpload(_ context.Context, input *s3.CompleteMul defer f.cleanup() for _, p := range parts { - pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", p.PartNumber))) + pf, err := os.Open(filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *p.PartNumber))) if err != nil { return nil, fmt.Errorf("open part %v: %v", p.PartNumber, err) } @@ -512,6 +544,16 @@ func mkdirAll(path string, perm os.FileMode, bucket, object string) error { } func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUploadInput) error { + if mpu.Bucket == nil { + return s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if mpu.Key == nil { + return s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if mpu.UploadId == nil { + return s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + bucket := *mpu.Bucket object := *mpu.Key uploadID := *mpu.UploadId @@ -542,6 +584,12 @@ func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUp } func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) { + var lmu s3response.ListMultipartUploadsResult + + if mpu.Bucket == nil { + return lmu, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + bucket := *mpu.Bucket var delimiter string if mpu.Delimiter != nil { @@ -552,8 +600,6 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl prefix = *mpu.Prefix } - var lmu s3response.ListMultipartUploadsResult - _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return lmu, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -626,12 +672,16 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl } } + maxUploads := 0 + if mpu.MaxUploads != nil { + maxUploads = int(*mpu.MaxUploads) + } if (uploadIDMarker != "" && !uploadIdMarkerFound) || (keyMarker != "" && keyMarkerInd == -1) { return s3response.ListMultipartUploadsResult{ Bucket: bucket, Delimiter: delimiter, KeyMarker: keyMarker, - MaxUploads: int(mpu.MaxUploads), + MaxUploads: maxUploads, Prefix: prefix, UploadIDMarker: uploadIDMarker, Uploads: []s3response.Upload{}, @@ -643,18 +693,18 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl }) for i := keyMarkerInd + 1; i < len(uploads); i++ { - if mpu.MaxUploads == 0 { + if maxUploads == 0 { break } if keyMarker != "" && uploadIDMarker != "" && uploads[i].UploadID < uploadIDMarker { continue } - if i != len(uploads)-1 && len(resultUpds) == int(mpu.MaxUploads) { + if i != len(uploads)-1 && len(resultUpds) == maxUploads { return s3response.ListMultipartUploadsResult{ Bucket: bucket, Delimiter: delimiter, KeyMarker: keyMarker, - MaxUploads: int(mpu.MaxUploads), + MaxUploads: maxUploads, NextKeyMarker: resultUpds[i-1].Key, NextUploadIDMarker: resultUpds[i-1].UploadID, IsTruncated: true, @@ -671,7 +721,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl Bucket: bucket, Delimiter: delimiter, KeyMarker: keyMarker, - MaxUploads: int(mpu.MaxUploads), + MaxUploads: maxUploads, Prefix: prefix, UploadIDMarker: uploadIDMarker, Uploads: resultUpds, @@ -679,13 +729,29 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl } func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) { + var lpr s3response.ListPartsResult + + if input.Bucket == nil { + return lpr, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return lpr, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if input.UploadId == nil { + return lpr, s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + bucket := *input.Bucket object := *input.Key uploadID := *input.UploadId - stringMarker := *input.PartNumberMarker - maxParts := int(input.MaxParts) - - var lpr s3response.ListPartsResult + stringMarker := "" + if input.PartNumberMarker != nil { + stringMarker = *input.PartNumberMarker + } + maxParts := 0 + if input.MaxParts != nil { + maxParts = int(*input.MaxParts) + } var partNumberMarker int if stringMarker != "" { @@ -777,11 +843,21 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon } func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string, error) { + if input.Bucket == nil { + return "", s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return "", s3err.GetAPIError(s3err.ErrNoSuchKey) + } + bucket := *input.Bucket object := *input.Key uploadID := *input.UploadId part := input.PartNumber - length := input.ContentLength + length := int64(0) + if input.ContentLength != nil { + length = *input.ContentLength + } r := input.Body _, err := os.Stat(bucket) @@ -803,7 +879,7 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string return "", fmt.Errorf("stat uploadid: %w", err) } - partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", part)) + partPath := filepath.Join(objdir, uploadID, fmt.Sprintf("%v", *part)) f, err := openTmpFile(filepath.Join(bucket, objdir), bucket, partPath, length) @@ -833,6 +909,13 @@ func (p *Posix) UploadPart(_ context.Context, input *s3.UploadPartInput) (string } func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { + if upi.Bucket == nil { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if upi.Key == nil { + return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + _, err := os.Stat(*upi.Bucket) if errors.Is(err, fs.ErrNotExist) { return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -852,7 +935,7 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) ( return s3response.CopyObjectResult{}, fmt.Errorf("stat uploadid: %w", err) } - partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", upi.PartNumber)) + partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", *upi.PartNumber)) substrs := strings.SplitN(*upi.CopySource, "/", 2) if len(substrs) != 2 { @@ -938,6 +1021,13 @@ func (p *Posix) UploadPartCopy(_ context.Context, upi *s3.UploadPartCopyInput) ( } func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, error) { + if po.Bucket == nil { + return "", s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if po.Key == nil { + return "", s3err.GetAPIError(s3err.ErrNoSuchKey) + } + tagsStr := getString(po.Tagging) tags := make(map[string]string) _, err := os.Stat(*po.Bucket) @@ -964,9 +1054,13 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e name := filepath.Join(*po.Bucket, *po.Key) + contentLength := int64(0) + if po.ContentLength != nil { + contentLength = *po.ContentLength + } if strings.HasSuffix(*po.Key, "/") { // object is directory - if po.ContentLength != 0 { + if contentLength != 0 { // posix directories can't contain data, send error // if reuests has a data payload associated with a // directory object @@ -995,7 +1089,7 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e } f, err := openTmpFile(filepath.Join(*po.Bucket, metaTmpDir), - *po.Bucket, *po.Key, po.ContentLength) + *po.Bucket, *po.Key, contentLength) if err != nil { return "", fmt.Errorf("open temp file: %w", err) } @@ -1039,6 +1133,13 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e } func (p *Posix) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) error { + if input.Bucket == nil { + return s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return s3err.GetAPIError(s3err.ErrNoSuchKey) + } + bucket := *input.Bucket object := *input.Key @@ -1131,6 +1232,16 @@ func (p *Posix) DeleteObjects(ctx context.Context, input *s3.DeleteObjectsInput) } func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if input.Range == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidRange) + } + bucket := *input.Bucket _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -1192,15 +1303,17 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io return nil, fmt.Errorf("get object tags: %w", err) } + tagCount := int32(len(tags)) + return &s3.GetObjectOutput{ AcceptRanges: &acceptRange, - ContentLength: length, + ContentLength: &length, ContentEncoding: &contentEncoding, ContentType: &contentType, ETag: &etag, LastModified: backend.GetTimePtr(fi.ModTime()), Metadata: userMetaData, - TagCount: int32(len(tags)), + TagCount: &tagCount, ContentRange: &contentRange, }, nil } @@ -1235,20 +1348,28 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput, writer io return nil, fmt.Errorf("get object tags: %w", err) } + tagCount := int32(len(tags)) + return &s3.GetObjectOutput{ AcceptRanges: &acceptRange, - ContentLength: length, + ContentLength: &length, ContentEncoding: &contentEncoding, ContentType: &contentType, ETag: &etag, LastModified: backend.GetTimePtr(fi.ModTime()), Metadata: userMetaData, - TagCount: int32(len(tags)), + TagCount: &tagCount, ContentRange: &contentRange, }, nil } func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) + } bucket := *input.Bucket object := *input.Key @@ -1278,8 +1399,10 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He etag = "" } + size := fi.Size() + return &s3.HeadObjectOutput{ - ContentLength: fi.Size(), + ContentLength: &size, ContentType: &contentType, ContentEncoding: &contentEncoding, ETag: &etag, @@ -1289,6 +1412,18 @@ func (p *Posix) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3.He } func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + if input.Key == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidCopyDest) + } + if input.CopySource == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource) + } + if input.ExpectedBucketOwner == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidRequest) + } srcBucket, srcObject, ok := strings.Cut(*input.CopySource, "/") if !ok { return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource) @@ -1361,7 +1496,16 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. } } - etag, err := p.PutObject(ctx, &s3.PutObjectInput{Bucket: &dstBucket, Key: &dstObject, Body: f, ContentLength: fInfo.Size(), Metadata: meta}) + contentLength := fInfo.Size() + + etag, err := p.PutObject(ctx, + &s3.PutObjectInput{ + Bucket: &dstBucket, + Key: &dstObject, + Body: f, + ContentLength: &contentLength, + Metadata: meta, + }) if err != nil { return nil, err } @@ -1380,11 +1524,26 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. } func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } bucket := *input.Bucket - prefix := *input.Prefix - marker := *input.Marker - delim := *input.Delimiter - maxkeys := input.MaxKeys + prefix := "" + if input.Prefix != nil { + prefix = *input.Prefix + } + marker := "" + if input.Marker != nil { + marker = *input.Marker + } + delim := "" + if input.Delimiter != nil { + delim = *input.Delimiter + } + maxkeys := int32(0) + if input.MaxKeys != nil { + maxkeys = *input.MaxKeys + } _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -1405,9 +1564,9 @@ func (p *Posix) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3. CommonPrefixes: results.CommonPrefixes, Contents: results.Objects, Delimiter: &delim, - IsTruncated: results.Truncated, + IsTruncated: &results.Truncated, Marker: &marker, - MaxKeys: maxkeys, + MaxKeys: &maxkeys, Name: &bucket, NextMarker: &results.NextMarker, Prefix: &prefix, @@ -1466,21 +1625,38 @@ func fileToObj(bucket string) backend.GetObjFunc { return types.Object{}, fmt.Errorf("get fileinfo: %w", err) } + size := fi.Size() + return types.Object{ ETag: &etag, Key: &path, LastModified: backend.GetTimePtr(fi.ModTime()), - Size: fi.Size(), + Size: &size, }, nil } } func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } bucket := *input.Bucket - prefix := *input.Prefix - marker := *input.ContinuationToken - delim := *input.Delimiter - maxkeys := input.MaxKeys + prefix := "" + if input.Prefix != nil { + prefix = *input.Prefix + } + marker := "" + if input.ContinuationToken != nil { + marker = *input.ContinuationToken + } + delim := "" + if input.Delimiter != nil { + delim = *input.Delimiter + } + maxkeys := int32(0) + if input.MaxKeys != nil { + maxkeys = *input.MaxKeys + } _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -1491,23 +1667,25 @@ func (p *Posix) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) ( } fileSystem := os.DirFS(bucket) - results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys), + results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys, fileToObj(bucket), []string{metaTmpDir}) if err != nil { return nil, fmt.Errorf("walk %v: %w", bucket, err) } + count := int32(len(results.Objects)) + return &s3.ListObjectsV2Output{ CommonPrefixes: results.CommonPrefixes, Contents: results.Objects, Delimiter: &delim, - IsTruncated: results.Truncated, + IsTruncated: &results.Truncated, ContinuationToken: &marker, - MaxKeys: int32(maxkeys), + MaxKeys: &maxkeys, Name: &bucket, NextContinuationToken: &results.NextMarker, Prefix: &prefix, - KeyCount: int32(len(results.Objects)), + KeyCount: &count, }, nil } @@ -1528,6 +1706,9 @@ func (p *Posix) PutBucketAcl(_ context.Context, bucket string, data []byte) erro } func (p *Posix) GetBucketAcl(_ context.Context, input *s3.GetBucketAclInput) ([]byte, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } _, err := os.Stat(*input.Bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) diff --git a/backend/s3proxy/s3.go b/backend/s3proxy/s3.go index 19e4375..b02ddbe 100644 --- a/backend/s3proxy/s3.go +++ b/backend/s3proxy/s3.go @@ -184,8 +184,8 @@ func (s *S3be) ListMultipartUploads(ctx context.Context, input *s3.ListMultipart Delimiter: *output.Delimiter, Prefix: *output.Prefix, EncodingType: string(output.EncodingType), - MaxUploads: int(output.MaxUploads), - IsTruncated: output.IsTruncated, + MaxUploads: int(*output.MaxUploads), + IsTruncated: *output.IsTruncated, Uploads: uploads, CommonPrefixes: cps, }, nil @@ -205,10 +205,10 @@ func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3respo var parts []s3response.Part for _, p := range output.Parts { parts = append(parts, s3response.Part{ - PartNumber: int(p.PartNumber), + PartNumber: int(*p.PartNumber), LastModified: p.LastModified.Format(iso8601Format), ETag: *p.ETag, - Size: p.Size, + Size: *p.Size, }) } pnm, err := strconv.Atoi(*output.PartNumberMarker) @@ -238,8 +238,8 @@ func (s *S3be) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3respo StorageClass: string(output.StorageClass), PartNumberMarker: pnm, NextPartNumberMarker: npmn, - MaxParts: int(output.MaxParts), - IsTruncated: output.IsTruncated, + MaxParts: int(*output.MaxParts), + IsTruncated: *output.IsTruncated, Parts: parts, }, nil } diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index 47f30de..35a6ecb 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -415,8 +415,10 @@ func (s *ScoutFS) HeadObject(_ context.Context, input *s3.HeadObjectInput) (*s3. } } + contentLength := fi.Size() + return &s3.HeadObjectOutput{ - ContentLength: fi.Size(), + ContentLength: &contentLength, ContentType: &contentType, ContentEncoding: &contentEncoding, ETag: &etag, @@ -507,15 +509,17 @@ func (s *ScoutFS) GetObject(_ context.Context, input *s3.GetObjectInput, writer return nil, fmt.Errorf("get object tags: %w", err) } + tagCount := int32(len(tags)) + return &s3.GetObjectOutput{ AcceptRanges: &acceptRange, - ContentLength: length, + ContentLength: &length, ContentEncoding: &contentEncoding, ContentType: &contentType, ETag: &etag, LastModified: backend.GetTimePtr(fi.ModTime()), Metadata: userMetaData, - TagCount: int32(len(tags)), + TagCount: &tagCount, StorageClass: types.StorageClassStandard, }, nil } @@ -542,11 +546,26 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error) } func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } bucket := *input.Bucket - prefix := *input.Prefix - marker := *input.Marker - delim := *input.Delimiter - maxkeys := input.MaxKeys + prefix := "" + if input.Prefix != nil { + prefix = *input.Prefix + } + marker := "" + if input.Marker != nil { + marker = *input.Marker + } + delim := "" + if input.Delimiter != nil { + delim = *input.Delimiter + } + maxkeys := int32(0) + if input.MaxKeys != nil { + maxkeys = *input.MaxKeys + } _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -567,9 +586,9 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s CommonPrefixes: results.CommonPrefixes, Contents: results.Objects, Delimiter: &delim, - IsTruncated: results.Truncated, + IsTruncated: &results.Truncated, Marker: &marker, - MaxKeys: maxkeys, + MaxKeys: &maxkeys, Name: &bucket, NextMarker: &results.NextMarker, Prefix: &prefix, @@ -577,11 +596,26 @@ func (s *ScoutFS) ListObjects(_ context.Context, input *s3.ListObjectsInput) (*s } func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + if input.Bucket == nil { + return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } bucket := *input.Bucket - prefix := *input.Prefix - marker := *input.ContinuationToken - delim := *input.Delimiter - maxkeys := input.MaxKeys + prefix := "" + if input.Prefix != nil { + prefix = *input.Prefix + } + marker := "" + if input.ContinuationToken != nil { + marker = *input.ContinuationToken + } + delim := "" + if input.Delimiter != nil { + delim = *input.Delimiter + } + maxkeys := int32(0) + if input.MaxKeys != nil { + maxkeys = *input.MaxKeys + } _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -602,9 +636,9 @@ func (s *ScoutFS) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) CommonPrefixes: results.CommonPrefixes, Contents: results.Objects, Delimiter: &delim, - IsTruncated: results.Truncated, + IsTruncated: &results.Truncated, ContinuationToken: &marker, - MaxKeys: int32(maxkeys), + MaxKeys: &maxkeys, Name: &bucket, NextContinuationToken: &results.NextMarker, Prefix: &prefix, @@ -677,11 +711,13 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc { } } + size := fi.Size() + return types.Object{ ETag: &etag, Key: &path, LastModified: backend.GetTimePtr(fi.ModTime()), - Size: fi.Size(), + Size: &size, StorageClass: sc, }, nil } diff --git a/backend/walk_test.go b/backend/walk_test.go index a7b3101..a7b4b82 100644 --- a/backend/walk_test.go +++ b/backend/walk_test.go @@ -55,11 +55,13 @@ func getObj(path string, d fs.DirEntry) (types.Object, error) { return types.Object{}, fmt.Errorf("get fileinfo: %w", err) } + size := fi.Size() + return types.Object{ ETag: &etag, Key: &path, LastModified: backend.GetTimePtr(fi.ModTime()), - Size: fi.Size(), + Size: &size, }, nil } diff --git a/integration/tests.go b/integration/tests.go index 7aa4d52..043782a 100644 --- a/integration/tests.go +++ b/integration/tests.go @@ -1123,8 +1123,12 @@ func HeadObject_success(s *S3Conf) { if !areMapsSame(out.Metadata, meta) { return fmt.Errorf("incorrect object metadata") } - if out.ContentLength != dataLen { - return fmt.Errorf("expected data length %v, instead got %v", dataLen, out.ContentLength) + contentLength := int64(0) + if out.ContentLength != nil { + contentLength = *out.ContentLength + } + if contentLength != dataLen { + return fmt.Errorf("expected data length %v, instead got %v", dataLen, contentLength) } return nil @@ -1251,7 +1255,7 @@ func GetObject_success(s *S3Conf) { if err != nil { return err } - if out.ContentLength != dataLength { + if *out.ContentLength != dataLength { return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) } @@ -1396,18 +1400,18 @@ func ListObject_truncated(s *S3Conf) { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out1, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ Bucket: &bucket, - MaxKeys: maxKeys, + MaxKeys: &maxKeys, }) cancel() if err != nil { return err } - if !out1.IsTruncated { + if !*out1.IsTruncated { return fmt.Errorf("expected out1put to be truncated") } - if out1.MaxKeys != maxKeys { + if *out1.MaxKeys != maxKeys { return fmt.Errorf("expected max-keys to be %v, instead got %v", maxKeys, out1.MaxKeys) } @@ -1429,7 +1433,7 @@ func ListObject_truncated(s *S3Conf) { return err } - if out2.IsTruncated { + if *out2.IsTruncated { return fmt.Errorf("expected output not to be truncated") } @@ -1446,11 +1450,12 @@ func ListObject_truncated(s *S3Conf) { func ListObjects_invalid_max_keys(s *S3Conf) { testName := "ListObjects_invalid_max_keys" + maxKeys := int32(-5) actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ Bucket: &bucket, - MaxKeys: -5, + MaxKeys: &maxKeys, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)); err != nil { @@ -1470,16 +1475,17 @@ func ListObjects_max_keys_0(s *S3Conf) { return err } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + maxKeys := int32(0) out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ Bucket: &bucket, - MaxKeys: 0, + MaxKeys: &maxKeys, }) cancel() if err != nil { return nil } - if !compareObjects(objects, out.Contents) { + if len(out.Contents) > 0 { return fmt.Errorf("unexpected output for list objects with max-keys 0") } @@ -1537,7 +1543,7 @@ func ListObjects_max_keys_none(s *S3Conf) { return err } - if out.MaxKeys != 1000 { + if *out.MaxKeys != 1000 { return fmt.Errorf("expected max-keys to be 1000, instead got %v", out.MaxKeys) } @@ -1944,7 +1950,7 @@ func CopyObject_success(s *S3Conf) { if err != nil { return err } - if out.ContentLength != dataLength { + if *out.ContentLength != dataLength { return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) } @@ -2252,11 +2258,13 @@ func UploadPart_non_existing_bucket(s *S3Conf) { testName := "UploadPart_non_existing_bucket" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { bucketName := getBucketName() + partNumber := int32(1) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucketName, - Key: getPtr("my-obj"), - UploadId: getPtr("uploadId"), + Bucket: &bucketName, + Key: getPtr("my-obj"), + UploadId: getPtr("uploadId"), + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { @@ -2269,13 +2277,14 @@ func UploadPart_non_existing_bucket(s *S3Conf) { func UploadPart_invalid_part_number(s *S3Conf) { testName := "UploadPart_invalid_part_number" + partNumber := int32(-10) actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: getPtr("my-obj"), UploadId: getPtr("uploadId"), - PartNumber: -10, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { @@ -2287,13 +2296,14 @@ func UploadPart_invalid_part_number(s *S3Conf) { func UploadPart_non_existing_mp_upload(s *S3Conf) { testName := "UploadPart_non_existing_mp_upload" + partNumber := int32(1) actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: getPtr("my-obj"), UploadId: getPtr("uploadId"), - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { @@ -2305,6 +2315,7 @@ func UploadPart_non_existing_mp_upload(s *S3Conf) { func UploadPart_non_existing_key(s *S3Conf) { testName := "UploadPart_non_existing_key" + partNumber := int32(1) actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" out, err := createMp(s3client, bucket, obj) @@ -2317,7 +2328,7 @@ func UploadPart_non_existing_key(s *S3Conf) { Bucket: &bucket, Key: getPtr("non-existing-object-key"), UploadId: out.UploadId, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { @@ -2329,6 +2340,7 @@ func UploadPart_non_existing_key(s *S3Conf) { func UploadPart_success(s *S3Conf) { testName := "UploadPart_success" + partNumber := int32(1) actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" out, err := createMp(s3client, bucket, obj) @@ -2340,7 +2352,7 @@ func UploadPart_success(s *S3Conf) { Bucket: &bucket, Key: &obj, UploadId: out.UploadId, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err != nil { @@ -2357,6 +2369,7 @@ func UploadPartCopy_non_existing_bucket(s *S3Conf) { testName := "UploadPartCopy_non_existing_bucket" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { bucketName := getBucketName() + partNumber := int32(1) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ @@ -2364,6 +2377,7 @@ func UploadPartCopy_non_existing_bucket(s *S3Conf) { CopySource: getPtr("Copy-Source"), UploadId: getPtr("uploadId"), Key: getPtr("my-obj"), + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { @@ -2392,12 +2406,13 @@ func UploadPartCopy_incorrect_uploadId(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), UploadId: getPtr("incorrect-upload-id"), Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { @@ -2432,12 +2447,13 @@ func UploadPartCopy_incorrect_object_key(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), UploadId: out.UploadId, Key: getPtr("non-existing-object-key"), - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { @@ -2457,12 +2473,13 @@ func UploadPartCopy_invalid_part_number(s *S3Conf) { testName := "UploadPartCopy_invalid_part_number" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(-10) _, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr("Copy-Source"), UploadId: getPtr("uploadId"), Key: getPtr("non-existing-object-key"), - PartNumber: -10, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { @@ -2484,12 +2501,13 @@ func UploadPartCopy_invalid_copy_source(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr("invalid-copy-source"), UploadId: out.UploadId, Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidCopySource)); err != nil { @@ -2511,12 +2529,13 @@ func UploadPartCopy_non_existing_source_bucket(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr("src/bucket/src/obj"), UploadId: out.UploadId, Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { @@ -2543,12 +2562,13 @@ func UploadPartCopy_non_existing_source_object_key(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/non/existing/obj/key"), UploadId: out.UploadId, Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { @@ -2587,12 +2607,13 @@ func UploadPartCopy_success(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), UploadId: out.UploadId, Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err != nil { @@ -2613,10 +2634,10 @@ func UploadPartCopy_success(s *S3Conf) { if len(res.Parts) != 1 { return fmt.Errorf("expected parts to be 1, instead got %v", len(res.Parts)) } - if res.Parts[0].PartNumber != 1 { + if *res.Parts[0].PartNumber != 1 { return fmt.Errorf("expected part-number to be 1, instead got %v", res.Parts[0].PartNumber) } - if res.Parts[0].Size != int64(objSize) { + if *res.Parts[0].Size != int64(objSize) { return fmt.Errorf("expected part size to be %v, instead got %v", objSize, res.Parts[0].Size) } if *res.Parts[0].ETag != *copyOut.CopyPartResult.ETag { @@ -2655,12 +2676,13 @@ func UploadPartCopy_by_range_invalid_range(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), UploadId: out.UploadId, Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, CopySourceRange: getPtr("invalid-range"), }) cancel() @@ -2700,13 +2722,14 @@ func UploadPartCopy_greater_range_than_obj_size(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), UploadId: out.UploadId, Key: &obj, CopySourceRange: getPtr(fmt.Sprintf("bytes=0-%v", srcObjSize+50)), // The specified range is greater than the actual object size - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { @@ -2745,13 +2768,14 @@ func UploadPartCopy_by_range_success(s *S3Conf) { } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), CopySourceRange: getPtr("bytes=100-200"), UploadId: out.UploadId, Key: &obj, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err != nil { @@ -2772,10 +2796,10 @@ func UploadPartCopy_by_range_success(s *S3Conf) { if len(res.Parts) != 1 { return fmt.Errorf("expected parts to be 1, instead got %v", len(res.Parts)) } - if res.Parts[0].PartNumber != 1 { + if *res.Parts[0].PartNumber != 1 { return fmt.Errorf("expected part-number to be 1, instead got %v", res.Parts[0].PartNumber) } - if res.Parts[0].Size != 101 { + if *res.Parts[0].Size != 101 { return fmt.Errorf("expected part size to be %v, instead got %v", 101, res.Parts[0].Size) } if *res.Parts[0].ETag != *copyOut.CopyPartResult.ETag { @@ -2904,11 +2928,12 @@ func ListMultipartUploads_empty_result(s *S3Conf) { func ListMultipartUploads_invalid_max_uploads(s *S3Conf) { testName := "ListMultipartUploads_invalid_max_uploads" + maxUploads := int32(-3) actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ Bucket: &bucket, - MaxUploads: -3, + MaxUploads: &maxUploads, }) cancel() if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)); err != nil { @@ -2931,18 +2956,19 @@ func ListMultipartUploads_max_uploads(s *S3Conf) { uploads = append(uploads, types.MultipartUpload{UploadId: out.UploadId, Key: out.Key}) } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + maxUploads := int32(2) out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ Bucket: &bucket, - MaxUploads: 2, + MaxUploads: &maxUploads, }) cancel() if err != nil { return err } - if !out.IsTruncated { + if !*out.IsTruncated { return fmt.Errorf("expected the output to be truncated") } - if out.MaxUploads != 2 { + if *out.MaxUploads != 2 { return fmt.Errorf("expected max-uploads to be 2, instead got %v", out.MaxUploads) } if ok := compareMultipartUploads(out.Uploads, uploads[:2]); !ok { @@ -3227,11 +3253,12 @@ func CompleteMultipartUpload_invalid_part_number(s *S3Conf) { return err } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: &obj, UploadId: out.UploadId, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err != nil { @@ -3239,6 +3266,7 @@ func CompleteMultipartUpload_invalid_part_number(s *S3Conf) { } ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + partNumber = int32(5) _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ Bucket: &bucket, Key: &obj, @@ -3247,7 +3275,7 @@ func CompleteMultipartUpload_invalid_part_number(s *S3Conf) { Parts: []types.CompletedPart{ { ETag: res.ETag, - PartNumber: 5, + PartNumber: &partNumber, }, }, }, @@ -3270,11 +3298,12 @@ func CompleteMultipartUpload_invalid_ETag(s *S3Conf) { return err } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + partNumber := int32(1) _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: &obj, UploadId: out.UploadId, - PartNumber: 1, + PartNumber: &partNumber, }) cancel() if err != nil { @@ -3290,7 +3319,7 @@ func CompleteMultipartUpload_invalid_ETag(s *S3Conf) { Parts: []types.CompletedPart{ { ETag: getPtr("invalidETag"), - PartNumber: 1, + PartNumber: &partNumber, }, }, }, @@ -3358,7 +3387,7 @@ func CompleteMultipartUpload_success(s *S3Conf) { if *resp.ETag != *res.ETag { return fmt.Errorf("expected the uploaded object etag to be %v, instead got %v", *res.ETag, *resp.ETag) } - if resp.ContentLength != int64(objSize) { + if *resp.ContentLength != int64(objSize) { return fmt.Errorf("expected the uploaded object size to be %v, instead got %v", objSize, resp.ContentLength) } diff --git a/integration/utils.go b/integration/utils.go index 402a249..2d4f281 100644 --- a/integration/utils.go +++ b/integration/utils.go @@ -81,7 +81,7 @@ func teardown(s *S3Conf, bucket string) error { } } - if out.IsTruncated { + if *out.IsTruncated { in.ContinuationToken = out.ContinuationToken } else { break @@ -293,7 +293,7 @@ func compareParts(parts1, parts2 []types.Part) bool { } for i, prt := range parts1 { - if prt.PartNumber != parts2[i].PartNumber { + if *prt.PartNumber != *parts2[i].PartNumber { return false } if *prt.ETag != *parts2[i].ETag { @@ -484,18 +484,22 @@ func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId s return parts, err } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + pn := int32(partNumber) out, err := client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: &key, UploadId: &uploadId, Body: bytes.NewReader(partBuffer), - PartNumber: int32(partNumber), + PartNumber: &pn, }) cancel() if err != nil { return parts, err } else { - parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)}) + parts = append(parts, types.Part{ + ETag: out.ETag, + PartNumber: &pn, + }) offset += partSize } } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 8df7730..02c1545 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -63,7 +63,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { key := ctx.Params("key") keyEnd := ctx.Params("*1") uploadId := ctx.Query("uploadId") - maxParts := ctx.QueryInt("max-parts", 0) + maxParts := int32(ctx.QueryInt("max-parts", 0)) partNumberMarker := ctx.Query("part-number-marker") acceptRange := ctx.Get("Range") acct := ctx.Locals("account").(auth.Account) @@ -111,7 +111,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { Key: &key, UploadId: &uploadId, PartNumberMarker: &partNumberMarker, - MaxParts: int32(maxParts), + MaxParts: &maxParts, }) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner}) } @@ -169,7 +169,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { utils.SetResponseHeaders(ctx, []utils.CustomHeader{ { Key: "Content-Length", - Value: fmt.Sprint(res.ContentLength), + Value: fmt.Sprint(getint64(res.ContentLength)), }, { Key: "Content-Type", @@ -199,11 +199,17 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { Key: "accept-ranges", Value: getstring(res.AcceptRanges), }, - { - Key: "x-amz-tagging-count", - Value: fmt.Sprint(res.TagCount), - }, }) + + if res.TagCount != nil { + utils.SetResponseHeaders(ctx, []utils.CustomHeader{ + { + Key: "x-amz-tagging-count", + Value: fmt.Sprint(*res.TagCount), + }, + }) + } + return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner}) } @@ -214,6 +220,13 @@ func getstring(s *string) string { return *s } +func getint64(i *int64) int64 { + if i == nil { + return 0 + } + return *i +} + func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { bucket := ctx.Params("bucket") prefix := ctx.Query("prefix") @@ -259,7 +272,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { Delimiter: &delimiter, Prefix: &prefix, UploadIdMarker: &uploadIdMarker, - MaxUploads: maxUploads, + MaxUploads: &maxUploads, KeyMarker: &keyMarker, }) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner}) @@ -282,7 +295,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { Prefix: &prefix, ContinuationToken: &cToken, Delimiter: &delimiter, - MaxKeys: maxkeys, + MaxKeys: &maxkeys, }) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner}) } @@ -305,7 +318,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { Prefix: &prefix, Marker: &marker, Delimiter: &delimiter, - MaxKeys: maxkeys, + MaxKeys: &maxkeys, }) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner}) } @@ -467,7 +480,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") && copySource != "" { - partNumber := ctx.QueryInt("partNumber", -1) + partNumber := int32(ctx.QueryInt("partNumber", -1)) if partNumber < 1 || partNumber > 10000 { return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPartCopy", BucketOwner: parsedAcl.Owner}) } @@ -476,7 +489,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { Bucket: &bucket, Key: &keyStart, CopySource: ©Source, - PartNumber: int32(partNumber), + PartNumber: &partNumber, UploadId: &uploadId, ExpectedBucketOwner: &bucketOwner, CopySourceRange: ©SrcRange, @@ -485,7 +498,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") { - partNumber := ctx.QueryInt("partNumber", -1) + partNumber := int32(ctx.QueryInt("partNumber", -1)) if partNumber < 1 || partNumber > 10000 { return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner}) } @@ -505,8 +518,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { Bucket: &bucket, Key: &keyStart, UploadId: &uploadId, - PartNumber: int32(partNumber), - ContentLength: contentLength, + PartNumber: &partNumber, + ContentLength: &contentLength, Body: body, }) ctx.Response().Header.Set("Etag", etag) @@ -641,7 +654,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { etag, err := c.be.PutObject(ctx.Context(), &s3.PutObjectInput{ Bucket: &bucket, Key: &keyStart, - ContentLength: contentLength, + ContentLength: &contentLength, Metadata: metadata, Body: bytes.NewReader(ctx.Request().Body()), Tagging: &tagging, @@ -805,7 +818,7 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { utils.SetResponseHeaders(ctx, []utils.CustomHeader{ { Key: "Content-Length", - Value: fmt.Sprint(res.ContentLength), + Value: fmt.Sprint(getint64(res.ContentLength)), }, { Key: "Content-Type", diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index 11e7f92..74d71c9 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -174,6 +174,7 @@ func TestS3ApiController_GetActions(t *testing.T) { now := time.Now() app := fiber.New() + contentLength := int64(1000) s3ApiController := S3ApiController{ be: &BackendMock{ GetBucketAclFunc: func(context.Context, *s3.GetBucketAclInput) ([]byte, error) { @@ -194,7 +195,7 @@ func TestS3ApiController_GetActions(t *testing.T) { ContentType: getPtr("application/xml"), ContentEncoding: getPtr("gzip"), ETag: getPtr("98sda7f97sa9df798sd79f8as9df"), - ContentLength: 1000, + ContentLength: &contentLength, LastModified: &now, StorageClass: "storage class", }, nil @@ -1198,6 +1199,7 @@ func TestS3ApiController_HeadObject(t *testing.T) { contentType := "application/xml" eTag := "Valid etag" lastModifie := time.Now() + contentLength := int64(64) s3ApiController := S3ApiController{ be: &BackendMock{ @@ -1207,7 +1209,7 @@ func TestS3ApiController_HeadObject(t *testing.T) { HeadObjectFunc: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { return &s3.HeadObjectOutput{ ContentEncoding: &contentEncoding, - ContentLength: 64, + ContentLength: &contentLength, ContentType: &contentType, LastModified: &lastModifie, ETag: &eTag,