From 2291c22eaaea4e1d435508e6080531e5e5066e91 Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Sat, 22 Jul 2023 22:23:58 -0700 Subject: [PATCH] fix: standardize Backend interface args for s3 types --- backend/backend.go | 79 ++- backend/common.go | 2 +- backend/posix/posix.go | 113 +++- backend/scoutfs/scoutfs.go | 43 +- backend/walk.go | 8 +- s3api/controllers/backend_moq_test.go | 746 ++++++++++---------------- s3api/controllers/base.go | 180 +++++-- s3api/controllers/base_test.go | 76 +-- 8 files changed, 618 insertions(+), 629 deletions(-) diff --git a/backend/backend.go b/backend/backend.go index 71e9654..f605882 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -19,7 +19,6 @@ import ( "io" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" ) @@ -29,33 +28,33 @@ type Backend interface { fmt.Stringer Shutdown() - ListBuckets() (s3response.ListAllMyBucketsResult, error) - HeadBucket(bucket string) (*s3.HeadBucketOutput, error) - GetBucketAcl(bucket string) ([]byte, error) - PutBucket(bucket, owner string) error + ListBuckets(owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error) + HeadBucket(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) + GetBucketAcl(*s3.GetBucketAclInput) ([]byte, error) + CreateBucket(*s3.CreateBucketInput) error PutBucketAcl(bucket string, data []byte) error - DeleteBucket(bucket string) error + DeleteBucket(*s3.DeleteBucketInput) error CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) - CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) + CompleteMultipartUpload(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) AbortMultipartUpload(*s3.AbortMultipartUploadInput) error ListMultipartUploads(*s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) - ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) - PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error) + ListParts(*s3.ListPartsInput) (s3response.ListPartsResponse, error) + UploadPart(*s3.UploadPartInput) (etag string, err error) UploadPartCopy(*s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) PutObject(*s3.PutObjectInput) (string, error) - HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) - GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) - GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error) - GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) - CopyObject(srcBucket, srcObject, dstBucket, dstObject string) (*s3.CopyObjectOutput, error) - ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) - ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) - DeleteObject(bucket, object string) error - DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error + HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) + GetObject(*s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) + GetObjectAcl(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) + GetObjectAttributes(*s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) + CopyObject(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error) + ListObjects(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) + ListObjectsV2(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) + DeleteObject(*s3.DeleteObjectInput) error + DeleteObjects(*s3.DeleteObjectsInput) error PutObjectAcl(*s3.PutObjectAclInput) error - RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error + RestoreObject(*s3.RestoreObjectInput) error GetTags(bucket, object string) (map[string]string, error) SetTags(bucket, object string, tags map[string]string) error @@ -73,7 +72,7 @@ func (BackendUnsupported) Shutdown() {} func (BackendUnsupported) String() string { return "Unsupported" } -func (BackendUnsupported) ListBuckets() (s3response.ListAllMyBucketsResult, error) { +func (BackendUnsupported) ListBuckets(string, bool) (s3response.ListAllMyBucketsResult, error) { return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) PutBucketAcl(bucket string, data []byte) error { @@ -82,72 +81,72 @@ func (BackendUnsupported) PutBucketAcl(bucket string, data []byte) error { func (BackendUnsupported) PutObjectAcl(*s3.PutObjectAclInput) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error { +func (BackendUnsupported) RestoreObject(*s3.RestoreObjectInput) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) UploadPartCopy(*s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { return s3response.CopyObjectResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) GetBucketAcl(bucket string) ([]byte, error) { +func (BackendUnsupported) GetBucketAcl(*s3.GetBucketAclInput) ([]byte, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) { +func (BackendUnsupported) HeadBucket(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) PutBucket(bucket, owner string) error { +func (BackendUnsupported) CreateBucket(*s3.CreateBucketInput) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) DeleteBucket(bucket string) error { +func (BackendUnsupported) DeleteBucket(*s3.DeleteBucketInput) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) CreateMultipartUpload(input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) { +func (BackendUnsupported) CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) { +func (BackendUnsupported) CompleteMultipartUpload(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) AbortMultipartUpload(input *s3.AbortMultipartUploadInput) error { +func (BackendUnsupported) AbortMultipartUpload(*s3.AbortMultipartUploadInput) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) ListMultipartUploads(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { +func (BackendUnsupported) ListMultipartUploads(*s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { return s3response.ListMultipartUploadsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { +func (BackendUnsupported) ListParts(*s3.ListPartsInput) (s3response.ListPartsResponse, error) { return s3response.ListPartsResponse{}, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (etag string, err error) { +func (BackendUnsupported) UploadPart(*s3.UploadPartInput) (etag string, err error) { return "", s3err.GetAPIError(s3err.ErrNotImplemented) } func (BackendUnsupported) PutObject(*s3.PutObjectInput) (string, error) { return "", s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) DeleteObject(bucket, object string) error { +func (BackendUnsupported) DeleteObject(*s3.DeleteObjectInput) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error { +func (BackendUnsupported) DeleteObjects(*s3.DeleteObjectsInput) error { return s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) { +func (BackendUnsupported) GetObject(*s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) { +func (BackendUnsupported) HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) GetObjectAcl(bucket, object string) (*s3.GetObjectAclOutput, error) { +func (BackendUnsupported) GetObjectAcl(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) GetObjectAttributes(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) { +func (BackendUnsupported) GetObjectAttributes(*s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) CopyObject(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) { +func (BackendUnsupported) CopyObject(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { +func (BackendUnsupported) ListObjects(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } -func (BackendUnsupported) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { +func (BackendUnsupported) ListObjectsV2(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) } diff --git a/backend/common.go b/backend/common.go index 794dece..03cc34d 100644 --- a/backend/common.go +++ b/backend/common.go @@ -99,7 +99,7 @@ func ParseRange(file fs.FileInfo, acceptRange string) (int64, int64, error) { return startOffset, endOffset - startOffset + 1, nil } -func GetMultipartMD5(parts []types.Part) string { +func GetMultipartMD5(parts []types.CompletedPart) string { var partsEtagBytes []byte for _, part := range parts { partsEtagBytes = append(partsEtagBytes, getEtagBytes(*part.ETag)...) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index c8ebbc1..8d49fc7 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -96,7 +96,7 @@ func (p *Posix) String() string { return "Posix Gateway" } -func (p *Posix) ListBuckets() (s3response.ListAllMyBucketsResult, error) { +func (p *Posix) ListBuckets(owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error) { entries, err := os.ReadDir(".") if err != nil { return s3response.ListAllMyBucketsResult{}, @@ -131,8 +131,8 @@ func (p *Posix) ListBuckets() (s3response.ListAllMyBucketsResult, error) { }, nil } -func (p *Posix) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) { - _, err := os.Lstat(bucket) +func (p *Posix) HeadBucket(input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { + _, err := os.Lstat(*input.Bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } @@ -143,7 +143,10 @@ func (p *Posix) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) { return &s3.HeadBucketOutput{}, nil } -func (p *Posix) PutBucket(bucket string, owner string) error { +func (p *Posix) CreateBucket(input *s3.CreateBucketInput) error { + bucket := *input.Bucket + owner := string(input.ObjectOwnership) + err := os.Mkdir(bucket, 0777) if err != nil && os.IsExist(err) { return s3err.GetAPIError(s3err.ErrBucketAlreadyExists) @@ -165,8 +168,8 @@ func (p *Posix) PutBucket(bucket string, owner string) error { return nil } -func (p *Posix) DeleteBucket(bucket string) error { - names, err := os.ReadDir(bucket) +func (p *Posix) DeleteBucket(input *s3.DeleteBucketInput) error { + names, err := os.ReadDir(*input.Bucket) if errors.Is(err, fs.ErrNotExist) { return s3err.GetAPIError(s3err.ErrNoSuchBucket) } @@ -177,13 +180,13 @@ func (p *Posix) DeleteBucket(bucket string) error { if len(names) == 1 && names[0].Name() == metaTmpDir { // if .sgwtmp is only item in directory // then clean this up before trying to remove the bucket - err = os.RemoveAll(filepath.Join(bucket, metaTmpDir)) + err = os.RemoveAll(filepath.Join(*input.Bucket, metaTmpDir)) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("remove temp dir: %w", err) } } - err = os.Remove(bucket) + err = os.Remove(*input.Bucket) if err != nil && err.(*os.PathError).Err == syscall.ENOTEMPTY { return s3err.GetAPIError(s3err.ErrBucketNotEmpty) } @@ -245,7 +248,12 @@ func (p *Posix) CreateMultipartUpload(mpu *s3.CreateMultipartUploadInput) (*s3.C }, nil } -func (p *Posix) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) { +func (p *Posix) CompleteMultipartUpload(input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { + bucket := *input.Bucket + object := *input.Key + uploadID := *input.UploadId + parts := input.MultipartUpload.Parts + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -611,8 +619,24 @@ func (p *Posix) ListMultipartUploads(mpu *s3.ListMultipartUploadsInput) (s3respo }, nil } -func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { +func (p *Posix) ListParts(input *s3.ListPartsInput) (s3response.ListPartsResponse, error) { + bucket := *input.Bucket + object := *input.Key + uploadID := *input.UploadId + stringMarker := *input.PartNumberMarker + maxParts := int(input.MaxParts) + var lpr s3response.ListPartsResponse + + var partNumberMarker int + if stringMarker != "" { + var err error + partNumberMarker, err = strconv.Atoi(stringMarker) + if err != nil { + return lpr, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker) + } + } + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return lpr, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -693,7 +717,14 @@ func (p *Posix) ListObjectParts(bucket, object, uploadID string, partNumberMarke }, nil } -func (p *Posix) PutObjectPart(bucket, object, uploadID string, part int, length int64, r io.Reader) (string, error) { +func (p *Posix) UploadPart(input *s3.UploadPartInput) (string, error) { + bucket := *input.Bucket + object := *input.Key + uploadID := *input.UploadId + part := input.PartNumber + length := input.ContentLength + r := input.Body + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return "", s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -912,7 +943,10 @@ func (p *Posix) PutObject(po *s3.PutObjectInput) (string, error) { return etag, nil } -func (p *Posix) DeleteObject(bucket, object string) error { +func (p *Posix) DeleteObject(input *s3.DeleteObjectInput) error { + bucket := *input.Bucket + object := *input.Key + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -966,10 +1000,13 @@ func (p *Posix) removeParents(bucket, object string) error { return nil } -func (p *Posix) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error { +func (p *Posix) DeleteObjects(input *s3.DeleteObjectsInput) error { // delete object already checks bucket - for _, obj := range objects.Delete.Objects { - err := p.DeleteObject(bucket, *obj.Key) + for _, obj := range input.Delete.Objects { + err := p.DeleteObject(&s3.DeleteObjectInput{ + Bucket: input.Bucket, + Key: obj.Key, + }) if err != nil { return err } @@ -978,7 +1015,11 @@ func (p *Posix) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) err return nil } -func (p *Posix) GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) { +func (p *Posix) GetObject(input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) { + bucket := *input.Bucket + object := *input.Key + acceptRange := *input.Range + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -1051,7 +1092,10 @@ func (p *Posix) GetObject(bucket, object, acceptRange string, writer io.Writer) }, nil } -func (p *Posix) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) { +func (p *Posix) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { + bucket := *input.Bucket + object := *input.Key + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -1088,7 +1132,14 @@ func (p *Posix) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) }, nil } -func (p *Posix) CopyObject(srcBucket, srcObject, dstBucket, dstObject string) (*s3.CopyObjectOutput, error) { +func (p *Posix) CopyObject(input *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) { + srcBucket, srcObject, ok := strings.Cut(*input.CopySource, "/") + if !ok { + return nil, s3err.GetAPIError(s3err.ErrInvalidCopySource) + } + dstBucket := *input.Bucket + dstObject := *input.Key + _, err := os.Stat(srcBucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -1138,7 +1189,13 @@ func (p *Posix) CopyObject(srcBucket, srcObject, dstBucket, dstObject string) (* }, nil } -func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { +func (p *Posix) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { + bucket := *input.Bucket + prefix := *input.Prefix + marker := *input.Marker + delim := *input.Delimiter + maxkeys := input.MaxKeys + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -1160,7 +1217,7 @@ func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) ( Delimiter: &delim, IsTruncated: results.Truncated, Marker: &marker, - MaxKeys: int32(maxkeys), + MaxKeys: maxkeys, Name: &bucket, NextMarker: &results.NextMarker, Prefix: &prefix, @@ -1228,7 +1285,13 @@ func fileToObj(bucket string) backend.GetObjFunc { } } -func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { +func (p *Posix) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + bucket := *input.Bucket + prefix := *input.Prefix + marker := *input.ContinuationToken + delim := *input.Delimiter + maxkeys := input.MaxKeys + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -1238,7 +1301,7 @@ func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) } fileSystem := os.DirFS(bucket) - results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys, + results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys), fileToObj(bucket), []string{metaTmpDir}) if err != nil { return nil, fmt.Errorf("walk %v: %w", bucket, err) @@ -1273,8 +1336,8 @@ func (p *Posix) PutBucketAcl(bucket string, data []byte) error { return nil } -func (p *Posix) GetBucketAcl(bucket string) ([]byte, error) { - _, err := os.Stat(bucket) +func (p *Posix) GetBucketAcl(input *s3.GetBucketAclInput) ([]byte, error) { + _, err := os.Stat(*input.Bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } @@ -1282,7 +1345,7 @@ func (p *Posix) GetBucketAcl(bucket string) ([]byte, error) { return nil, fmt.Errorf("stat bucket: %w", err) } - b, err := xattr.Get(bucket, aclkey) + b, err := xattr.Get(*input.Bucket, aclkey) if isNoAttr(err) { return []byte{}, nil } diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index b6577c0..1dae13d 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -114,7 +114,12 @@ func (*ScoutFS) String() string { // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks // ioctl to not have to read and copy the part data to the final object. This // saves a read and write cycle for all mutlipart uploads. -func (s *ScoutFS) CompleteMultipartUpload(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) { +func (s *ScoutFS) CompleteMultipartUpload(input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { + bucket := *input.Bucket + object := *input.Key + uploadID := *input.UploadId + parts := input.MultipartUpload.Parts + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -347,7 +352,10 @@ func mkdirAll(path string, perm os.FileMode, bucket, object string) error { return nil } -func (s *ScoutFS) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error) { +func (s *ScoutFS) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { + bucket := *input.Bucket + object := *input.Key + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -417,7 +425,11 @@ func (s *ScoutFS) HeadObject(bucket, object string) (*s3.HeadObjectOutput, error }, nil } -func (s *ScoutFS) GetObject(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) { +func (s *ScoutFS) GetObject(input *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) { + bucket := *input.Bucket + object := *input.Key + acceptRange := *input.Range + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -527,7 +539,13 @@ func (s *ScoutFS) getXattrTags(bucket, object string) (map[string]string, error) return tags, nil } -func (s *ScoutFS) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { +func (s *ScoutFS) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { + bucket := *input.Bucket + prefix := *input.Prefix + marker := *input.Marker + delim := *input.Delimiter + maxkeys := input.MaxKeys + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -549,14 +567,20 @@ func (s *ScoutFS) ListObjects(bucket, prefix, marker, delim string, maxkeys int) Delimiter: &delim, IsTruncated: results.Truncated, Marker: &marker, - MaxKeys: int32(maxkeys), + MaxKeys: maxkeys, Name: &bucket, NextMarker: &results.NextMarker, Prefix: &prefix, }, nil } -func (s *ScoutFS) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { +func (s *ScoutFS) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + bucket := *input.Bucket + prefix := *input.Prefix + marker := *input.ContinuationToken + delim := *input.Delimiter + maxkeys := input.MaxKeys + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -566,7 +590,7 @@ func (s *ScoutFS) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys in } fileSystem := os.DirFS(bucket) - results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys, + results, err := backend.Walk(fileSystem, prefix, delim, marker, int32(maxkeys), s.fileToObj(bucket), []string{metaTmpDir}) if err != nil { return nil, fmt.Errorf("walk %v: %w", bucket, err) @@ -663,7 +687,10 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc { // RestoreObject will set stage request on file if offline and do nothing if // file is online -func (s *ScoutFS) RestoreObject(bucket, object string, restoreRequest *s3.RestoreObjectInput) error { +func (s *ScoutFS) RestoreObject(input *s3.RestoreObjectInput) error { + bucket := *input.Bucket + object := *input.Key + _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { return s3err.GetAPIError(s3err.ErrNoSuchBucket) diff --git a/backend/walk.go b/backend/walk.go index 39bf800..7d1e6b8 100644 --- a/backend/walk.go +++ b/backend/walk.go @@ -38,7 +38,7 @@ var ErrSkipObj = errors.New("skip this object") // Walk walks the supplied fs.FS and returns results compatible with list // objects responses -func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj GetObjFunc, skipdirs []string) (WalkResults, error) { +func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) { cpmap := make(map[string]struct{}) var objects []types.Object @@ -129,7 +129,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj Ge } objects = append(objects, obj) - if max > 0 && (len(objects)+len(cpmap)) == max { + if max > 0 && (len(objects)+len(cpmap)) == int(max) { pastMax = true } @@ -168,7 +168,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj Ge return fmt.Errorf("file to object %q: %w", path, err) } objects = append(objects, obj) - if (len(objects) + len(cpmap)) == max { + if (len(objects) + len(cpmap)) == int(max) { pastMax = true } return nil @@ -178,7 +178,7 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj Ge // These are abstractly a "directory", so need to include the // delimiter at the end. cpmap[prefix+before+delimiter] = struct{}{} - if (len(objects) + len(cpmap)) == max { + if (len(objects) + len(cpmap)) == int(max) { pastMax = true } diff --git a/s3api/controllers/backend_moq_test.go b/s3api/controllers/backend_moq_test.go index 03d6420..1280b90 100644 --- a/s3api/controllers/backend_moq_test.go +++ b/s3api/controllers/backend_moq_test.go @@ -5,7 +5,6 @@ package controllers import ( "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3response" "io" @@ -25,62 +24,62 @@ var _ backend.Backend = &BackendMock{} // AbortMultipartUploadFunc: func(abortMultipartUploadInput *s3.AbortMultipartUploadInput) error { // panic("mock out the AbortMultipartUpload method") // }, -// CompleteMultipartUploadFunc: func(bucket string, object string, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) { +// CompleteMultipartUploadFunc: func(completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { // panic("mock out the CompleteMultipartUpload method") // }, -// CopyObjectFunc: func(srcBucket string, srcObject string, dstBucket string, dstObject string) (*s3.CopyObjectOutput, error) { +// CopyObjectFunc: func(copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) { // panic("mock out the CopyObject method") // }, +// CreateBucketFunc: func(createBucketInput *s3.CreateBucketInput) error { +// panic("mock out the CreateBucket method") +// }, // CreateMultipartUploadFunc: func(createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) { // panic("mock out the CreateMultipartUpload method") // }, -// DeleteBucketFunc: func(bucket string) error { +// DeleteBucketFunc: func(deleteBucketInput *s3.DeleteBucketInput) error { // panic("mock out the DeleteBucket method") // }, -// DeleteObjectFunc: func(bucket string, object string) error { +// DeleteObjectFunc: func(deleteObjectInput *s3.DeleteObjectInput) error { // panic("mock out the DeleteObject method") // }, -// DeleteObjectsFunc: func(bucket string, objects *s3.DeleteObjectsInput) error { +// DeleteObjectsFunc: func(deleteObjectsInput *s3.DeleteObjectsInput) error { // panic("mock out the DeleteObjects method") // }, -// GetBucketAclFunc: func(bucket string) ([]byte, error) { +// GetBucketAclFunc: func(getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) { // panic("mock out the GetBucketAcl method") // }, -// GetObjectFunc: func(bucket string, object string, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) { +// GetObjectFunc: func(getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) { // panic("mock out the GetObject method") // }, -// GetObjectAclFunc: func(bucket string, object string) (*s3.GetObjectAclOutput, error) { +// GetObjectAclFunc: func(getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) { // panic("mock out the GetObjectAcl method") // }, -// GetObjectAttributesFunc: func(bucket string, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) { +// GetObjectAttributesFunc: func(getObjectAttributesInput *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) { // panic("mock out the GetObjectAttributes method") // }, // GetTagsFunc: func(bucket string, object string) (map[string]string, error) { // panic("mock out the GetTags method") // }, -// HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) { +// HeadBucketFunc: func(headBucketInput *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { // panic("mock out the HeadBucket method") // }, -// HeadObjectFunc: func(bucket string, object string) (*s3.HeadObjectOutput, error) { +// HeadObjectFunc: func(headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { // panic("mock out the HeadObject method") // }, -// ListBucketsFunc: func() (s3response.ListAllMyBucketsResult, error) { +// ListBucketsFunc: func(owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error) { // panic("mock out the ListBuckets method") // }, // ListMultipartUploadsFunc: func(listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { // panic("mock out the ListMultipartUploads method") // }, -// ListObjectPartsFunc: func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { -// panic("mock out the ListObjectParts method") -// }, -// ListObjectsFunc: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { +// ListObjectsFunc: func(listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { // panic("mock out the ListObjects method") // }, -// ListObjectsV2Func: func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { +// ListObjectsV2Func: func(listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { // panic("mock out the ListObjectsV2 method") // }, -// PutBucketFunc: func(bucket string, owner string) error { -// panic("mock out the PutBucket method") +// ListPartsFunc: func(listPartsInput *s3.ListPartsInput) (s3response.ListPartsResponse, error) { +// panic("mock out the ListParts method") // }, // PutBucketAclFunc: func(bucket string, data []byte) error { // panic("mock out the PutBucketAcl method") @@ -91,13 +90,10 @@ var _ backend.Backend = &BackendMock{} // PutObjectAclFunc: func(putObjectAclInput *s3.PutObjectAclInput) error { // panic("mock out the PutObjectAcl method") // }, -// PutObjectPartFunc: func(bucket string, object string, uploadID string, part int, length int64, r io.Reader) (string, error) { -// panic("mock out the PutObjectPart method") -// }, // RemoveTagsFunc: func(bucket string, object string) error { // panic("mock out the RemoveTags method") // }, -// RestoreObjectFunc: func(bucket string, object string, restoreRequest *s3.RestoreObjectInput) error { +// RestoreObjectFunc: func(restoreObjectInput *s3.RestoreObjectInput) error { // panic("mock out the RestoreObject method") // }, // SetTagsFunc: func(bucket string, object string, tags map[string]string) error { @@ -109,6 +105,9 @@ var _ backend.Backend = &BackendMock{} // StringFunc: func() string { // panic("mock out the String method") // }, +// UploadPartFunc: func(uploadPartInput *s3.UploadPartInput) (string, error) { +// panic("mock out the UploadPart method") +// }, // UploadPartCopyFunc: func(uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { // panic("mock out the UploadPartCopy method") // }, @@ -123,61 +122,61 @@ type BackendMock struct { AbortMultipartUploadFunc func(abortMultipartUploadInput *s3.AbortMultipartUploadInput) error // CompleteMultipartUploadFunc mocks the CompleteMultipartUpload method. - CompleteMultipartUploadFunc func(bucket string, object string, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) + CompleteMultipartUploadFunc func(completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) // CopyObjectFunc mocks the CopyObject method. - CopyObjectFunc func(srcBucket string, srcObject string, dstBucket string, dstObject string) (*s3.CopyObjectOutput, error) + CopyObjectFunc func(copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) + + // CreateBucketFunc mocks the CreateBucket method. + CreateBucketFunc func(createBucketInput *s3.CreateBucketInput) error // CreateMultipartUploadFunc mocks the CreateMultipartUpload method. CreateMultipartUploadFunc func(createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) // DeleteBucketFunc mocks the DeleteBucket method. - DeleteBucketFunc func(bucket string) error + DeleteBucketFunc func(deleteBucketInput *s3.DeleteBucketInput) error // DeleteObjectFunc mocks the DeleteObject method. - DeleteObjectFunc func(bucket string, object string) error + DeleteObjectFunc func(deleteObjectInput *s3.DeleteObjectInput) error // DeleteObjectsFunc mocks the DeleteObjects method. - DeleteObjectsFunc func(bucket string, objects *s3.DeleteObjectsInput) error + DeleteObjectsFunc func(deleteObjectsInput *s3.DeleteObjectsInput) error // GetBucketAclFunc mocks the GetBucketAcl method. - GetBucketAclFunc func(bucket string) ([]byte, error) + GetBucketAclFunc func(getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) // GetObjectFunc mocks the GetObject method. - GetObjectFunc func(bucket string, object string, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) + GetObjectFunc func(getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) // GetObjectAclFunc mocks the GetObjectAcl method. - GetObjectAclFunc func(bucket string, object string) (*s3.GetObjectAclOutput, error) + GetObjectAclFunc func(getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) // GetObjectAttributesFunc mocks the GetObjectAttributes method. - GetObjectAttributesFunc func(bucket string, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) + GetObjectAttributesFunc func(getObjectAttributesInput *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) // GetTagsFunc mocks the GetTags method. GetTagsFunc func(bucket string, object string) (map[string]string, error) // HeadBucketFunc mocks the HeadBucket method. - HeadBucketFunc func(bucket string) (*s3.HeadBucketOutput, error) + HeadBucketFunc func(headBucketInput *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) // HeadObjectFunc mocks the HeadObject method. - HeadObjectFunc func(bucket string, object string) (*s3.HeadObjectOutput, error) + HeadObjectFunc func(headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) // ListBucketsFunc mocks the ListBuckets method. - ListBucketsFunc func() (s3response.ListAllMyBucketsResult, error) + ListBucketsFunc func(owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error) // ListMultipartUploadsFunc mocks the ListMultipartUploads method. ListMultipartUploadsFunc func(listMultipartUploadsInput *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) - // ListObjectPartsFunc mocks the ListObjectParts method. - ListObjectPartsFunc func(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) - // ListObjectsFunc mocks the ListObjects method. - ListObjectsFunc func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) + ListObjectsFunc func(listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) // ListObjectsV2Func mocks the ListObjectsV2 method. - ListObjectsV2Func func(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) + ListObjectsV2Func func(listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) - // PutBucketFunc mocks the PutBucket method. - PutBucketFunc func(bucket string, owner string) error + // ListPartsFunc mocks the ListParts method. + ListPartsFunc func(listPartsInput *s3.ListPartsInput) (s3response.ListPartsResponse, error) // PutBucketAclFunc mocks the PutBucketAcl method. PutBucketAclFunc func(bucket string, data []byte) error @@ -188,14 +187,11 @@ type BackendMock struct { // PutObjectAclFunc mocks the PutObjectAcl method. PutObjectAclFunc func(putObjectAclInput *s3.PutObjectAclInput) error - // PutObjectPartFunc mocks the PutObjectPart method. - PutObjectPartFunc func(bucket string, object string, uploadID string, part int, length int64, r io.Reader) (string, error) - // RemoveTagsFunc mocks the RemoveTags method. RemoveTagsFunc func(bucket string, object string) error // RestoreObjectFunc mocks the RestoreObject method. - RestoreObjectFunc func(bucket string, object string, restoreRequest *s3.RestoreObjectInput) error + RestoreObjectFunc func(restoreObjectInput *s3.RestoreObjectInput) error // SetTagsFunc mocks the SetTags method. SetTagsFunc func(bucket string, object string, tags map[string]string) error @@ -206,6 +202,9 @@ type BackendMock struct { // StringFunc mocks the String method. StringFunc func() string + // UploadPartFunc mocks the UploadPart method. + UploadPartFunc func(uploadPartInput *s3.UploadPartInput) (string, error) + // UploadPartCopyFunc mocks the UploadPartCopy method. UploadPartCopyFunc func(uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) @@ -218,25 +217,18 @@ type BackendMock struct { } // CompleteMultipartUpload holds details about calls to the CompleteMultipartUpload method. CompleteMultipartUpload []struct { - // Bucket is the bucket argument value. - Bucket string - // Object is the object argument value. - Object string - // UploadID is the uploadID argument value. - UploadID string - // Parts is the parts argument value. - Parts []types.Part + // CompleteMultipartUploadInput is the completeMultipartUploadInput argument value. + CompleteMultipartUploadInput *s3.CompleteMultipartUploadInput } // CopyObject holds details about calls to the CopyObject method. CopyObject []struct { - // SrcBucket is the srcBucket argument value. - SrcBucket string - // SrcObject is the srcObject argument value. - SrcObject string - // DstBucket is the dstBucket argument value. - DstBucket string - // DstObject is the dstObject argument value. - DstObject string + // CopyObjectInput is the copyObjectInput argument value. + CopyObjectInput *s3.CopyObjectInput + } + // CreateBucket holds details about calls to the CreateBucket method. + CreateBucket []struct { + // CreateBucketInput is the createBucketInput argument value. + CreateBucketInput *s3.CreateBucketInput } // CreateMultipartUpload holds details about calls to the CreateMultipartUpload method. CreateMultipartUpload []struct { @@ -245,54 +237,40 @@ type BackendMock struct { } // DeleteBucket holds details about calls to the DeleteBucket method. DeleteBucket []struct { - // Bucket is the bucket argument value. - Bucket string + // DeleteBucketInput is the deleteBucketInput argument value. + DeleteBucketInput *s3.DeleteBucketInput } // DeleteObject holds details about calls to the DeleteObject method. DeleteObject []struct { - // Bucket is the bucket argument value. - Bucket string - // Object is the object argument value. - Object string + // DeleteObjectInput is the deleteObjectInput argument value. + DeleteObjectInput *s3.DeleteObjectInput } // DeleteObjects holds details about calls to the DeleteObjects method. DeleteObjects []struct { - // Bucket is the bucket argument value. - Bucket string - // Objects is the objects argument value. - Objects *s3.DeleteObjectsInput + // DeleteObjectsInput is the deleteObjectsInput argument value. + DeleteObjectsInput *s3.DeleteObjectsInput } // GetBucketAcl holds details about calls to the GetBucketAcl method. GetBucketAcl []struct { - // Bucket is the bucket argument value. - Bucket string + // GetBucketAclInput is the getBucketAclInput argument value. + GetBucketAclInput *s3.GetBucketAclInput } // GetObject holds details about calls to the GetObject method. GetObject []struct { - // Bucket is the bucket argument value. - Bucket string - // Object is the object argument value. - Object string - // AcceptRange is the acceptRange argument value. - AcceptRange string + // GetObjectInput is the getObjectInput argument value. + GetObjectInput *s3.GetObjectInput // Writer is the writer argument value. Writer io.Writer } // GetObjectAcl holds details about calls to the GetObjectAcl method. GetObjectAcl []struct { - // Bucket is the bucket argument value. - Bucket string - // Object is the object argument value. - Object string + // GetObjectAclInput is the getObjectAclInput argument value. + GetObjectAclInput *s3.GetObjectAclInput } // GetObjectAttributes holds details about calls to the GetObjectAttributes method. GetObjectAttributes []struct { - // Bucket is the bucket argument value. - Bucket string - // Object is the object argument value. - Object string - // Attributes is the attributes argument value. - Attributes []string + // GetObjectAttributesInput is the getObjectAttributesInput argument value. + GetObjectAttributesInput *s3.GetObjectAttributesInput } // GetTags holds details about calls to the GetTags method. GetTags []struct { @@ -303,69 +281,40 @@ type BackendMock struct { } // HeadBucket holds details about calls to the HeadBucket method. HeadBucket []struct { - // Bucket is the bucket argument value. - Bucket string + // HeadBucketInput is the headBucketInput argument value. + HeadBucketInput *s3.HeadBucketInput } // HeadObject holds details about calls to the HeadObject method. HeadObject []struct { - // Bucket is the bucket argument value. - Bucket string - // Object is the object argument value. - Object string + // HeadObjectInput is the headObjectInput argument value. + HeadObjectInput *s3.HeadObjectInput } // ListBuckets holds details about calls to the ListBuckets method. ListBuckets []struct { + // Owner is the owner argument value. + Owner string + // IsRoot is the isRoot argument value. + IsRoot bool } // ListMultipartUploads holds details about calls to the ListMultipartUploads method. ListMultipartUploads []struct { // ListMultipartUploadsInput is the listMultipartUploadsInput argument value. ListMultipartUploadsInput *s3.ListMultipartUploadsInput } - // ListObjectParts holds details about calls to the ListObjectParts method. - ListObjectParts []struct { - // Bucket is the bucket argument value. - Bucket string - // Object is the object argument value. - Object string - // UploadID is the uploadID argument value. - UploadID string - // PartNumberMarker is the partNumberMarker argument value. - PartNumberMarker int - // MaxParts is the maxParts argument value. - MaxParts int - } // ListObjects holds details about calls to the ListObjects method. ListObjects []struct { - // Bucket is the bucket argument value. - Bucket string - // Prefix is the prefix argument value. - Prefix string - // Marker is the marker argument value. - Marker string - // Delim is the delim argument value. - Delim string - // Maxkeys is the maxkeys argument value. - Maxkeys int + // ListObjectsInput is the listObjectsInput argument value. + ListObjectsInput *s3.ListObjectsInput } // ListObjectsV2 holds details about calls to the ListObjectsV2 method. ListObjectsV2 []struct { - // Bucket is the bucket argument value. - Bucket string - // Prefix is the prefix argument value. - Prefix string - // Marker is the marker argument value. - Marker string - // Delim is the delim argument value. - Delim string - // Maxkeys is the maxkeys argument value. - Maxkeys int + // ListObjectsV2Input is the listObjectsV2Input argument value. + ListObjectsV2Input *s3.ListObjectsV2Input } - // PutBucket holds details about calls to the PutBucket method. - PutBucket []struct { - // Bucket is the bucket argument value. - Bucket string - // Owner is the owner argument value. - Owner string + // ListParts holds details about calls to the ListParts method. + ListParts []struct { + // ListPartsInput is the listPartsInput argument value. + ListPartsInput *s3.ListPartsInput } // PutBucketAcl holds details about calls to the PutBucketAcl method. PutBucketAcl []struct { @@ -384,21 +333,6 @@ type BackendMock struct { // PutObjectAclInput is the putObjectAclInput argument value. PutObjectAclInput *s3.PutObjectAclInput } - // PutObjectPart holds details about calls to the PutObjectPart method. - PutObjectPart []struct { - // Bucket is the bucket argument value. - Bucket string - // Object is the object argument value. - Object string - // UploadID is the uploadID argument value. - UploadID string - // Part is the part argument value. - Part int - // Length is the length argument value. - Length int64 - // R is the r argument value. - R io.Reader - } // RemoveTags holds details about calls to the RemoveTags method. RemoveTags []struct { // Bucket is the bucket argument value. @@ -408,12 +342,8 @@ type BackendMock struct { } // RestoreObject holds details about calls to the RestoreObject method. RestoreObject []struct { - // Bucket is the bucket argument value. - Bucket string - // Object is the object argument value. - Object string - // RestoreRequest is the restoreRequest argument value. - RestoreRequest *s3.RestoreObjectInput + // RestoreObjectInput is the restoreObjectInput argument value. + RestoreObjectInput *s3.RestoreObjectInput } // SetTags holds details about calls to the SetTags method. SetTags []struct { @@ -430,6 +360,11 @@ type BackendMock struct { // String holds details about calls to the String method. String []struct { } + // UploadPart holds details about calls to the UploadPart method. + UploadPart []struct { + // UploadPartInput is the uploadPartInput argument value. + UploadPartInput *s3.UploadPartInput + } // UploadPartCopy holds details about calls to the UploadPartCopy method. UploadPartCopy []struct { // UploadPartCopyInput is the uploadPartCopyInput argument value. @@ -439,6 +374,7 @@ type BackendMock struct { lockAbortMultipartUpload sync.RWMutex lockCompleteMultipartUpload sync.RWMutex lockCopyObject sync.RWMutex + lockCreateBucket sync.RWMutex lockCreateMultipartUpload sync.RWMutex lockDeleteBucket sync.RWMutex lockDeleteObject sync.RWMutex @@ -452,19 +388,18 @@ type BackendMock struct { lockHeadObject sync.RWMutex lockListBuckets sync.RWMutex lockListMultipartUploads sync.RWMutex - lockListObjectParts sync.RWMutex lockListObjects sync.RWMutex lockListObjectsV2 sync.RWMutex - lockPutBucket sync.RWMutex + lockListParts sync.RWMutex lockPutBucketAcl sync.RWMutex lockPutObject sync.RWMutex lockPutObjectAcl sync.RWMutex - lockPutObjectPart sync.RWMutex lockRemoveTags sync.RWMutex lockRestoreObject sync.RWMutex lockSetTags sync.RWMutex lockShutdown sync.RWMutex lockString sync.RWMutex + lockUploadPart sync.RWMutex lockUploadPartCopy sync.RWMutex } @@ -501,25 +436,19 @@ func (mock *BackendMock) AbortMultipartUploadCalls() []struct { } // CompleteMultipartUpload calls CompleteMultipartUploadFunc. -func (mock *BackendMock) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) { +func (mock *BackendMock) CompleteMultipartUpload(completeMultipartUploadInput *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { if mock.CompleteMultipartUploadFunc == nil { panic("BackendMock.CompleteMultipartUploadFunc: method is nil but Backend.CompleteMultipartUpload was just called") } callInfo := struct { - Bucket string - Object string - UploadID string - Parts []types.Part + CompleteMultipartUploadInput *s3.CompleteMultipartUploadInput }{ - Bucket: bucket, - Object: object, - UploadID: uploadID, - Parts: parts, + CompleteMultipartUploadInput: completeMultipartUploadInput, } mock.lockCompleteMultipartUpload.Lock() mock.calls.CompleteMultipartUpload = append(mock.calls.CompleteMultipartUpload, callInfo) mock.lockCompleteMultipartUpload.Unlock() - return mock.CompleteMultipartUploadFunc(bucket, object, uploadID, parts) + return mock.CompleteMultipartUploadFunc(completeMultipartUploadInput) } // CompleteMultipartUploadCalls gets all the calls that were made to CompleteMultipartUpload. @@ -527,16 +456,10 @@ func (mock *BackendMock) CompleteMultipartUpload(bucket string, object string, u // // len(mockedBackend.CompleteMultipartUploadCalls()) func (mock *BackendMock) CompleteMultipartUploadCalls() []struct { - Bucket string - Object string - UploadID string - Parts []types.Part + CompleteMultipartUploadInput *s3.CompleteMultipartUploadInput } { var calls []struct { - Bucket string - Object string - UploadID string - Parts []types.Part + CompleteMultipartUploadInput *s3.CompleteMultipartUploadInput } mock.lockCompleteMultipartUpload.RLock() calls = mock.calls.CompleteMultipartUpload @@ -545,25 +468,19 @@ func (mock *BackendMock) CompleteMultipartUploadCalls() []struct { } // CopyObject calls CopyObjectFunc. -func (mock *BackendMock) CopyObject(srcBucket string, srcObject string, dstBucket string, dstObject string) (*s3.CopyObjectOutput, error) { +func (mock *BackendMock) CopyObject(copyObjectInput *s3.CopyObjectInput) (*s3.CopyObjectOutput, error) { if mock.CopyObjectFunc == nil { panic("BackendMock.CopyObjectFunc: method is nil but Backend.CopyObject was just called") } callInfo := struct { - SrcBucket string - SrcObject string - DstBucket string - DstObject string + CopyObjectInput *s3.CopyObjectInput }{ - SrcBucket: srcBucket, - SrcObject: srcObject, - DstBucket: dstBucket, - DstObject: dstObject, + CopyObjectInput: copyObjectInput, } mock.lockCopyObject.Lock() mock.calls.CopyObject = append(mock.calls.CopyObject, callInfo) mock.lockCopyObject.Unlock() - return mock.CopyObjectFunc(srcBucket, srcObject, dstBucket, dstObject) + return mock.CopyObjectFunc(copyObjectInput) } // CopyObjectCalls gets all the calls that were made to CopyObject. @@ -571,16 +488,10 @@ func (mock *BackendMock) CopyObject(srcBucket string, srcObject string, dstBucke // // len(mockedBackend.CopyObjectCalls()) func (mock *BackendMock) CopyObjectCalls() []struct { - SrcBucket string - SrcObject string - DstBucket string - DstObject string + CopyObjectInput *s3.CopyObjectInput } { var calls []struct { - SrcBucket string - SrcObject string - DstBucket string - DstObject string + CopyObjectInput *s3.CopyObjectInput } mock.lockCopyObject.RLock() calls = mock.calls.CopyObject @@ -588,6 +499,38 @@ func (mock *BackendMock) CopyObjectCalls() []struct { return calls } +// CreateBucket calls CreateBucketFunc. +func (mock *BackendMock) CreateBucket(createBucketInput *s3.CreateBucketInput) error { + if mock.CreateBucketFunc == nil { + panic("BackendMock.CreateBucketFunc: method is nil but Backend.CreateBucket was just called") + } + callInfo := struct { + CreateBucketInput *s3.CreateBucketInput + }{ + CreateBucketInput: createBucketInput, + } + mock.lockCreateBucket.Lock() + mock.calls.CreateBucket = append(mock.calls.CreateBucket, callInfo) + mock.lockCreateBucket.Unlock() + return mock.CreateBucketFunc(createBucketInput) +} + +// CreateBucketCalls gets all the calls that were made to CreateBucket. +// Check the length with: +// +// len(mockedBackend.CreateBucketCalls()) +func (mock *BackendMock) CreateBucketCalls() []struct { + CreateBucketInput *s3.CreateBucketInput +} { + var calls []struct { + CreateBucketInput *s3.CreateBucketInput + } + mock.lockCreateBucket.RLock() + calls = mock.calls.CreateBucket + mock.lockCreateBucket.RUnlock() + return calls +} + // CreateMultipartUpload calls CreateMultipartUploadFunc. func (mock *BackendMock) CreateMultipartUpload(createMultipartUploadInput *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) { if mock.CreateMultipartUploadFunc == nil { @@ -621,19 +564,19 @@ func (mock *BackendMock) CreateMultipartUploadCalls() []struct { } // DeleteBucket calls DeleteBucketFunc. -func (mock *BackendMock) DeleteBucket(bucket string) error { +func (mock *BackendMock) DeleteBucket(deleteBucketInput *s3.DeleteBucketInput) error { if mock.DeleteBucketFunc == nil { panic("BackendMock.DeleteBucketFunc: method is nil but Backend.DeleteBucket was just called") } callInfo := struct { - Bucket string + DeleteBucketInput *s3.DeleteBucketInput }{ - Bucket: bucket, + DeleteBucketInput: deleteBucketInput, } mock.lockDeleteBucket.Lock() mock.calls.DeleteBucket = append(mock.calls.DeleteBucket, callInfo) mock.lockDeleteBucket.Unlock() - return mock.DeleteBucketFunc(bucket) + return mock.DeleteBucketFunc(deleteBucketInput) } // DeleteBucketCalls gets all the calls that were made to DeleteBucket. @@ -641,10 +584,10 @@ func (mock *BackendMock) DeleteBucket(bucket string) error { // // len(mockedBackend.DeleteBucketCalls()) func (mock *BackendMock) DeleteBucketCalls() []struct { - Bucket string + DeleteBucketInput *s3.DeleteBucketInput } { var calls []struct { - Bucket string + DeleteBucketInput *s3.DeleteBucketInput } mock.lockDeleteBucket.RLock() calls = mock.calls.DeleteBucket @@ -653,21 +596,19 @@ func (mock *BackendMock) DeleteBucketCalls() []struct { } // DeleteObject calls DeleteObjectFunc. -func (mock *BackendMock) DeleteObject(bucket string, object string) error { +func (mock *BackendMock) DeleteObject(deleteObjectInput *s3.DeleteObjectInput) error { if mock.DeleteObjectFunc == nil { panic("BackendMock.DeleteObjectFunc: method is nil but Backend.DeleteObject was just called") } callInfo := struct { - Bucket string - Object string + DeleteObjectInput *s3.DeleteObjectInput }{ - Bucket: bucket, - Object: object, + DeleteObjectInput: deleteObjectInput, } mock.lockDeleteObject.Lock() mock.calls.DeleteObject = append(mock.calls.DeleteObject, callInfo) mock.lockDeleteObject.Unlock() - return mock.DeleteObjectFunc(bucket, object) + return mock.DeleteObjectFunc(deleteObjectInput) } // DeleteObjectCalls gets all the calls that were made to DeleteObject. @@ -675,12 +616,10 @@ func (mock *BackendMock) DeleteObject(bucket string, object string) error { // // len(mockedBackend.DeleteObjectCalls()) func (mock *BackendMock) DeleteObjectCalls() []struct { - Bucket string - Object string + DeleteObjectInput *s3.DeleteObjectInput } { var calls []struct { - Bucket string - Object string + DeleteObjectInput *s3.DeleteObjectInput } mock.lockDeleteObject.RLock() calls = mock.calls.DeleteObject @@ -689,21 +628,19 @@ func (mock *BackendMock) DeleteObjectCalls() []struct { } // DeleteObjects calls DeleteObjectsFunc. -func (mock *BackendMock) DeleteObjects(bucket string, objects *s3.DeleteObjectsInput) error { +func (mock *BackendMock) DeleteObjects(deleteObjectsInput *s3.DeleteObjectsInput) error { if mock.DeleteObjectsFunc == nil { panic("BackendMock.DeleteObjectsFunc: method is nil but Backend.DeleteObjects was just called") } callInfo := struct { - Bucket string - Objects *s3.DeleteObjectsInput + DeleteObjectsInput *s3.DeleteObjectsInput }{ - Bucket: bucket, - Objects: objects, + DeleteObjectsInput: deleteObjectsInput, } mock.lockDeleteObjects.Lock() mock.calls.DeleteObjects = append(mock.calls.DeleteObjects, callInfo) mock.lockDeleteObjects.Unlock() - return mock.DeleteObjectsFunc(bucket, objects) + return mock.DeleteObjectsFunc(deleteObjectsInput) } // DeleteObjectsCalls gets all the calls that were made to DeleteObjects. @@ -711,12 +648,10 @@ func (mock *BackendMock) DeleteObjects(bucket string, objects *s3.DeleteObjectsI // // len(mockedBackend.DeleteObjectsCalls()) func (mock *BackendMock) DeleteObjectsCalls() []struct { - Bucket string - Objects *s3.DeleteObjectsInput + DeleteObjectsInput *s3.DeleteObjectsInput } { var calls []struct { - Bucket string - Objects *s3.DeleteObjectsInput + DeleteObjectsInput *s3.DeleteObjectsInput } mock.lockDeleteObjects.RLock() calls = mock.calls.DeleteObjects @@ -725,19 +660,19 @@ func (mock *BackendMock) DeleteObjectsCalls() []struct { } // GetBucketAcl calls GetBucketAclFunc. -func (mock *BackendMock) GetBucketAcl(bucket string) ([]byte, error) { +func (mock *BackendMock) GetBucketAcl(getBucketAclInput *s3.GetBucketAclInput) ([]byte, error) { if mock.GetBucketAclFunc == nil { panic("BackendMock.GetBucketAclFunc: method is nil but Backend.GetBucketAcl was just called") } callInfo := struct { - Bucket string + GetBucketAclInput *s3.GetBucketAclInput }{ - Bucket: bucket, + GetBucketAclInput: getBucketAclInput, } mock.lockGetBucketAcl.Lock() mock.calls.GetBucketAcl = append(mock.calls.GetBucketAcl, callInfo) mock.lockGetBucketAcl.Unlock() - return mock.GetBucketAclFunc(bucket) + return mock.GetBucketAclFunc(getBucketAclInput) } // GetBucketAclCalls gets all the calls that were made to GetBucketAcl. @@ -745,10 +680,10 @@ func (mock *BackendMock) GetBucketAcl(bucket string) ([]byte, error) { // // len(mockedBackend.GetBucketAclCalls()) func (mock *BackendMock) GetBucketAclCalls() []struct { - Bucket string + GetBucketAclInput *s3.GetBucketAclInput } { var calls []struct { - Bucket string + GetBucketAclInput *s3.GetBucketAclInput } mock.lockGetBucketAcl.RLock() calls = mock.calls.GetBucketAcl @@ -757,25 +692,21 @@ func (mock *BackendMock) GetBucketAclCalls() []struct { } // GetObject calls GetObjectFunc. -func (mock *BackendMock) GetObject(bucket string, object string, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) { +func (mock *BackendMock) GetObject(getObjectInput *s3.GetObjectInput, writer io.Writer) (*s3.GetObjectOutput, error) { if mock.GetObjectFunc == nil { panic("BackendMock.GetObjectFunc: method is nil but Backend.GetObject was just called") } callInfo := struct { - Bucket string - Object string - AcceptRange string - Writer io.Writer + GetObjectInput *s3.GetObjectInput + Writer io.Writer }{ - Bucket: bucket, - Object: object, - AcceptRange: acceptRange, - Writer: writer, + GetObjectInput: getObjectInput, + Writer: writer, } mock.lockGetObject.Lock() mock.calls.GetObject = append(mock.calls.GetObject, callInfo) mock.lockGetObject.Unlock() - return mock.GetObjectFunc(bucket, object, acceptRange, writer) + return mock.GetObjectFunc(getObjectInput, writer) } // GetObjectCalls gets all the calls that were made to GetObject. @@ -783,16 +714,12 @@ func (mock *BackendMock) GetObject(bucket string, object string, acceptRange str // // len(mockedBackend.GetObjectCalls()) func (mock *BackendMock) GetObjectCalls() []struct { - Bucket string - Object string - AcceptRange string - Writer io.Writer + GetObjectInput *s3.GetObjectInput + Writer io.Writer } { var calls []struct { - Bucket string - Object string - AcceptRange string - Writer io.Writer + GetObjectInput *s3.GetObjectInput + Writer io.Writer } mock.lockGetObject.RLock() calls = mock.calls.GetObject @@ -801,21 +728,19 @@ func (mock *BackendMock) GetObjectCalls() []struct { } // GetObjectAcl calls GetObjectAclFunc. -func (mock *BackendMock) GetObjectAcl(bucket string, object string) (*s3.GetObjectAclOutput, error) { +func (mock *BackendMock) GetObjectAcl(getObjectAclInput *s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) { if mock.GetObjectAclFunc == nil { panic("BackendMock.GetObjectAclFunc: method is nil but Backend.GetObjectAcl was just called") } callInfo := struct { - Bucket string - Object string + GetObjectAclInput *s3.GetObjectAclInput }{ - Bucket: bucket, - Object: object, + GetObjectAclInput: getObjectAclInput, } mock.lockGetObjectAcl.Lock() mock.calls.GetObjectAcl = append(mock.calls.GetObjectAcl, callInfo) mock.lockGetObjectAcl.Unlock() - return mock.GetObjectAclFunc(bucket, object) + return mock.GetObjectAclFunc(getObjectAclInput) } // GetObjectAclCalls gets all the calls that were made to GetObjectAcl. @@ -823,12 +748,10 @@ func (mock *BackendMock) GetObjectAcl(bucket string, object string) (*s3.GetObje // // len(mockedBackend.GetObjectAclCalls()) func (mock *BackendMock) GetObjectAclCalls() []struct { - Bucket string - Object string + GetObjectAclInput *s3.GetObjectAclInput } { var calls []struct { - Bucket string - Object string + GetObjectAclInput *s3.GetObjectAclInput } mock.lockGetObjectAcl.RLock() calls = mock.calls.GetObjectAcl @@ -837,23 +760,19 @@ func (mock *BackendMock) GetObjectAclCalls() []struct { } // GetObjectAttributes calls GetObjectAttributesFunc. -func (mock *BackendMock) GetObjectAttributes(bucket string, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) { +func (mock *BackendMock) GetObjectAttributes(getObjectAttributesInput *s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) { if mock.GetObjectAttributesFunc == nil { panic("BackendMock.GetObjectAttributesFunc: method is nil but Backend.GetObjectAttributes was just called") } callInfo := struct { - Bucket string - Object string - Attributes []string + GetObjectAttributesInput *s3.GetObjectAttributesInput }{ - Bucket: bucket, - Object: object, - Attributes: attributes, + GetObjectAttributesInput: getObjectAttributesInput, } mock.lockGetObjectAttributes.Lock() mock.calls.GetObjectAttributes = append(mock.calls.GetObjectAttributes, callInfo) mock.lockGetObjectAttributes.Unlock() - return mock.GetObjectAttributesFunc(bucket, object, attributes) + return mock.GetObjectAttributesFunc(getObjectAttributesInput) } // GetObjectAttributesCalls gets all the calls that were made to GetObjectAttributes. @@ -861,14 +780,10 @@ func (mock *BackendMock) GetObjectAttributes(bucket string, object string, attri // // len(mockedBackend.GetObjectAttributesCalls()) func (mock *BackendMock) GetObjectAttributesCalls() []struct { - Bucket string - Object string - Attributes []string + GetObjectAttributesInput *s3.GetObjectAttributesInput } { var calls []struct { - Bucket string - Object string - Attributes []string + GetObjectAttributesInput *s3.GetObjectAttributesInput } mock.lockGetObjectAttributes.RLock() calls = mock.calls.GetObjectAttributes @@ -913,19 +828,19 @@ func (mock *BackendMock) GetTagsCalls() []struct { } // HeadBucket calls HeadBucketFunc. -func (mock *BackendMock) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) { +func (mock *BackendMock) HeadBucket(headBucketInput *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { if mock.HeadBucketFunc == nil { panic("BackendMock.HeadBucketFunc: method is nil but Backend.HeadBucket was just called") } callInfo := struct { - Bucket string + HeadBucketInput *s3.HeadBucketInput }{ - Bucket: bucket, + HeadBucketInput: headBucketInput, } mock.lockHeadBucket.Lock() mock.calls.HeadBucket = append(mock.calls.HeadBucket, callInfo) mock.lockHeadBucket.Unlock() - return mock.HeadBucketFunc(bucket) + return mock.HeadBucketFunc(headBucketInput) } // HeadBucketCalls gets all the calls that were made to HeadBucket. @@ -933,10 +848,10 @@ func (mock *BackendMock) HeadBucket(bucket string) (*s3.HeadBucketOutput, error) // // len(mockedBackend.HeadBucketCalls()) func (mock *BackendMock) HeadBucketCalls() []struct { - Bucket string + HeadBucketInput *s3.HeadBucketInput } { var calls []struct { - Bucket string + HeadBucketInput *s3.HeadBucketInput } mock.lockHeadBucket.RLock() calls = mock.calls.HeadBucket @@ -945,21 +860,19 @@ func (mock *BackendMock) HeadBucketCalls() []struct { } // HeadObject calls HeadObjectFunc. -func (mock *BackendMock) HeadObject(bucket string, object string) (*s3.HeadObjectOutput, error) { +func (mock *BackendMock) HeadObject(headObjectInput *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { if mock.HeadObjectFunc == nil { panic("BackendMock.HeadObjectFunc: method is nil but Backend.HeadObject was just called") } callInfo := struct { - Bucket string - Object string + HeadObjectInput *s3.HeadObjectInput }{ - Bucket: bucket, - Object: object, + HeadObjectInput: headObjectInput, } mock.lockHeadObject.Lock() mock.calls.HeadObject = append(mock.calls.HeadObject, callInfo) mock.lockHeadObject.Unlock() - return mock.HeadObjectFunc(bucket, object) + return mock.HeadObjectFunc(headObjectInput) } // HeadObjectCalls gets all the calls that were made to HeadObject. @@ -967,12 +880,10 @@ func (mock *BackendMock) HeadObject(bucket string, object string) (*s3.HeadObjec // // len(mockedBackend.HeadObjectCalls()) func (mock *BackendMock) HeadObjectCalls() []struct { - Bucket string - Object string + HeadObjectInput *s3.HeadObjectInput } { var calls []struct { - Bucket string - Object string + HeadObjectInput *s3.HeadObjectInput } mock.lockHeadObject.RLock() calls = mock.calls.HeadObject @@ -981,16 +892,21 @@ func (mock *BackendMock) HeadObjectCalls() []struct { } // ListBuckets calls ListBucketsFunc. -func (mock *BackendMock) ListBuckets() (s3response.ListAllMyBucketsResult, error) { +func (mock *BackendMock) ListBuckets(owner string, isRoot bool) (s3response.ListAllMyBucketsResult, error) { if mock.ListBucketsFunc == nil { panic("BackendMock.ListBucketsFunc: method is nil but Backend.ListBuckets was just called") } callInfo := struct { - }{} + Owner string + IsRoot bool + }{ + Owner: owner, + IsRoot: isRoot, + } mock.lockListBuckets.Lock() mock.calls.ListBuckets = append(mock.calls.ListBuckets, callInfo) mock.lockListBuckets.Unlock() - return mock.ListBucketsFunc() + return mock.ListBucketsFunc(owner, isRoot) } // ListBucketsCalls gets all the calls that were made to ListBuckets. @@ -998,8 +914,12 @@ func (mock *BackendMock) ListBuckets() (s3response.ListAllMyBucketsResult, error // // len(mockedBackend.ListBucketsCalls()) func (mock *BackendMock) ListBucketsCalls() []struct { + Owner string + IsRoot bool } { var calls []struct { + Owner string + IsRoot bool } mock.lockListBuckets.RLock() calls = mock.calls.ListBuckets @@ -1039,76 +959,20 @@ func (mock *BackendMock) ListMultipartUploadsCalls() []struct { return calls } -// ListObjectParts calls ListObjectPartsFunc. -func (mock *BackendMock) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { - if mock.ListObjectPartsFunc == nil { - panic("BackendMock.ListObjectPartsFunc: method is nil but Backend.ListObjectParts was just called") - } - callInfo := struct { - Bucket string - Object string - UploadID string - PartNumberMarker int - MaxParts int - }{ - Bucket: bucket, - Object: object, - UploadID: uploadID, - PartNumberMarker: partNumberMarker, - MaxParts: maxParts, - } - mock.lockListObjectParts.Lock() - mock.calls.ListObjectParts = append(mock.calls.ListObjectParts, callInfo) - mock.lockListObjectParts.Unlock() - return mock.ListObjectPartsFunc(bucket, object, uploadID, partNumberMarker, maxParts) -} - -// ListObjectPartsCalls gets all the calls that were made to ListObjectParts. -// Check the length with: -// -// len(mockedBackend.ListObjectPartsCalls()) -func (mock *BackendMock) ListObjectPartsCalls() []struct { - Bucket string - Object string - UploadID string - PartNumberMarker int - MaxParts int -} { - var calls []struct { - Bucket string - Object string - UploadID string - PartNumberMarker int - MaxParts int - } - mock.lockListObjectParts.RLock() - calls = mock.calls.ListObjectParts - mock.lockListObjectParts.RUnlock() - return calls -} - // ListObjects calls ListObjectsFunc. -func (mock *BackendMock) ListObjects(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { +func (mock *BackendMock) ListObjects(listObjectsInput *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { if mock.ListObjectsFunc == nil { panic("BackendMock.ListObjectsFunc: method is nil but Backend.ListObjects was just called") } callInfo := struct { - Bucket string - Prefix string - Marker string - Delim string - Maxkeys int + ListObjectsInput *s3.ListObjectsInput }{ - Bucket: bucket, - Prefix: prefix, - Marker: marker, - Delim: delim, - Maxkeys: maxkeys, + ListObjectsInput: listObjectsInput, } mock.lockListObjects.Lock() mock.calls.ListObjects = append(mock.calls.ListObjects, callInfo) mock.lockListObjects.Unlock() - return mock.ListObjectsFunc(bucket, prefix, marker, delim, maxkeys) + return mock.ListObjectsFunc(listObjectsInput) } // ListObjectsCalls gets all the calls that were made to ListObjects. @@ -1116,18 +980,10 @@ func (mock *BackendMock) ListObjects(bucket string, prefix string, marker string // // len(mockedBackend.ListObjectsCalls()) func (mock *BackendMock) ListObjectsCalls() []struct { - Bucket string - Prefix string - Marker string - Delim string - Maxkeys int + ListObjectsInput *s3.ListObjectsInput } { var calls []struct { - Bucket string - Prefix string - Marker string - Delim string - Maxkeys int + ListObjectsInput *s3.ListObjectsInput } mock.lockListObjects.RLock() calls = mock.calls.ListObjects @@ -1136,27 +992,19 @@ func (mock *BackendMock) ListObjectsCalls() []struct { } // ListObjectsV2 calls ListObjectsV2Func. -func (mock *BackendMock) ListObjectsV2(bucket string, prefix string, marker string, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { +func (mock *BackendMock) ListObjectsV2(listObjectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { if mock.ListObjectsV2Func == nil { panic("BackendMock.ListObjectsV2Func: method is nil but Backend.ListObjectsV2 was just called") } callInfo := struct { - Bucket string - Prefix string - Marker string - Delim string - Maxkeys int + ListObjectsV2Input *s3.ListObjectsV2Input }{ - Bucket: bucket, - Prefix: prefix, - Marker: marker, - Delim: delim, - Maxkeys: maxkeys, + ListObjectsV2Input: listObjectsV2Input, } mock.lockListObjectsV2.Lock() mock.calls.ListObjectsV2 = append(mock.calls.ListObjectsV2, callInfo) mock.lockListObjectsV2.Unlock() - return mock.ListObjectsV2Func(bucket, prefix, marker, delim, maxkeys) + return mock.ListObjectsV2Func(listObjectsV2Input) } // ListObjectsV2Calls gets all the calls that were made to ListObjectsV2. @@ -1164,18 +1012,10 @@ func (mock *BackendMock) ListObjectsV2(bucket string, prefix string, marker stri // // len(mockedBackend.ListObjectsV2Calls()) func (mock *BackendMock) ListObjectsV2Calls() []struct { - Bucket string - Prefix string - Marker string - Delim string - Maxkeys int + ListObjectsV2Input *s3.ListObjectsV2Input } { var calls []struct { - Bucket string - Prefix string - Marker string - Delim string - Maxkeys int + ListObjectsV2Input *s3.ListObjectsV2Input } mock.lockListObjectsV2.RLock() calls = mock.calls.ListObjectsV2 @@ -1183,39 +1023,35 @@ func (mock *BackendMock) ListObjectsV2Calls() []struct { return calls } -// PutBucket calls PutBucketFunc. -func (mock *BackendMock) PutBucket(bucket string, owner string) error { - if mock.PutBucketFunc == nil { - panic("BackendMock.PutBucketFunc: method is nil but Backend.PutBucket was just called") +// ListParts calls ListPartsFunc. +func (mock *BackendMock) ListParts(listPartsInput *s3.ListPartsInput) (s3response.ListPartsResponse, error) { + if mock.ListPartsFunc == nil { + panic("BackendMock.ListPartsFunc: method is nil but Backend.ListParts was just called") } callInfo := struct { - Bucket string - Owner string + ListPartsInput *s3.ListPartsInput }{ - Bucket: bucket, - Owner: owner, + ListPartsInput: listPartsInput, } - mock.lockPutBucket.Lock() - mock.calls.PutBucket = append(mock.calls.PutBucket, callInfo) - mock.lockPutBucket.Unlock() - return mock.PutBucketFunc(bucket, owner) + mock.lockListParts.Lock() + mock.calls.ListParts = append(mock.calls.ListParts, callInfo) + mock.lockListParts.Unlock() + return mock.ListPartsFunc(listPartsInput) } -// PutBucketCalls gets all the calls that were made to PutBucket. +// ListPartsCalls gets all the calls that were made to ListParts. // Check the length with: // -// len(mockedBackend.PutBucketCalls()) -func (mock *BackendMock) PutBucketCalls() []struct { - Bucket string - Owner string +// len(mockedBackend.ListPartsCalls()) +func (mock *BackendMock) ListPartsCalls() []struct { + ListPartsInput *s3.ListPartsInput } { var calls []struct { - Bucket string - Owner string + ListPartsInput *s3.ListPartsInput } - mock.lockPutBucket.RLock() - calls = mock.calls.PutBucket - mock.lockPutBucket.RUnlock() + mock.lockListParts.RLock() + calls = mock.calls.ListParts + mock.lockListParts.RUnlock() return calls } @@ -1319,58 +1155,6 @@ func (mock *BackendMock) PutObjectAclCalls() []struct { return calls } -// PutObjectPart calls PutObjectPartFunc. -func (mock *BackendMock) PutObjectPart(bucket string, object string, uploadID string, part int, length int64, r io.Reader) (string, error) { - if mock.PutObjectPartFunc == nil { - panic("BackendMock.PutObjectPartFunc: method is nil but Backend.PutObjectPart was just called") - } - callInfo := struct { - Bucket string - Object string - UploadID string - Part int - Length int64 - R io.Reader - }{ - Bucket: bucket, - Object: object, - UploadID: uploadID, - Part: part, - Length: length, - R: r, - } - mock.lockPutObjectPart.Lock() - mock.calls.PutObjectPart = append(mock.calls.PutObjectPart, callInfo) - mock.lockPutObjectPart.Unlock() - return mock.PutObjectPartFunc(bucket, object, uploadID, part, length, r) -} - -// PutObjectPartCalls gets all the calls that were made to PutObjectPart. -// Check the length with: -// -// len(mockedBackend.PutObjectPartCalls()) -func (mock *BackendMock) PutObjectPartCalls() []struct { - Bucket string - Object string - UploadID string - Part int - Length int64 - R io.Reader -} { - var calls []struct { - Bucket string - Object string - UploadID string - Part int - Length int64 - R io.Reader - } - mock.lockPutObjectPart.RLock() - calls = mock.calls.PutObjectPart - mock.lockPutObjectPart.RUnlock() - return calls -} - // RemoveTags calls RemoveTagsFunc. func (mock *BackendMock) RemoveTags(bucket string, object string) error { if mock.RemoveTagsFunc == nil { @@ -1408,23 +1192,19 @@ func (mock *BackendMock) RemoveTagsCalls() []struct { } // RestoreObject calls RestoreObjectFunc. -func (mock *BackendMock) RestoreObject(bucket string, object string, restoreRequest *s3.RestoreObjectInput) error { +func (mock *BackendMock) RestoreObject(restoreObjectInput *s3.RestoreObjectInput) error { if mock.RestoreObjectFunc == nil { panic("BackendMock.RestoreObjectFunc: method is nil but Backend.RestoreObject was just called") } callInfo := struct { - Bucket string - Object string - RestoreRequest *s3.RestoreObjectInput + RestoreObjectInput *s3.RestoreObjectInput }{ - Bucket: bucket, - Object: object, - RestoreRequest: restoreRequest, + RestoreObjectInput: restoreObjectInput, } mock.lockRestoreObject.Lock() mock.calls.RestoreObject = append(mock.calls.RestoreObject, callInfo) mock.lockRestoreObject.Unlock() - return mock.RestoreObjectFunc(bucket, object, restoreRequest) + return mock.RestoreObjectFunc(restoreObjectInput) } // RestoreObjectCalls gets all the calls that were made to RestoreObject. @@ -1432,14 +1212,10 @@ func (mock *BackendMock) RestoreObject(bucket string, object string, restoreRequ // // len(mockedBackend.RestoreObjectCalls()) func (mock *BackendMock) RestoreObjectCalls() []struct { - Bucket string - Object string - RestoreRequest *s3.RestoreObjectInput + RestoreObjectInput *s3.RestoreObjectInput } { var calls []struct { - Bucket string - Object string - RestoreRequest *s3.RestoreObjectInput + RestoreObjectInput *s3.RestoreObjectInput } mock.lockRestoreObject.RLock() calls = mock.calls.RestoreObject @@ -1541,6 +1317,38 @@ func (mock *BackendMock) StringCalls() []struct { return calls } +// UploadPart calls UploadPartFunc. +func (mock *BackendMock) UploadPart(uploadPartInput *s3.UploadPartInput) (string, error) { + if mock.UploadPartFunc == nil { + panic("BackendMock.UploadPartFunc: method is nil but Backend.UploadPart was just called") + } + callInfo := struct { + UploadPartInput *s3.UploadPartInput + }{ + UploadPartInput: uploadPartInput, + } + mock.lockUploadPart.Lock() + mock.calls.UploadPart = append(mock.calls.UploadPart, callInfo) + mock.lockUploadPart.Unlock() + return mock.UploadPartFunc(uploadPartInput) +} + +// UploadPartCalls gets all the calls that were made to UploadPart. +// Check the length with: +// +// len(mockedBackend.UploadPartCalls()) +func (mock *BackendMock) UploadPartCalls() []struct { + UploadPartInput *s3.UploadPartInput +} { + var calls []struct { + UploadPartInput *s3.UploadPartInput + } + mock.lockUploadPart.RLock() + calls = mock.calls.UploadPart + mock.lockUploadPart.RUnlock() + return calls +} + // UploadPartCopy calls UploadPartCopyFunc. func (mock *BackendMock) UploadPartCopy(uploadPartCopyInput *s3.UploadPartCopyInput) (s3response.CopyObjectResult, error) { if mock.UploadPartCopyFunc == nil { diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 6975127..acfd9a2 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -23,6 +23,7 @@ import ( "net/http" "strconv" "strings" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -44,6 +45,10 @@ type S3ApiController struct { evSender s3event.S3EventSender } +const ( + iso8601Format = "20060102T150405Z" +) + func New(be backend.Backend, iam auth.IAMService, logger s3log.AuditLogger, evs s3event.S3EventSender) S3ApiController { return S3ApiController{be: be, iam: iam, logger: logger, evSender: evs} } @@ -53,7 +58,7 @@ func (c S3ApiController) ListBuckets(ctx *fiber.Ctx) error { if err := auth.IsAdmin(access, isRoot); err != nil { return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListBucket"}) } - res, err := c.be.ListBuckets() + res, err := c.be.ListBuckets(access, isRoot) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListBucket"}) } @@ -63,7 +68,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { keyEnd := ctx.Params("*1") uploadId := ctx.Query("uploadId") maxParts := ctx.QueryInt("max-parts", 0) - partNumberMarker := ctx.QueryInt("part-number-marker", 0) + partNumberMarker := ctx.Query("part-number-marker") acceptRange := ctx.Get("Range") access := ctx.Locals("access").(string) isRoot := ctx.Locals("isRoot").(bool) @@ -71,7 +76,7 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { key = strings.Join([]string{key, keyEnd}, "/") } - data, err := c.be.GetBucketAcl(bucket) + data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket}) if err != nil { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger}) } @@ -101,25 +106,37 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { if uploadId != "" { if maxParts < 0 || (maxParts == 0 && ctx.Query("max-parts") != "") { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts), &MetaOpts{Logger: c.logger, Action: "ListObjectParts", BucketOwner: parsedAcl.Owner}) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidMaxParts), &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner}) } - if partNumberMarker < 0 || (partNumberMarker == 0 && ctx.Query("part-number-marker") != "") { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker), &MetaOpts{Logger: c.logger, Action: "ListObjectParts", BucketOwner: parsedAcl.Owner}) + if partNumberMarker != "" { + n, err := strconv.Atoi(partNumberMarker) + if err != nil || n < 0 { + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPartNumberMarker), &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner}) + } } if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { - return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListObjectParts", BucketOwner: parsedAcl.Owner}) + return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner}) } - res, err := c.be.ListObjectParts(bucket, key, uploadId, partNumberMarker, maxParts) - return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjectParts", BucketOwner: parsedAcl.Owner}) + res, err := c.be.ListParts(&s3.ListPartsInput{ + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + PartNumberMarker: &partNumberMarker, + MaxParts: int32(maxParts), + }) + return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListParts", BucketOwner: parsedAcl.Owner}) } if ctx.Request().URI().QueryArgs().Has("acl") { if err := auth.VerifyACL(parsedAcl, bucket, access, "READ_ACP", isRoot); err != nil { return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAcl", BucketOwner: parsedAcl.Owner}) } - res, err := c.be.GetObjectAcl(bucket, key) + res, err := c.be.GetObjectAcl(&s3.GetObjectAclInput{ + Bucket: &bucket, + Key: &key, + }) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAcl", BucketOwner: parsedAcl.Owner}) } @@ -127,7 +144,15 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAttributes", BucketOwner: parsedAcl.Owner}) } - res, err := c.be.GetObjectAttributes(bucket, key, strings.Split(attrs, ",")) + var oattrs []types.ObjectAttributes + for _, a := range strings.Split(attrs, ",") { + oattrs = append(oattrs, types.ObjectAttributes(a)) + } + res, err := c.be.GetObjectAttributes(&s3.GetObjectAttributesInput{ + Bucket: &bucket, + Key: &key, + ObjectAttributes: oattrs, + }) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "GetObjectAttributes", BucketOwner: parsedAcl.Owner}) } @@ -136,7 +161,11 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { } ctx.Locals("logResBody", false) - res, err := c.be.GetObject(bucket, key, acceptRange, ctx.Response().BodyWriter()) + res, err := c.be.GetObject(&s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + Range: &acceptRange, + }, ctx.Response().BodyWriter()) if err != nil { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "GetObject", BucketOwner: parsedAcl.Owner}) } @@ -194,7 +223,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { access := ctx.Locals("access").(string) isRoot := ctx.Locals("isRoot").(bool) - data, err := c.be.GetBucketAcl(bucket) + data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket}) if err != nil { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger}) } @@ -225,7 +254,13 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner}) } - res, err := c.be.ListObjectsV2(bucket, prefix, marker, delimiter, maxkeys) + res, err := c.be.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: &bucket, + Prefix: &prefix, + ContinuationToken: &marker, + Delimiter: &delimiter, + MaxKeys: int32(maxkeys), + }) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner}) } @@ -233,7 +268,13 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner}) } - res, err := c.be.ListObjects(bucket, prefix, marker, delimiter, maxkeys) + res, err := c.be.ListObjects(&s3.ListObjectsInput{ + Bucket: &bucket, + Prefix: &prefix, + Marker: &marker, + Delimiter: &delimiter, + MaxKeys: int32(maxkeys), + }) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner}) } @@ -255,7 +296,7 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { if ctx.Request().URI().QueryArgs().Has("acl") { var input *s3.PutBucketAclInput - data, err := c.be.GetBucketAcl(bucket) + data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket}) if err != nil { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl"}) } @@ -322,7 +363,10 @@ func (c S3ApiController) PutBucketActions(ctx *fiber.Ctx) error { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucketAcl", BucketOwner: parsedAcl.Owner}) } - err := c.be.PutBucket(bucket, access) + err := c.be.CreateBucket(&s3.CreateBucketInput{ + Bucket: &bucket, + ObjectOwnership: types.ObjectOwnership(access), + }) return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutBucket", BucketOwner: ctx.Locals("access").(string)}) } @@ -364,7 +408,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { keyStart = keyStart + "/" } - data, err := c.be.GetBucketAcl(bucket) + data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket}) if err != nil { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger}) } @@ -422,24 +466,30 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { if ctx.Request().URI().QueryArgs().Has("uploadId") && ctx.Request().URI().QueryArgs().Has("partNumber") { partNumber := ctx.QueryInt("partNumber", -1) if partNumber < 1 || partNumber > 10000 { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "PutObjectPart", BucketOwner: parsedAcl.Owner}) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidPart), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner}) } if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { - return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutObjectPart", BucketOwner: parsedAcl.Owner}) + return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner}) } contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) if err != nil { - return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "PutObjectPart", BucketOwner: parsedAcl.Owner}) + return SendResponse(ctx, s3err.GetAPIError(s3err.ErrInvalidRequest), &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner}) } body := io.ReadSeeker(bytes.NewReader([]byte(ctx.Body()))) ctx.Locals("logReqBody", false) - etag, err := c.be.PutObjectPart(bucket, keyStart, uploadId, - partNumber, contentLength, body) + etag, err := c.be.UploadPart(&s3.UploadPartInput{ + Bucket: &bucket, + Key: &keyStart, + UploadId: &uploadId, + PartNumber: int32(partNumber), + ContentLength: contentLength, + Body: body, + }) ctx.Response().Header.Set("Etag", etag) - return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "PutObjectPart", BucketOwner: parsedAcl.Owner}) + return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "UploadPart", BucketOwner: parsedAcl.Owner}) } if ctx.Request().URI().QueryArgs().Has("acl") { @@ -503,16 +553,33 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { } if copySource != "" { - _, _, _, _ = copySrcIfMatch, copySrcIfNoneMatch, - copySrcModifSince, copySrcUnmodifSince - copySourceSplit := strings.Split(copySource, "/") - srcBucket, srcObject := copySourceSplit[0], copySourceSplit[1:] - if err := auth.VerifyACL(parsedAcl, bucket, access, "WRITE", isRoot); err != nil { return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner}) } - res, err := c.be.CopyObject(srcBucket, strings.Join(srcObject, "/"), bucket, keyStart) + var mtime time.Time + if copySrcModifSince != "" { + mtime, err = time.Parse(iso8601Format, copySrcModifSince) + if err != nil { + return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidCopySource), &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner}) + } + } + var umtime time.Time + if copySrcModifSince != "" { + mtime, err = time.Parse(iso8601Format, copySrcUnmodifSince) + if err != nil { + return SendXMLResponse(ctx, nil, s3err.GetAPIError(s3err.ErrInvalidCopySource), &MetaOpts{Logger: c.logger, Action: "CopyObject", BucketOwner: parsedAcl.Owner}) + } + } + res, err := c.be.CopyObject(&s3.CopyObjectInput{ + Bucket: &bucket, + Key: &keyStart, + CopySource: ©Source, + CopySourceIfMatch: ©SrcIfMatch, + CopySourceIfNoneMatch: ©SrcIfNoneMatch, + CopySourceIfModifiedSince: &mtime, + CopySourceIfUnmodifiedSince: &umtime, + }) if err == nil { return SendXMLResponse(ctx, res, err, &MetaOpts{ Logger: c.logger, @@ -566,7 +633,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error { bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool) - data, err := c.be.GetBucketAcl(bucket) + data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket}) if err != nil { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBuckets"}) } @@ -580,7 +647,9 @@ func (c S3ApiController) DeleteBucket(ctx *fiber.Ctx) error { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBucket", BucketOwner: parsedAcl.Owner}) } - err = c.be.DeleteBucket(bucket) + err = c.be.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: &bucket, + }) return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteBucket", BucketOwner: parsedAcl.Owner}) } @@ -588,7 +657,7 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error { bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool) var dObj types.Delete - data, err := c.be.GetBucketAcl(bucket) + data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket}) if err != nil { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects"}) } @@ -606,7 +675,10 @@ func (c S3ApiController) DeleteObjects(ctx *fiber.Ctx) error { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner}) } - err = c.be.DeleteObjects(bucket, &s3.DeleteObjectsInput{Delete: &dObj}) + err = c.be.DeleteObjects(&s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &dObj, + }) return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObjects", BucketOwner: parsedAcl.Owner}) } @@ -622,7 +694,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { key = strings.Join([]string{key, keyEnd}, "/") } - data, err := c.be.GetBucketAcl(bucket) + data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket}) if err != nil { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger}) } @@ -668,7 +740,10 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "DeleteObject", BucketOwner: parsedAcl.Owner}) } - err = c.be.DeleteObject(bucket, key) + err = c.be.DeleteObject(&s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &key, + }) return SendResponse(ctx, err, &MetaOpts{ Logger: c.logger, EvSender: c.evSender, @@ -681,7 +756,7 @@ func (c S3ApiController) DeleteActions(ctx *fiber.Ctx) error { func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error { bucket, access, isRoot := ctx.Params("bucket"), ctx.Locals("access").(string), ctx.Locals("isRoot").(bool) - data, err := c.be.GetBucketAcl(bucket) + data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket}) if err != nil { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket"}) } @@ -695,7 +770,9 @@ func (c S3ApiController) HeadBucket(ctx *fiber.Ctx) error { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket", BucketOwner: parsedAcl.Owner}) } - _, err = c.be.HeadBucket(bucket) + _, err = c.be.HeadBucket(&s3.HeadBucketInput{ + Bucket: &bucket, + }) // TODO: set bucket response headers return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadBucket", BucketOwner: parsedAcl.Owner}) } @@ -712,7 +789,9 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { key = strings.Join([]string{key, keyEnd}, "/") } - data, err := c.be.GetBucketAcl(bucket) + data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{ + Bucket: &bucket, + }) if err != nil { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject"}) } @@ -726,7 +805,10 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject", BucketOwner: parsedAcl.Owner}) } - res, err := c.be.HeadObject(bucket, key) + res, err := c.be.HeadObject(&s3.HeadObjectInput{ + Bucket: &bucket, + Key: &key, + }) if err != nil { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "HeadObject", BucketOwner: parsedAcl.Owner}) } @@ -785,7 +867,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { key = strings.Join([]string{key, keyEnd}, "/") } - data, err := c.be.GetBucketAcl(bucket) + data, err := c.be.GetBucketAcl(&s3.GetBucketAclInput{Bucket: &bucket}) if err != nil { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger}) } @@ -806,7 +888,10 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { return SendResponse(ctx, err, &MetaOpts{Logger: c.logger, Action: "RestoreObject", BucketOwner: parsedAcl.Owner}) } - err = c.be.RestoreObject(bucket, key, &restoreRequest) + restoreRequest.Bucket = &bucket + restoreRequest.Key = &key + + err = c.be.RestoreObject(&restoreRequest) return SendResponse(ctx, err, &MetaOpts{ Logger: c.logger, EvSender: c.evSender, @@ -818,7 +903,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { if uploadId != "" { data := struct { - Parts []types.Part `xml:"Part"` + Parts []types.CompletedPart `xml:"Part"` }{} if err := xml.Unmarshal(ctx.Body(), &data); err != nil { @@ -829,7 +914,14 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "CompleteMultipartUpload", BucketOwner: parsedAcl.Owner}) } - res, err := c.be.CompleteMultipartUpload(bucket, key, uploadId, data.Parts) + res, err := c.be.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: data.Parts, + }, + }) if err == nil { return SendXMLResponse(ctx, res, err, &MetaOpts{ Logger: c.logger, diff --git a/s3api/controllers/base_test.go b/s3api/controllers/base_test.go index d1db2da..af331ce 100644 --- a/s3api/controllers/base_test.go +++ b/s3api/controllers/base_test.go @@ -89,10 +89,10 @@ func TestS3ApiController_ListBuckets(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - ListBucketsFunc: func() (s3response.ListAllMyBucketsResult, error) { + ListBucketsFunc: func(string, bool) (s3response.ListAllMyBucketsResult, error) { return s3response.ListAllMyBucketsResult{}, nil }, }, @@ -110,10 +110,10 @@ func TestS3ApiController_ListBuckets(t *testing.T) { appErr := fiber.New() s3ApiControllerErr := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - ListBucketsFunc: func() (s3response.ListAllMyBucketsResult, error) { + ListBucketsFunc: func(string, bool) (s3response.ListAllMyBucketsResult, error) { return s3response.ListAllMyBucketsResult{}, s3err.GetAPIError(s3err.ErrMethodNotAllowed) }, }, @@ -201,19 +201,19 @@ func TestS3ApiController_GetActions(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - ListObjectPartsFunc: func(bucket, object, uploadID string, partNumberMarker int, maxParts int) (s3response.ListPartsResponse, error) { + ListPartsFunc: func(*s3.ListPartsInput) (s3response.ListPartsResponse, error) { return s3response.ListPartsResponse{}, nil }, - GetObjectAclFunc: func(bucket, object string) (*s3.GetObjectAclOutput, error) { + GetObjectAclFunc: func(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) { return &s3.GetObjectAclOutput{}, nil }, - GetObjectAttributesFunc: func(bucket, object string, attributes []string) (*s3.GetObjectAttributesOutput, error) { + GetObjectAttributesFunc: func(*s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error) { return &s3.GetObjectAttributesOutput{}, nil }, - GetObjectFunc: func(bucket, object, acceptRange string, writer io.Writer) (*s3.GetObjectOutput, error) { + GetObjectFunc: func(*s3.GetObjectInput, io.Writer) (*s3.GetObjectOutput, error) { return &s3.GetObjectOutput{ Metadata: map[string]string{"hello": "world"}, ContentType: getPtr("application/xml"), @@ -353,16 +353,16 @@ func TestS3ApiController_ListActions(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, ListMultipartUploadsFunc: func(output *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResponse, error) { return s3response.ListMultipartUploadsResponse{}, nil }, - ListObjectsV2Func: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { + ListObjectsV2Func: func(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { return &s3.ListObjectsV2Output{}, nil }, - ListObjectsFunc: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { + ListObjectsFunc: func(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { return &s3.ListObjectsOutput{}, nil }, }, @@ -380,10 +380,10 @@ func TestS3ApiController_ListActions(t *testing.T) { //Error case s3ApiControllerError := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - ListObjectsFunc: func(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsOutput, error) { + ListObjectsFunc: func(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { return nil, s3err.GetAPIError(s3err.ErrNotImplemented) }, }, @@ -498,13 +498,13 @@ func TestS3ApiController_PutBucketActions(t *testing.T) { s3ApiController := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, PutBucketAclFunc: func(string, []byte) error { return nil }, - PutBucketFunc: func(bucket, owner string) error { + CreateBucketFunc: func(*s3.CreateBucketInput) error { return nil }, }, @@ -650,13 +650,13 @@ func TestS3ApiController_PutActions(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, PutObjectAclFunc: func(*s3.PutObjectAclInput) error { return nil }, - CopyObjectFunc: func(srcBucket, srcObject, DstBucket, dstObject string) (*s3.CopyObjectOutput, error) { + CopyObjectFunc: func(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error) { return &s3.CopyObjectOutput{ CopyObjectResult: &types.CopyObjectResult{}, }, nil @@ -664,7 +664,7 @@ func TestS3ApiController_PutActions(t *testing.T) { PutObjectFunc: func(*s3.PutObjectInput) (string, error) { return "ETag", nil }, - PutObjectPartFunc: func(bucket, object, uploadID string, part int, length int64, r io.Reader) (string, error) { + UploadPartFunc: func(*s3.UploadPartInput) (string, error) { return "hello", nil }, SetTagsFunc: func(bucket, object string, tags map[string]string) error { @@ -864,10 +864,10 @@ func TestS3ApiController_DeleteBucket(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - DeleteBucketFunc: func(bucket string) error { + DeleteBucketFunc: func(*s3.DeleteBucketInput) error { return nil }, }, @@ -920,10 +920,10 @@ func TestS3ApiController_DeleteObjects(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - DeleteObjectsFunc: func(bucket string, objects *s3.DeleteObjectsInput) error { + DeleteObjectsFunc: func(*s3.DeleteObjectsInput) error { return nil }, }, @@ -990,10 +990,10 @@ func TestS3ApiController_DeleteActions(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - DeleteObjectFunc: func(bucket, object string) error { + DeleteObjectFunc: func(*s3.DeleteObjectInput) error { return nil }, AbortMultipartUploadFunc: func(*s3.AbortMultipartUploadInput) error { @@ -1017,10 +1017,10 @@ func TestS3ApiController_DeleteActions(t *testing.T) { appErr := fiber.New() s3ApiControllerErr := S3ApiController{be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - DeleteObjectFunc: func(bucket, object string) error { + DeleteObjectFunc: func(*s3.DeleteObjectInput) error { return s3err.GetAPIError(7) }, }} @@ -1098,10 +1098,10 @@ func TestS3ApiController_HeadBucket(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) { + HeadBucketFunc: func(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { return &s3.HeadBucketOutput{}, nil }, }, @@ -1120,10 +1120,10 @@ func TestS3ApiController_HeadBucket(t *testing.T) { appErr := fiber.New() s3ApiControllerErr := S3ApiController{be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - HeadBucketFunc: func(bucket string) (*s3.HeadBucketOutput, error) { + HeadBucketFunc: func(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { return nil, s3err.GetAPIError(3) }, }, @@ -1192,10 +1192,10 @@ func TestS3ApiController_HeadObject(t *testing.T) { s3ApiController := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - HeadObjectFunc: func(bucket, object string) (*s3.HeadObjectOutput, error) { + HeadObjectFunc: func(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { return &s3.HeadObjectOutput{ ContentEncoding: &contentEncoding, ContentLength: 64, @@ -1220,10 +1220,10 @@ func TestS3ApiController_HeadObject(t *testing.T) { s3ApiControllerErr := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - HeadObjectFunc: func(bucket, object string) (*s3.HeadObjectOutput, error) { + HeadObjectFunc: func(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { return nil, s3err.GetAPIError(42) }, }, @@ -1283,13 +1283,13 @@ func TestS3ApiController_CreateActions(t *testing.T) { app := fiber.New() s3ApiController := S3ApiController{ be: &BackendMock{ - GetBucketAclFunc: func(bucket string) ([]byte, error) { + GetBucketAclFunc: func(*s3.GetBucketAclInput) ([]byte, error) { return acldata, nil }, - RestoreObjectFunc: func(bucket, object string, restoreRequest *s3.RestoreObjectInput) error { + RestoreObjectFunc: func(restoreRequest *s3.RestoreObjectInput) error { return nil }, - CompleteMultipartUploadFunc: func(bucket, object, uploadID string, parts []types.Part) (*s3.CompleteMultipartUploadOutput, error) { + CompleteMultipartUploadFunc: func(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { return &s3.CompleteMultipartUploadOutput{}, nil }, CreateMultipartUploadFunc: func(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {