diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 87f47e2..1c45ab9 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -536,6 +536,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl objs, _ := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir)) var uploads []s3response.Upload + var resultUpds []s3response.Upload var keyMarker string if mpu.KeyMarker != nil { @@ -545,12 +546,9 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl if mpu.UploadIdMarker != nil { uploadIDMarker = *mpu.UploadIdMarker } - var pastMarker bool - if keyMarker == "" && uploadIDMarker == "" { - pastMarker = true - } + keyMarkerInd, uploadIdMarkerFound := -1, false - for i, obj := range objs { + for _, obj := range objs { if !obj.IsDir() { continue } @@ -569,22 +567,14 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl continue } - for j, upid := range upids { + for _, upid := range upids { if !upid.IsDir() { continue } - if objectName == keyMarker || upid.Name() == uploadIDMarker { - pastMarker = true - continue - } - if keyMarker != "" && uploadIDMarker != "" && !pastMarker { - continue - } - - userMetaData := make(map[string]string) - upiddir := filepath.Join(bucket, metaTmpMultipartDir, obj.Name(), upid.Name()) - loadUserMetaData(upiddir, userMetaData) + // userMetaData := make(map[string]string) + // upiddir := filepath.Join(bucket, metaTmpMultipartDir, obj.Name(), upid.Name()) + // loadUserMetaData(upiddir, userMetaData) fi, err := upid.Info() if err != nil { @@ -592,28 +582,61 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl } uploadID := upid.Name() + if !uploadIdMarkerFound && uploadIDMarker == uploadID { + uploadIdMarkerFound = true + } + if keyMarkerInd == -1 && objectName == keyMarker { + keyMarkerInd = len(uploads) + } uploads = append(uploads, s3response.Upload{ Key: objectName, UploadID: uploadID, Initiated: fi.ModTime().Format(backend.RFC3339TimeFormat), }) - if len(uploads) == int(mpu.MaxUploads) { - return s3response.ListMultipartUploadsResult{ - Bucket: bucket, - Delimiter: delimiter, - IsTruncated: i != len(objs) || j != len(upids), - KeyMarker: keyMarker, - MaxUploads: int(mpu.MaxUploads), - NextKeyMarker: objectName, - NextUploadIDMarker: uploadID, - Prefix: prefix, - UploadIDMarker: uploadIDMarker, - Uploads: uploads, - }, nil - } } } + if (uploadIDMarker != "" && !uploadIdMarkerFound) || (keyMarker != "" && keyMarkerInd == -1) { + return s3response.ListMultipartUploadsResult{ + Bucket: bucket, + Delimiter: delimiter, + KeyMarker: keyMarker, + MaxUploads: int(mpu.MaxUploads), + Prefix: prefix, + UploadIDMarker: uploadIDMarker, + Uploads: []s3response.Upload{}, + }, nil + } + + sort.SliceStable(uploads, func(i, j int) bool { + return uploads[i].Key < uploads[j].Key + }) + + for i := keyMarkerInd + 1; i < len(uploads); i++ { + if mpu.MaxUploads == 0 { + break + } + if keyMarker != "" && uploadIDMarker != "" && uploads[i].UploadID < uploadIDMarker { + continue + } + if i != len(uploads)-1 && len(resultUpds) == int(mpu.MaxUploads) { + return s3response.ListMultipartUploadsResult{ + Bucket: bucket, + Delimiter: delimiter, + KeyMarker: keyMarker, + MaxUploads: int(mpu.MaxUploads), + NextKeyMarker: resultUpds[i-1].Key, + NextUploadIDMarker: resultUpds[i-1].UploadID, + IsTruncated: true, + Prefix: prefix, + UploadIDMarker: uploadIDMarker, + Uploads: resultUpds, + }, nil + } + + resultUpds = append(resultUpds, uploads[i]) + } + return s3response.ListMultipartUploadsResult{ Bucket: bucket, Delimiter: delimiter, @@ -621,7 +644,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl MaxUploads: int(mpu.MaxUploads), Prefix: prefix, UploadIDMarker: uploadIDMarker, - Uploads: uploads, + Uploads: resultUpds, }, nil } diff --git a/integration/action-tests.go b/integration/action-tests.go index c3ea1b5..d1650dd 100644 --- a/integration/action-tests.go +++ b/integration/action-tests.go @@ -133,6 +133,10 @@ func TestListParts(s *S3Conf) { func TestListMultipartUploads(s *S3Conf) { ListMultipartUploads_non_existing_bucket(s) ListMultipartUploads_empty_result(s) + ListMultipartUploads_invalid_max_uploads(s) + ListMultipartUploads_max_uploads(s) + ListMultipartUploads_incorrect_next_key_marker(s) + ListMultipartUploads_ignore_upload_id_marker(s) ListMultipartUploads_success(s) } diff --git a/integration/tests.go b/integration/tests.go index 30d12bd..aa7e0fc 100644 --- a/integration/tests.go +++ b/integration/tests.go @@ -1652,12 +1652,7 @@ func CreateMultipartUpload_non_existing_bucket(s *S3Conf) { testName := "CreateMultipartUpload_non_existing_bucket" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { bucketName := getBucketName() - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucketName, - Key: getPtr("my-obj"), - }) - cancel() + _, err := CreateMp(s3client, bucketName, "my-obj") if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { return err } @@ -1670,12 +1665,7 @@ func CreateMultipartUpload_success(s *S3Conf) { testName := "CreateMultipartUpload_success" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } @@ -1753,17 +1743,12 @@ func UploadPart_non_existing_key(s *S3Conf) { testName := "UploadPart_non_existing_key" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: getPtr("non-existing-object-key"), @@ -1782,16 +1767,11 @@ func UploadPart_success(s *S3Conf) { testName := "UploadPart_success" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: &obj, @@ -1842,17 +1822,12 @@ func UploadPartCopy_incorrect_uploadId(s *S3Conf) { return err } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + _, err = CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), @@ -1887,17 +1862,12 @@ func UploadPartCopy_incorrect_object_key(s *S3Conf) { return err } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), @@ -1944,17 +1914,12 @@ func UploadPartCopy_invalid_copy_source(s *S3Conf) { actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr("invalid-copy-source"), @@ -1976,17 +1941,12 @@ func UploadPartCopy_non_existing_source_bucket(s *S3Conf) { actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr("src/bucket/src/obj"), @@ -2013,17 +1973,12 @@ func UploadPartCopy_non_existing_source_object_key(s *S3Conf) { return nil } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/non/existing/obj/key"), @@ -2062,17 +2017,12 @@ func UploadPartCopy_success(s *S3Conf) { return err } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), @@ -2135,17 +2085,12 @@ func UploadPartCopy_by_range_invalid_range(s *S3Conf) { return err } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), @@ -2185,17 +2130,12 @@ func UploadPartCopy_by_range_success(s *S3Conf) { return err } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ Bucket: &bucket, CopySource: getPtr(srcBucket + "/" + srcObj), @@ -2264,17 +2204,12 @@ func ListParts_incorrect_object_key(s *S3Conf) { testName := "ListParts_incorrect_object_key" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.ListParts(ctx, &s3.ListPartsInput{ Bucket: &bucket, Key: getPtr("incorrect-object-key"), @@ -2293,12 +2228,7 @@ func ListParts_success(s *S3Conf) { testName := "ListParts_success" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } @@ -2308,7 +2238,7 @@ func ListParts_success(s *S3Conf) { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ Bucket: &bucket, Key: &obj, @@ -2363,31 +2293,146 @@ func ListMultipartUploads_empty_result(s *S3Conf) { }) } +func ListMultipartUploads_invalid_max_uploads(s *S3Conf) { + testName := "ListMultipartUploads_invalid_max_uploads" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + MaxUploads: -3, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)); err != nil { + return err + } + + return nil + }) +} + +func ListMultipartUploads_max_uploads(s *S3Conf) { + testName := "ListMultipartUploads_max_uploads" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + uploads := []types.MultipartUpload{} + for i := 1; i < 6; i++ { + out, err := CreateMp(s3client, bucket, fmt.Sprintf("obj%v", i)) + if err != nil { + return err + } + uploads = append(uploads, types.MultipartUpload{UploadId: out.UploadId, Key: out.Key}) + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + MaxUploads: 2, + }) + cancel() + if err != nil { + return err + } + if !out.IsTruncated { + return fmt.Errorf("expected the output to be truncated") + } + if out.MaxUploads != 2 { + return fmt.Errorf("expected max-uploads to be 2, instead got %v", out.MaxUploads) + } + if ok := compareMultipartUploads(out.Uploads, uploads[:2]); !ok { + return fmt.Errorf("expected multipart uploads to be %v, instead got %v", uploads[:2], out.Uploads) + } + if *out.NextKeyMarker != *uploads[1].Key { + return fmt.Errorf("expected next-key-marker to be %v, instead got %v", *uploads[1].Key, *out.NextKeyMarker) + } + if *out.NextUploadIdMarker != *uploads[1].UploadId { + return fmt.Errorf("expected next-upload-id-marker to be %v, instead got %v", *uploads[1].Key, *out.NextKeyMarker) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + KeyMarker: out.NextKeyMarker, + }) + cancel() + if err != nil { + return err + } + if ok := compareMultipartUploads(out.Uploads, uploads[2:]); !ok { + return fmt.Errorf("expected multipart uploads to be %v, instead got %v", uploads[2:], out.Uploads) + } + + return nil + }) +} + +func ListMultipartUploads_incorrect_next_key_marker(s *S3Conf) { + testName := "ListMultipartUploads_incorrect_next_key_marker" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for i := 1; i < 6; i++ { + _, err := CreateMp(s3client, bucket, fmt.Sprintf("obj%v", i)) + if err != nil { + return err + } + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + KeyMarker: getPtr("incorrect_object_key"), + }) + cancel() + if err != nil { + return err + } + + if len(out.Uploads) != 0 { + return fmt.Errorf("expected empty list of multipart uploads, instead got %v", out.Uploads) + } + + return nil + }) +} + +func ListMultipartUploads_ignore_upload_id_marker(s *S3Conf) { + testName := "ListMultipartUploads_ignore_upload_id_marker" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + uploads := []types.MultipartUpload{} + for i := 1; i < 6; i++ { + out, err := CreateMp(s3client, bucket, fmt.Sprintf("obj%v", i)) + if err != nil { + return err + } + uploads = append(uploads, types.MultipartUpload{UploadId: out.UploadId, Key: out.Key}) + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + UploadIdMarker: uploads[2].UploadId, + }) + cancel() + if err != nil { + return err + } + if ok := compareMultipartUploads(out.Uploads, uploads); !ok { + return fmt.Errorf("expected multipart uploads to be %v, instead got %v", uploads, out.Uploads) + } + + return nil + }) +} + func ListMultipartUploads_success(s *S3Conf) { testName := "ListMultipartUploads_max_uploads" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj1, obj2 := "my-obj-1", "my-obj-2" + out1, err := CreateMp(s3client, bucket, obj1) + if err != nil { + return err + } + + out2, err := CreateMp(s3client, bucket, obj2) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out1, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj1, - }) - cancel() - if err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out2, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj2, - }) - cancel() - if err != nil { - return err - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ Bucket: &bucket, }) @@ -2397,14 +2442,14 @@ func ListMultipartUploads_success(s *S3Conf) { } expected := []types.MultipartUpload{ - { - Key: &obj2, - UploadId: out2.UploadId, - }, { Key: &obj1, UploadId: out1.UploadId, }, + { + Key: &obj2, + UploadId: out2.UploadId, + }, } if len(out.Uploads) != 2 { @@ -2458,17 +2503,12 @@ func AbortMultipartUpload_incorrect_object_key(s *S3Conf) { testName := "AbortMultipartUpload_incorrect_object_key" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ Bucket: &bucket, Key: getPtr("incorrect-object-key"), @@ -2487,17 +2527,12 @@ func AbortMultipartUpload_success(s *S3Conf) { testName := "AbortMultipartUpload_success" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ Bucket: &bucket, Key: &obj, @@ -2547,16 +2582,11 @@ func CompleteMultipartUpload_invalid_part_number(s *S3Conf) { testName := "CompleteMultipartUpload_invalid_part_number" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: &obj, @@ -2595,16 +2625,11 @@ func CompleteMultipartUpload_invalid_ETag(s *S3Conf) { testName := "CompleteMultipartUpload_invalid_ETag" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, Key: &obj, @@ -2643,12 +2668,7 @@ func CompleteMultipartUpload_success(s *S3Conf) { testName := "CompleteMultipartUpload_success" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() + out, err := CreateMp(s3client, bucket, obj) if err != nil { return err } @@ -2667,7 +2687,7 @@ func CompleteMultipartUpload_success(s *S3Conf) { }) } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ Bucket: &bucket, Key: &obj, diff --git a/integration/utils.go b/integration/utils.go index ac71d11..8a3fd18 100644 --- a/integration/utils.go +++ b/integration/utils.go @@ -240,6 +240,16 @@ func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) return } +func CreateMp(s3client *s3.Client, bucket, key string) (*s3.CreateMultipartUploadOutput, error) { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &key, + }) + cancel() + return out, err +} + func isEqual(a, b []byte) bool { if len(a) != len(b) { return false diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 927c6f4..8ce0ea0 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -25,7 +25,6 @@ import ( "strings" "time" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gofiber/fiber/v2" @@ -222,6 +221,9 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { marker := ctx.Query("marker") delimiter := ctx.Query("delimiter") maxkeysStr := ctx.Query("max-keys") + keyMarker := ctx.Query("key-marker") + maxUploadsStr := ctx.Query("max-uploads") + uploadIdMarker := ctx.Query("upload-id-marker") access := ctx.Locals("access").(string) isRoot := ctx.Locals("isRoot").(bool) parsedAcl := ctx.Locals("parsedAcl").(auth.ACL) @@ -244,7 +246,22 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner}) } - res, err := c.be.ListMultipartUploads(ctx.Context(), &s3.ListMultipartUploadsInput{Bucket: aws.String(ctx.Params("bucket"))}) + maxUploads, err := utils.ParseUint(maxUploadsStr) + if err != nil { + return SendXMLResponse(ctx, nil, err, &MetaOpts{ + Logger: c.logger, + Action: "ListMultipartUploads", + BucketOwner: parsedAcl.Owner, + }) + } + res, err := c.be.ListMultipartUploads(ctx.Context(), &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + Delimiter: &delimiter, + Prefix: &prefix, + UploadIdMarker: &uploadIdMarker, + MaxUploads: maxUploads, + KeyMarker: &keyMarker, + }) return SendXMLResponse(ctx, res, err, &MetaOpts{Logger: c.logger, Action: "ListMultipartUploads", BucketOwner: parsedAcl.Owner}) } @@ -252,7 +269,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { if err := auth.VerifyACL(parsedAcl, bucket, access, "READ", isRoot); err != nil { return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListObjectsV2", BucketOwner: parsedAcl.Owner}) } - maxkeys, err := utils.ParseMaxKeys(maxkeysStr) + maxkeys, err := utils.ParseUint(maxkeysStr) if err != nil { return SendXMLResponse(ctx, nil, err, &MetaOpts{ Logger: c.logger, @@ -274,7 +291,7 @@ func (c S3ApiController) ListActions(ctx *fiber.Ctx) error { return SendXMLResponse(ctx, nil, err, &MetaOpts{Logger: c.logger, Action: "ListObjects", BucketOwner: parsedAcl.Owner}) } - maxkeys, err := utils.ParseMaxKeys(maxkeysStr) + maxkeys, err := utils.ParseUint(maxkeysStr) if err != nil { return SendXMLResponse(ctx, nil, err, &MetaOpts{ Logger: c.logger, diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 4103d72..45a998b 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -80,7 +80,7 @@ func SetMetaHeaders(ctx *fiber.Ctx, meta map[string]string) { } } -func ParseMaxKeys(str string) (int32, error) { +func ParseUint(str string) (int32, error) { if str == "" { return -1, nil } diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index 4fbaa6f..d473d0b 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -222,7 +222,7 @@ func TestIsValidBucketName(t *testing.T) { } } -func TestParseMaxKeys(t *testing.T) { +func TestParseUint(t *testing.T) { type args struct { str string } @@ -233,7 +233,7 @@ func TestParseMaxKeys(t *testing.T) { wantErr bool }{ { - name: "Parse-max-keys-empty-string", + name: "Parse-uint-empty-string", args: args{ str: "", }, @@ -241,7 +241,7 @@ func TestParseMaxKeys(t *testing.T) { wantErr: false, }, { - name: "Parse-max-keys-invalid-number-string", + name: "Parse-uint-invalid-number-string", args: args{ str: "bla", }, @@ -249,7 +249,7 @@ func TestParseMaxKeys(t *testing.T) { wantErr: true, }, { - name: "Parse-max-keys-invalid-negative-number", + name: "Parse-uint-invalid-negative-number", args: args{ str: "-5", }, @@ -259,7 +259,7 @@ func TestParseMaxKeys(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := ParseMaxKeys(tt.args.str) + got, err := ParseUint(tt.args.str) if (err != nil) != tt.wantErr { t.Errorf("ParseMaxKeys() error = %v, wantErr %v", err, tt.wantErr) return