diff --git a/cmd/versitygw/test.go b/cmd/versitygw/test.go index 578b30a4..1bcfcd96 100644 --- a/cmd/versitygw/test.go +++ b/cmd/versitygw/test.go @@ -67,114 +67,6 @@ func initTestFlags() []cli.Flag { func initTestCommands() []*cli.Command { return []*cli.Command{ - { - Name: "bucket-actions", - Usage: "Test bucket creation, checking the existence, deletes it.", - Description: `Calls s3 gateway create-bucket action to create a new bucket, - calls head-bucket action to check the existence, then calls delete-bucket action to delete the bucket.`, - Action: getAction(integration.TestMakeBucket), - }, - { - Name: "object-actions", - Usage: "Test put/get/delete/copy objects.", - Description: `Creates a bucket with s3 gateway action, puts an object in it, - tries to copy into another bucket, that doesn't exist, creates the destination bucket for copying, - copies the object, get's the object to check the length and content, - get's the copied object to check the length and content, deletes all the objects inside the source bucket, - deletes both the objects and buckets.`, - Action: getAction(integration.TestPutGetObject), - }, - { - Name: "put-get-mp-object", - Usage: "Test put & get multipart object.", - Description: `Creates a bucket with s3 gateway action, puts an object in it with multipart upload, - gets the object from the bucket, deletes both the object and bucket.`, - Action: getAction(integration.TestPutGetMPObject), - }, - { - Name: "put-dir-object", - Usage: "Test put directory object.", - Description: `Creates a bucket with s3 gateway action, puts a directory object in it, - lists the bucket's objects, deletes both the objects and bucket.`, - Action: getAction(integration.TestPutDirObject), - }, - { - Name: "list-objects", - Usage: "Test list-objects action.", - Description: `Creates a bucket with s3 gateway action, puts 2 directory objects in it, - lists the bucket's objects, deletes both the objects and bucket.`, - Action: getAction(integration.TestListObject), - }, - { - Name: "abort-mp", - Usage: "Tests abort-multipart-upload action.", - Description: `Creates a bucket with s3 gateway action, creates a multipart upload, - lists the multipart upload, aborts the multipart upload, lists the multipart upload again, - deletes both the objects and bucket.`, - Action: getAction(integration.TestListAbortMultiPartObject), - }, - { - Name: "list-parts", - Usage: "Tests list-parts action.", - Description: `Creates a bucket with s3 gateway action, creates a multipart upload, - lists the upload parts, deletes both the objects and bucket.`, - Action: getAction(integration.TestListMultiParts), - }, - { - Name: "incorrect-mp", - Usage: "Tests incorrect multipart case.", - Description: `Creates a bucket with s3 gateway action, creates a multipart upload, - uploads different parts, completes the multipart upload with incorrect part numbers, - calls the head-object action, compares the content length, removes both the object and bucket`, - Action: getAction(integration.TestIncorrectMultiParts), - }, - { - Name: "incomplete-mp", - Usage: "Tests incomplete multi parts.", - Description: `Creates a bucket with s3 gateway action, creates a multipart upload, - upload a part, lists the parts, checks if the uploaded part is in the list, - removes both the object and the bucket`, - Action: getAction(integration.TestIncompleteMultiParts), - }, - { - Name: "incomplete-put-object", - Usage: "Tests incomplete put objects case.", - Description: `Creates a bucket with s3 gateway action, puts an object in it, - gets the object with head-object action, expects the object to be got, - removes both the object and bucket`, - Action: getAction(integration.TestIncompletePutObject), - }, - { - Name: "get-range", - Usage: "Tests get object by range.", - Description: `Creates a bucket with s3 gateway action, puts an object in it, - gets the object by specifying the object range, compares the range with the original one, - removes both the object and the bucket`, - Action: getAction(integration.TestRangeGet), - }, - { - Name: "invalid-mp", - Usage: "Tests invalid multi part case.", - Description: `Creates a bucket with s3 gateway action, creates a multi part upload, - uploads an invalid part, gets the object with head-object action, expects to get error, - removes both the object and bucket`, - Action: getAction(integration.TestInvalidMultiParts), - }, - { - Name: "object-tag-actions", - Usage: "Tests get/put/delete object tag actions.", - Description: `Creates a bucket with s3 gateway action, puts an object in it, - puts some tags for the object, gets the tags, compares the results, removes the tags, - gets the tags again, checks it to be empty, then removes both the object and bucket`, - Action: getAction(integration.TestPutGetRemoveTags), - }, - { - Name: "bucket-acl-actions", - Usage: "Tests put/get bucket actions.", - Description: `Creates a bucket with s3 gateway action, puts some bucket acls - gets the acl, verifies it, then removes the bucket`, - Action: getAction(integration.TestAclActions), - }, { Name: "full-flow", Usage: "Tests the full flow of gateway.", diff --git a/integration/action-tests.go b/integration/action-tests.go new file mode 100644 index 00000000..0cb7699e --- /dev/null +++ b/integration/action-tests.go @@ -0,0 +1,170 @@ +package integration + +func TestCreateBucket(s *S3Conf) { + CreateBucket_invalid_bucket_name(s) + CreateBucket_existing_bucket(s) + CreateDeleteBucket_success(s) +} + +func TestHeadBucket(s *S3Conf) { + HeadBucket_non_existing_bucket(s) + HeadBucket_success(s) +} + +func TestDeleteBucket(s *S3Conf) { + DeleteBucket_non_existing_bucket(s) + DeleteBucket_non_empty_bucket(s) +} + +func TestPutObject(s *S3Conf) { + PutObject_non_existing_bucket(s) + PutObject_special_chars(s) + PutObject_existing_dir_obj(s) + PutObject_obj_parent_is_file(s) + PutObject_success(s) +} + +func TestHeadObject(s *S3Conf) { + HeadObject_non_existing_object(s) + HeadObject_success(s) +} + +func TestGetObject(s *S3Conf) { + GetObject_non_existing_key(s) + GetObject_invalid_ranges(s) + GetObject_with_meta(s) + GetObject_success(s) + GetObject_by_range_success(s) +} + +func TestListObjects(s *S3Conf) { + ListObjects_non_existing_bucket(s) + ListObjects_with_prefix(s) + ListObject_truncated(s) + ListObjects_invalid_max_keys(s) + ListObjects_max_keys_0(s) +} + +func TestDeleteObject(s *S3Conf) { + DeleteObject_non_existing_object(s) + DeleteObject_success(s) +} + +func TestDeleteObjects(s *S3Conf) { + DeleteObjects_empty_input(s) + //TODO: Uncomment this after fixing the bug: #195 + // DeleteObjects_non_existing_objects(s) + DeleteObjects_success(s) +} + +func TestCopyObject(s *S3Conf) { + CopyObject_non_existing_dst_bucket(s) + CopyObject_success(s) +} + +func TestPutObjectTagging(s *S3Conf) { + PutObjectTagging_non_existing_object(s) + PutObjectTagging_success(s) +} + +func TestGetObjectTagging(s *S3Conf) { + GetObjectTagging_non_existing_object(s) + GetObjectTagging_success(s) +} + +func TestDeleteObjectTagging(s *S3Conf) { + DeleteObjectTagging_non_existing_object(s) + DeleteObjectTagging_success(s) +} + +func TestCreateMultipartUpload(s *S3Conf) { + CreateMultipartUpload_non_existing_bucket(s) + CreateMultipartUpload_success(s) +} + +func TestUploadPart(s *S3Conf) { + UploadPart_non_existing_bucket(s) + UploadPart_invalid_part_number(s) + UploadPart_non_existing_key(s) + UploadPart_non_existing_mp_upload(s) + UploadPart_success(s) +} + +func TestUploadPartCopy(s *S3Conf) { + UploadPartCopy_non_existing_bucket(s) + UploadPartCopy_incorrect_uploadId(s) + UploadPartCopy_incorrect_object_key(s) + UploadPartCopy_invalid_part_number(s) + UploadPartCopy_invalid_copy_source(s) + UploadPartCopy_non_existing_source_bucket(s) + UploadPartCopy_non_existing_source_object_key(s) + UploadPartCopy_success(s) + UploadPartCopy_by_range_invalid_range(s) + UploadPartCopy_by_range_success(s) +} + +func TestListParts(s *S3Conf) { + ListParts_incorrect_uploadId(s) + ListParts_incorrect_object_key(s) + ListParts_success(s) +} + +func TestListMultipartUploads(s *S3Conf) { + ListMultipartUploads_non_existing_bucket(s) + ListMultipartUploads_empty_result(s) + ListMultipartUploads_success(s) +} + +func TestAbortMultipartUpload(s *S3Conf) { + AbortMultipartUpload_non_existing_bucket(s) + AbortMultipartUpload_incorrect_uploadId(s) + AbortMultipartUpload_incorrect_object_key(s) + AbortMultipartUpload_success(s) +} + +func TestCompleteMultipartUpload(s *S3Conf) { + CompletedMultipartUpload_non_existing_bucket(s) + CompleteMultipartUpload_invalid_part_number(s) + CompleteMultipartUpload_invalid_ETag(s) + CompleteMultipartUpload_success(s) +} + +func TestPutBucketAcl(s *S3Conf) { + PutBucketAcl_non_existing_bucket(s) + PutBucketAcl_invalid_acl_canned_and_acp(s) + PutBucketAcl_invalid_acl_canned_and_grants(s) + PutBucketAcl_invalid_acl_acp_and_grants(s) + PutBucketAcl_invalid_owner(s) + PutBucketAcl_success_access_denied(s) + PutBucketAcl_success(s) +} + +func TestGetBucketAcl(s *S3Conf) { + GetBucketAcl_non_existing_bucket(s) + GetBucketAcl_access_denied(s) + GetBucketAcl_success(s) +} + +func TestFullFlow(s *S3Conf) { + TestCreateBucket(s) + TestHeadBucket(s) + TestDeleteBucket(s) + TestPutObject(s) + TestHeadObject(s) + TestGetObject(s) + TestListObjects(s) + TestDeleteObject(s) + TestDeleteObjects(s) + TestCopyObject(s) + TestPutObjectTagging(s) + TestDeleteObjectTagging(s) + TestCreateMultipartUpload(s) + TestUploadPart(s) + TestUploadPartCopy(s) + TestListParts(s) + TestListMultipartUploads(s) + TestAbortMultipartUpload(s) + TestCompleteMultipartUpload(s) + TestPutBucketAcl(s) + TestGetBucketAcl(s) +} diff --git a/integration/tests.go b/integration/tests.go index aed74d99..5bb23f9c 100644 --- a/integration/tests.go +++ b/integration/tests.go @@ -1,1293 +1,2430 @@ package integration import ( - "bytes" "context" - "crypto/rand" "crypto/sha256" + "errors" "fmt" "io" "math" - "os" - "strings" "sync" "time" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/google/uuid" + "github.com/versity/versitygw/s3err" ) var ( shortTimeout = 10 * time.Second ) -func TestMakeBucket(s *S3Conf) { - testname := "test make/head/delete bucket" - runF(testname) - - s3client := s3.NewFromConfig(s.Config()) - - invBucket := "aa" - err := setup(s, invBucket) - if err == nil { - failF("%v: expected bucket name validation error", testname) +func CreateBucket_invalid_bucket_name(s *S3Conf) { + testName := "CreateBucket_invalid_bucket_name" + runF(testName) + err := setup(s, "aa") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketName)); err != nil { + failF("%v: %v", testName, err.Error()) return } - bucket := "testbucket" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket}) - cancel() - if err == nil { - failF("%v: expected error, instead got success response", testname) + err = setup(s, ".gitignore") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketName)); err != nil { + failF("%v: %v", testName, err.Error()) return } + err = setup(s, "my-bucket.") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketName)); err != nil { + failF("%v: %v", testName, err.Error()) + return + } + + err = setup(s, "bucket-%") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketName)); err != nil { + failF("%v: %v", testName, err.Error()) + return + } + passF(testName) +} + +func CreateBucket_existing_bucket(s *S3Conf) { + testName := "CreateBucket_existing_bucket" + runF(testName) + bucket := getBucketName() + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testName, err.Error()) + return + } err = setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket}) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return + var bne *types.BucketAlreadyExists + if !errors.As(err, &bne) { + failF("%v: %v", testName, err.Error()) } err = teardown(s, bucket) if err != nil { - failF("%v: %v", testname, err) + failF("%v: %v", err.Error()) return } - passF(testname) + passF(testName) } -func TestPutGetObject(s *S3Conf) { - testname := "test put/get/delete/copy objects" - runF(testname) +func HeadBucket_non_existing_bucket(s *S3Conf) { + testName := "HeadBucket_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bcktName := getBucketName() - bucket := "testbucket1" - dstBucket := "testdstbucket" - obj := "myobject" - obj2 := "myobject2" - obj3 := "myobject%%3" - copySource := bucket + "/" + obj - - s3client := s3.NewFromConfig(s.Config()) - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - // use funny size to prevent accidental alignments - datalen := 1234567 - data := make([]byte, datalen) - rand.Read(data) - csum := sha256.Sum256(data) - r := bytes.NewReader(data) - meta := map[string]string{ - "key1": "val1", - "key2": "val2", - } - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - Body: r, - Metadata: meta, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj3, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj2, - Body: r, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj3, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - if !areMapsSame(out.Metadata, meta) { - failF("%v: incorrect object metadata", testname) - return - } - defer out.Body.Close() - - if out.ContentLength != int64(datalen) { - failF("%v: content length got %v expected %v", testname, out.ContentLength, datalen) - return - } - - b, err := io.ReadAll(out.Body) - if err != nil { - failF("%v: read body %v", testname, err) - return - } - - newsum := sha256.Sum256(b) - if csum != newsum { - failF("%v: checksum got %x expected %x", testname, newsum, csum) - return - } - - // Expected error: destination bucket doesn't exist - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: &dstBucket, Key: &obj, CopySource: ©Source}) - cancel() - if err == nil { - failF("%v: expect bucket not found error instead got success response", testname) - return - } - - err = setup(s, dstBucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: &dstBucket, Key: &obj, CopySource: ©Source}) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - copyObjOut, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &dstBucket, - Key: &obj, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - defer copyObjOut.Body.Close() - - if copyObjOut.ContentLength != int64(datalen) { - failF("%v: content length got %v expected %v", testname, copyObjOut.ContentLength, datalen) - return - } - - b, err = io.ReadAll(copyObjOut.Body) - if err != nil { - failF("%v: read body %v", testname, err) - return - } - - copysum := sha256.Sum256(b) - if csum != copysum { - failF("%v: copied object checksum got %x expected %x", testname, copysum, csum) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{Bucket: &bucket, Delete: &types.Delete{Objects: []types.ObjectIdentifier{{Key: &obj}, {Key: &obj2}, {Key: &obj3}}}}) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - objCount := 0 - - in := &s3.ListObjectsV2Input{Bucket: &bucket} - for { ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectsV2(ctx, in) + _, err := s3client.HeadBucket(ctx, &s3.HeadBucketInput{ + Bucket: &bcktName, + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + return nil + }) +} + +func HeadBucket_success(s *S3Conf) { + testName := "HeadBucket_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.HeadBucket(ctx, &s3.HeadBucketInput{ + Bucket: &bucket, + }) cancel() if err != nil { - failF("%v: %v", testname, err) - return + return err } - objCount += len(out.Contents) - if out.IsTruncated { - in.ContinuationToken = out.ContinuationToken - } else { - break - } - } + return nil + }) +} - if objCount != 0 { - failF("%v: expected object count %v instead got %v", testname, 0, objCount) +func CreateDeleteBucket_success(s *S3Conf) { + testName := "CreateBucket_success" + runF(testName) + bucket := getBucketName() + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", err.Error()) return } err = teardown(s, bucket) if err != nil { - failF("%v: %v", testname, err) + failF("%v: %v", err.Error()) return } - err = teardown(s, dstBucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - passF(testname) + passF(testName) } -func TestPutGetMPObject(s *S3Conf) { - testname := "test put/get multipart object" - runF(testname) - - bucket := "testbucket2" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - name := "mympuobject" +func DeleteBucket_non_existing_bucket(s *S3Conf) { + testName := "DeleteBucket_non_existing_bucket" + runF(testName) + bucket := getBucketName() s3client := s3.NewFromConfig(s.Config()) - datalen := 10*1024*1024 + 15 - dr := NewDataReader(datalen, 5*1024*1024) - WithPartSize(5 * 1024 * 1024) - s.PartSize = 5 * 1024 * 1024 - err = s.UploadData(dr, bucket, name) - if err != nil { - failF("%v: %v", testname, err) - return - } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + _, err := s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ Bucket: &bucket, - Key: &name, }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + failF("%v: %v", testName, err.Error()) return } - defer out.Body.Close() + passF(testName) +} - if out.ContentLength != int64(datalen) { - failF("%v: content length got %v expected %v", testname, out.ContentLength, datalen) - return - } - - b := make([]byte, 1048576) - h := sha256.New() - for { - n, err := out.Body.Read(b) - if err == io.EOF { - h.Write(b[:n]) - break - } +func DeleteBucket_non_empty_bucket(s *S3Conf) { + testName := "DeleteBucket_non_empty_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"foo"}, bucket) if err != nil { - failF("%v: read %v", err) - return + return err } - h.Write(b[:n]) - } - - if !isEqual(dr.Sum(), h.Sum(nil)) { - failF("%v: checksum got %x expected %x", testname, h.Sum(nil), dr.Sum()) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -func TestPutDirObject(s *S3Conf) { - testname := "test put directory object" - runF(testname) - - bucket := "testbucket3" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - name := "myobjectdir/" - s3client := s3.NewFromConfig(s.Config()) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &name, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket, MaxKeys: -4}) - cancel() - if err == nil { - failF("%v: expected invalid argument error", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) - cancel() - if err != nil { - failF("failed to list objects: %v", err) - return - } - - if !contains(name, out.Contents) { - failF("directory object not found") - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -func TestListObject(s *S3Conf) { - testname := "list objects" - runF(testname) - - bucket := "testbucket4" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - dir1 := "myobjectdir/" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &dir1, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - obj1 := "myobjectdir/myobject" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj1, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - obj2 := "myobjectdir1/myobject" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj2, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - // put: - // "myobjectdir/" - // "myobjectdir/myobject" - // "myobjectdir1/myobject" - // should return: - // "myobjectdir/myobject" - // "myobjectdir1/myobject" - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) - cancel() - if err != nil { - failF("failed to list objects: %v", err) - return - } - - if !contains(obj1, out.Contents) { - failF("object %v not found", obj1) - return - } - if !contains(obj2, out.Contents) { - failF("object %v not found", obj2) - return - } - if out.KeyCount != 2 { - failF("%v: expected key count: %v, instead got: %v", testname, 2, out.KeyCount) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj1, - }) - cancel() - if err != nil { - failF("failed to delete %v: %v", obj1, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj2, - }) - cancel() - if err != nil { - failF("failed to delete %v: %v", obj2, err) - return - } - - // put: - // "myobjectdir/" - // "myobjectdir/myobject" - // "myobjectdir1/myobject" - // delete: - // "myobjectdir/myobject" - // "myobjectdir1/myobject" - // should return: - // "myobjectdir/" - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) - cancel() - if err != nil { - failF("failed to list objects: %v", err) - return - } - - if !contains(dir1, out.Contents) { - failF("dir %v not found", dir1) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -func TestListAbortMultiPartObject(s *S3Conf) { - testname := "list/abort multipart objects" - runF(testname) - - bucket := "testbucket6" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - obj := "mympuobject" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lmpu, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: list multipart upload: %v", testname, err) - return - } - - //for _, item := range lmpu.Uploads { - // fmt.Println(" -- ", *item.Key, *item.UploadId) - //} - - if !containsUID(obj, *mpu.UploadId, lmpu.Uploads) { - failF("%v: upload %v/%v not found", testname, obj, *mpu.UploadId) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err != nil { - failF("%v: abort multipart upload: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lmpu, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: list multipart upload: %v", testname, err) - return - } - - if len(lmpu.Uploads) != 0 { - for _, item := range lmpu.Uploads { - fmt.Println(" D- ", *item.Key, *item.UploadId) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: &bucket, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrBucketNotEmpty)); err != nil { + return err } - failF("%v: unexpected multipart uploads found", testname) - return - } - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) + return nil + }) } -func TestListMultiParts(s *S3Conf) { - testname := "list multipart parts" - runF(testname) - - bucket := "testbucket7" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - obj := "mympuobject" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, +func PutObject_non_existing_bucket(s *S3Conf) { + testName := "PutObject_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"my-obj"}, "non-existing-bucket") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + return nil }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } - - // check list parts of no parts is good - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err != nil { - failF("%v: list parts: %v", testname, err) - return - } - - if len(lp.Parts) != 0 { - failF("%v: list parts: expected no parts, got %v", - testname, len(lp.Parts)) - return - } - - // upload 1 part and check list parts - size5MB := 5 * 1024 * 1024 - dr := NewDataReader(size5MB, size5MB) - - datafile := "rand.data" - w, err := os.Create(datafile) - if err != nil { - failF("%v: create %v: %v", testname, datafile, err) - return - } - defer w.Close() - - _, err = io.Copy(w, dr) - if err != nil { - failF("%v: write %v: %v", testname, datafile, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: 42, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), - }) - cancel() - if err != nil { - failF("%v: multipart put part: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lp, err = s3client.ListParts(ctx, &s3.ListPartsInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err != nil { - failF("%v: list parts: %v", testname, err) - return - } - - //for _, part := range lp.Parts { - // fmt.Println(" -- ", part.PartNumber, part.ETag) - //} - - if len(lp.Parts) != 1 || lp.Parts[0].PartNumber != 42 { - fmt.Printf("%+v, %v, %v\n", lp.Parts, *lp.Key, *lp.UploadId) - failF("%v: list parts: unexpected parts listing", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestIncorrectMultiParts(s *S3Conf) { - testname := "incorrect multipart parts" - runF(testname) - - bucket := "testbucket8" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - obj := "mympuobject" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, +func PutObject_special_chars(s *S3Conf) { + testName := "PutObject_special_chars" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"foo%%", "bar^", "baz**"}, bucket) + if err != nil { + return err + } + return nil }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } +} - // upload 2 parts - size5MB := 5 * 1024 * 1024 - dr := NewDataReader(size5MB, size5MB) - - datafile := "rand.data" - w, err := os.Create(datafile) - if err != nil { - failF("%v: create %v: %v", testname, datafile, err) - return - } - defer w.Close() - - _, err = io.Copy(w, dr) - if err != nil { - failF("%v: write %v: %v", testname, datafile, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - mp1, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: 42, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), +func PutObject_existing_dir_obj(s *S3Conf) { + testName := "PutObject_existing_dir_obj" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"foo/bar", "foo"}, bucket) + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)); err != nil { + return err + } + return nil }) - cancel() - if err != nil { - failF("%v: multipart put part 1: %v", testname, err) - return - } +} - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - mp2, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: 96, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), +func PutObject_obj_parent_is_file(s *S3Conf) { + testName := "PutObject_obj_parent_is_file" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"foo", "foo/bar/"}, bucket) + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectParentIsFile)); err != nil { + return err + } + return nil }) - cancel() - if err != nil { - failF("%v: multipart put part 2: %v", testname, err) - return - } +} - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - MultipartUpload: &types.CompletedMultipartUpload{ - Parts: []types.CompletedPart{ - { - ETag: mp2.ETag, - PartNumber: 96, - }, - { - ETag: mp1.ETag, - PartNumber: 99, +func PutObject_success(s *S3Conf) { + testName := "PutObject_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"my-obj"}, bucket) + if err != nil { + return err + } + return nil + }) +} + +func HeadObject_non_existing_object(s *S3Conf) { + testName := "HeadObject_non_existing_object" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + return nil + }) +} + +func HeadObject_success(s *S3Conf) { + testName := "HeadObject_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, dataLen := "my-obj", int64(1234567) + meta := map[string]string{ + "key1": "val1", + "key2": "val2", + } + + _, _, err := putObjectWithData(dataLen, &s3.PutObjectInput{Bucket: &bucket, Key: &obj, Metadata: meta}, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + + if !areMapsSame(out.Metadata, meta) { + return fmt.Errorf("incorrect object metadata") + } + if out.ContentLength != dataLen { + return fmt.Errorf("expected data length %v, instead got %v", dataLen, out.ContentLength) + } + + return nil + }) +} + +func GetObject_non_existing_key(s *S3Conf) { + testName := "GetObject_non_existing_key" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: getPtr("non-existing-key"), + }) + cancel() + var bae *types.NoSuchKey + if !errors.As(err, &bae) { + return err + } + return nil + }) +} + +func GetObject_invalid_ranges(s *S3Conf) { + testName := "GetObject_invalid_ranges" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + + _, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: getPtr("bytes=invalid-range"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: getPtr("bytes=33-10"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: getPtr("bytes=1000000000-999999999999"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { + return err + } + return nil + }) +} + +func GetObject_with_meta(s *S3Conf) { + testName := "GetObject_with_meta" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + meta := map[string]string{ + "key1": "val1", + "key2": "val2", + } + + _, _, err := putObjectWithData(0, &s3.PutObjectInput{Bucket: &bucket, Key: &obj, Metadata: meta}, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + + if !areMapsSame(out.Metadata, meta) { + return fmt.Errorf("incorrect object metadata") + } + + return nil + }) +} + +func GetObject_success(s *S3Conf) { + testName := "GetObject_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + + csum, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + if out.ContentLength != dataLength { + return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) + } + + bdy, err := io.ReadAll(out.Body) + if err != nil { + return err + } + defer out.Body.Close() + outCsum := sha256.Sum256(bdy) + if outCsum != csum { + return fmt.Errorf("invalid object data") + } + return nil + }) +} + +func GetObject_by_range_success(s *S3Conf) { + testName := "GetObject_by_range_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + + _, data, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + rangeString := "bytes=100-200" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: &rangeString, + }) + defer cancel() + if err != nil { + return err + } + defer out.Body.Close() + + if getString(out.ContentRange) != fmt.Sprintf("bytes 100-200/%v", dataLength) { + return fmt.Errorf("expected content range: %v, instead got: %v", fmt.Sprintf("bytes 100-200/%v", dataLength), getString(out.ContentRange)) + } + if getString(out.AcceptRanges) != rangeString { + return fmt.Errorf("expected accept range: %v, instead got: %v", rangeString, getString(out.AcceptRanges)) + } + b, err := io.ReadAll(out.Body) + if err != nil { + return err + } + + // bytes range is inclusive, go range for second value is not + if !isEqual(b, data[100:201]) { + return fmt.Errorf("data mismatch of range") + } + + rangeString = "bytes=100-" + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: &rangeString, + }) + defer cancel() + if err != nil { + return err + } + defer out.Body.Close() + + b, err = io.ReadAll(out.Body) + if err != nil { + return err + } + + // bytes range is inclusive, go range for second value is not + if !isEqual(b, data[100:]) { + return fmt.Errorf("data mismatch of range") + } + return nil + }) +} + +func ListObjects_non_existing_bucket(s *S3Conf) { + testName := "ListObjects_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bckt := getBucketName() + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bckt, + }) + cancel() + if err := checkSdkApiErr(err, "NoSuchBucket"); err != nil { + return err + } + return nil + }) +} + +func ListObjects_with_prefix(s *S3Conf) { + testName := "ListObjects_with_prefix" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + prefix := "obj" + objWithPrefix := []string{prefix + "/foo", prefix + "/bar", prefix + "/baz/bla"} + err := putObjects(s3client, append(objWithPrefix, []string{"xzy/csf", "hell"}...), bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + Prefix: &prefix, + }) + cancel() + if err != nil { + return err + } + + if *out.Prefix != prefix { + return fmt.Errorf("expected prefix %v, instead got %v", prefix, *out.Prefix) + } + if !compareObjects(objWithPrefix, out.Contents) { + return fmt.Errorf("unexpected output for list objects with prefix") + } + + return nil + }) +} + +func ListObject_truncated(s *S3Conf) { + testName := "ListObject_truncated" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + maxKeys := int32(2) + err := putObjects(s3client, []string{"foo", "bar", "baz"}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + MaxKeys: maxKeys, + }) + cancel() + if err != nil { + return err + } + + if !out.IsTruncated { + return fmt.Errorf("expected output to be truncated") + } + + if out.MaxKeys != maxKeys { + return fmt.Errorf("expected max-keys to be %v, instead got %v", maxKeys, out.MaxKeys) + } + + if !compareObjects([]string{"bar", "baz"}, out.Contents) { + return fmt.Errorf("unexpected output for list objects with max-keys") + } + + //TODO: Add next marker checker after bug-fixing + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err = s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + Marker: out.NextMarker, + }) + cancel() + if err != nil { + return err + } + + if out.IsTruncated { + return fmt.Errorf("expected output not to be truncated") + } + + if !compareObjects([]string{"foo"}, out.Contents) { + return fmt.Errorf("unexpected output for list objects with max-keys") + } + return nil + }) +} + +func ListObjects_invalid_max_keys(s *S3Conf) { + testName := "ListObjects_invalid_max_keys" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + MaxKeys: -5, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)); err != nil { + return err + } + + return nil + }) +} + +func ListObjects_max_keys_0(s *S3Conf) { + testName := "ListObjects_max_keys_0" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objects := []string{"foo", "bar", "baz"} + err := putObjects(s3client, objects, bucket) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + MaxKeys: 0, + }) + cancel() + if err != nil { + return nil + } + + if !compareObjects(objects, out.Contents) { + return fmt.Errorf("unexpected output for list objects with max-keys 0") + } + + return nil + }) +} + +//TODO: Add a test case for delimiter after bug-fixing, as delimiter doesn't work as intended + +func DeleteObject_non_existing_object(s *S3Conf) { + testName := "DeleteObject_non_existing_object" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + return nil + }) +} + +func DeleteObject_success(s *S3Conf) { + testName := "DeleteObject_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err := checkSdkApiErr(err, "NoSuchKey"); err != nil { + return err + } + return nil + }) +} + +func DeleteObjects_empty_input(s *S3Conf) { + testName := "DeleteObjects_empty_input" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objects := []string{"foo", "bar", "baz"} + err := putObjects(s3client, objects, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: []types.ObjectIdentifier{}, + }, + }) + cancel() + if err != nil { + return err + } + + if len(out.Deleted) != 0 { + return fmt.Errorf("expected deleted object count 0, instead got %v", len(out.Deleted)) + } + if len(out.Errors) != 0 { + return fmt.Errorf("expected 0 errors, instead got %v", len(out.Errors)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(objects, res.Contents) { + return fmt.Errorf("unexpected output for list objects with prefix") + } + + return nil + }) +} + +//TODO: Uncomment the test after fixing the bug: #195 +// func DeleteObjects_non_existing_objects(s *S3Conf) { +// testName := "DeleteObjects_empty_input" +// actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { +// delObjects := []types.ObjectIdentifier{{Key: getPtr("obj1")}, {Key: getPtr("obj2")}} +// +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ +// Bucket: &bucket, +// Delete: &types.Delete{ +// Objects: delObjects, +// }, +// }) +// cancel() +// if err != nil { +// return err +// } + +// if len(out.Deleted) != 0 { +// return fmt.Errorf("expected deleted object count 0, instead got %v", len(out.Deleted)) +// } +// if len(out.Errors) != 2 { +// return fmt.Errorf("expected 2 errors, instead got %v", len(out.Errors)) +// } + +// for _, delErr := range out.Errors { +// if *delErr.Code != "NoSuchKey" { +// return fmt.Errorf("expected NoSuchKey error, instead got %v", *delErr.Code) +// } +// } + +// return nil +// }) +// } + +func DeleteObjects_success(s *S3Conf) { + testName := "DeleteObjects_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objects, objToDel := []string{"obj1", "obj2", "obj3"}, []string{"foo", "bar", "baz"} + err := putObjects(s3client, append(objToDel, objects...), bucket) + if err != nil { + return err + } + + delObjects := []types.ObjectIdentifier{} + for _, key := range objToDel { + k := key + delObjects = append(delObjects, types.ObjectIdentifier{Key: &k}) + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: delObjects, + }, + }) + cancel() + if err != nil { + return err + } + + if len(out.Deleted) != 3 { + return fmt.Errorf("expected deleted object count 3, instead got %v", len(out.Deleted)) + } + if len(out.Errors) != 0 { + return fmt.Errorf("expected 2 errors, instead got %v", len(out.Errors)) + } + + if !compareDelObjects(objToDel, out.Deleted) { + return fmt.Errorf("unexpected deleted output") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(objects, res.Contents) { + return fmt.Errorf("unexpected output for list objects with prefix") + } + + return nil + }) +} + +func CopyObject_non_existing_dst_bucket(s *S3Conf) { + testName := "CopyObject_non_existing_dst_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr("bucket/obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + return nil + }) +} + +func CopyObject_success(s *S3Conf) { + testName := "CopyObject_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + dstBucket := getBucketName() + err := setup(s, dstBucket) + if err != nil { + return err + } + + csum, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &dstBucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, obj)), + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &dstBucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + if out.ContentLength != dataLength { + return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) + } + + bdy, err := io.ReadAll(out.Body) + if err != nil { + return err + } + defer out.Body.Close() + outCsum := sha256.Sum256(bdy) + if outCsum != csum { + return fmt.Errorf("invalid object data") + } + + err = teardown(s, dstBucket) + if err != nil { + return nil + } + + return nil + }) +} + +func PutObjectTagging_non_existing_object(s *S3Conf) { + testName := "PutObjectTagging_non_existing_object" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + Tagging: &types.Tagging{TagSet: []types.Tag{}}}) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + return nil + }) +} + +func PutObjectTagging_success(s *S3Conf) { + testName := "PutObjectTagging_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + tagging := types.Tagging{TagSet: []types.Tag{{Key: getPtr("key1"), Value: getPtr("val2")}, {Key: getPtr("key2"), Value: getPtr("val2")}}} + err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + Tagging: &tagging}) + cancel() + if err != nil { + return err + } + + return nil + }) +} + +func GetObjectTagging_non_existing_object(s *S3Conf) { + testName := "GetObjectTagging_non_existing_object" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + return nil + }) +} + +func GetObjectTagging_success(s *S3Conf) { + testName := "PutObjectTagging_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + tagging := types.Tagging{TagSet: []types.Tag{{Key: getPtr("key1"), Value: getPtr("val2")}, {Key: getPtr("key2"), Value: getPtr("val2")}}} + err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + Tagging: &tagging}) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return nil + } + + if !areTagsSame(out.TagSet, tagging.TagSet) { + return fmt.Errorf("expected %v instead got %v", tagging.TagSet, out.TagSet) + } + + return nil + }) +} + +func DeleteObjectTagging_non_existing_object(s *S3Conf) { + testName := "DeleteObjectTagging_non_existing_object" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + return nil + }) +} + +func DeleteObjectTagging_success(s *S3Conf) { + testName := "DeleteObjectTagging_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + tagging := types.Tagging{TagSet: []types.Tag{{Key: getPtr("key1"), Value: getPtr("val2")}, {Key: getPtr("key2"), Value: getPtr("val2")}}} + err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + Tagging: &tagging}) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return nil + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return nil + } + + if len(out.TagSet) > 0 { + return fmt.Errorf("expected empty tag set, instead got %v", out.TagSet) + } + + return nil + }) +} + +func CreateMultipartUpload_non_existing_bucket(s *S3Conf) { + testName := "CreateMultipartUpload_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bucketName := getBucketName() + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucketName, + Key: getPtr("my-obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func CreateMultipartUpload_success(s *S3Conf) { + testName := "CreateMultipartUpload_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if *out.Bucket != bucket { + return fmt.Errorf("expected bucket name %v, instead got %v", bucket, *out.Bucket) + } + if *out.Key != obj { + return fmt.Errorf("expected object name %v, instead got %v", obj, *out.Key) + } + if _, err := uuid.Parse(*out.UploadId); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_non_existing_bucket(s *S3Conf) { + testName := "UploadPart_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bucketName := getBucketName() + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucketName, + Key: getPtr("my-obj"), + UploadId: getPtr("uploadId"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_invalid_part_number(s *S3Conf) { + testName := "UploadPart_invalid_part_number" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + UploadId: getPtr("uploadId"), + PartNumber: -10, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { + return err + } + return nil + }) +} + +func UploadPart_non_existing_mp_upload(s *S3Conf) { + testName := "UploadPart_non_existing_mp_upload" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + UploadId: getPtr("uploadId"), + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + return nil + }) +} + +func UploadPart_non_existing_key(s *S3Conf) { + testName := "UploadPart_non_existing_key" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: getPtr("non-existing-object-key"), + UploadId: out.UploadId, + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + return nil + }) +} + +func UploadPart_success(s *S3Conf) { + testName := "UploadPart_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + PartNumber: 1, + }) + cancel() + if err != nil { + return err + } + if *res.ETag == "" { + return fmt.Errorf("expected a valid etag, instead got empty") + } + return nil + }) +} + +func UploadPartCopy_non_existing_bucket(s *S3Conf) { + testName := "UploadPartCopy_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bucketName := getBucketName() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucketName, + CopySource: getPtr("Copy-Source"), + UploadId: getPtr("uploadId"), + Key: getPtr("my-obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + return nil + }) +} + +func UploadPartCopy_incorrect_uploadId(s *S3Conf) { + testName := "UploadPartCopy_incorrect_uploadId" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, srcBucket, srcObj := "my-obj", getBucketName(), "src-obj" + err := setup(s, srcBucket) + if err != nil { + return err + } + err = putObjects(s3client, []string{srcObj}, srcBucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr(srcBucket + "/" + srcObj), + UploadId: getPtr("incorrect-upload-id"), + Key: &obj, + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + + err = teardown(s, srcBucket) + if err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_incorrect_object_key(s *S3Conf) { + testName := "UploadPartCopy_incorrect_object_key" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, srcBucket, srcObj := "my-obj", getBucketName(), "src-obj" + err := setup(s, srcBucket) + if err != nil { + return err + } + err = putObjects(s3client, []string{srcObj}, srcBucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr(srcBucket + "/" + srcObj), + UploadId: out.UploadId, + Key: getPtr("non-existing-object-key"), + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + + err = teardown(s, srcBucket) + if err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_invalid_part_number(s *S3Conf) { + testName := "UploadPartCopy_invalid_part_number" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr("Copy-Source"), + UploadId: getPtr("uploadId"), + Key: getPtr("non-existing-object-key"), + PartNumber: -10, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_invalid_copy_source(s *S3Conf) { + testName := "UploadPartCopy_invalid_copy_source" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr("invalid-copy-source"), + UploadId: out.UploadId, + Key: &obj, + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidCopySource)); err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_non_existing_source_bucket(s *S3Conf) { + testName := "UploadPartCopy_non_existing_source_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr("src/bucket/src/obj"), + UploadId: out.UploadId, + Key: &obj, + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_non_existing_source_object_key(s *S3Conf) { + testName := "UploadPartCopy_non_existing_source_object_key" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, srcBucket := "my-obj", getBucketName() + + err := setup(s, srcBucket) + if err != nil { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr(srcBucket + "/non/existing/obj/key"), + UploadId: out.UploadId, + Key: &obj, + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + + err = teardown(s, srcBucket) + if err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_success(s *S3Conf) { + testName := "UploadPartCopy_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, srcBucket, srcObj := "my-obj", getBucketName(), "src-obj" + err := setup(s, srcBucket) + if err != nil { + return err + } + objSize := 5 * 1024 * 1024 + _, _, err = putObjectWithData(int64(objSize), &s3.PutObjectInput{ + Bucket: &srcBucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr(srcBucket + "/" + srcObj), + UploadId: out.UploadId, + Key: &obj, + PartNumber: 1, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + }) + cancel() + if err != nil { + return err + } + + if len(res.Parts) != 1 { + return fmt.Errorf("expected parts to be 1, instead got %v", len(res.Parts)) + } + if res.Parts[0].PartNumber != 1 { + return fmt.Errorf("expected part-number to be 1, instead got %v", res.Parts[0].PartNumber) + } + if res.Parts[0].Size != int64(objSize) { + return fmt.Errorf("expected part size to be %v, instead got %v", objSize, res.Parts[0].Size) + } + if *res.Parts[0].ETag != *copyOut.CopyPartResult.ETag { + return fmt.Errorf("expected part etag to be %v, instead got %v", *copyOut.CopyPartResult.ETag, *res.Parts[0].ETag) + } + + err = teardown(s, srcBucket) + if err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_by_range_invalid_range(s *S3Conf) { + testName := "UploadPartCopy_by_range_invalid_range" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, srcBucket, srcObj := "my-obj", getBucketName(), "src-obj" + err := setup(s, srcBucket) + if err != nil { + return err + } + objSize := 5 * 1024 * 1024 + _, _, err = putObjectWithData(int64(objSize), &s3.PutObjectInput{ + Bucket: &srcBucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr(srcBucket + "/" + srcObj), + UploadId: out.UploadId, + Key: &obj, + PartNumber: 1, + CopySourceRange: getPtr("invalid-range"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { + return err + } + + err = teardown(s, srcBucket) + if err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_by_range_success(s *S3Conf) { + testName := "UploadPartCopy_by_range_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, srcBucket, srcObj := "my-obj", getBucketName(), "src-obj" + err := setup(s, srcBucket) + if err != nil { + return err + } + objSize := 5 * 1024 * 1024 + _, _, err = putObjectWithData(int64(objSize), &s3.PutObjectInput{ + Bucket: &srcBucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr(srcBucket + "/" + srcObj), + CopySourceRange: getPtr("bytes=100-200"), + UploadId: out.UploadId, + Key: &obj, + PartNumber: 1, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + }) + cancel() + if err != nil { + return err + } + + if len(res.Parts) != 1 { + return fmt.Errorf("expected parts to be 1, instead got %v", len(res.Parts)) + } + if res.Parts[0].PartNumber != 1 { + return fmt.Errorf("expected part-number to be 1, instead got %v", res.Parts[0].PartNumber) + } + if res.Parts[0].Size != 101 { + return fmt.Errorf("expected part size to be %v, instead got %v", 101, res.Parts[0].Size) + } + if *res.Parts[0].ETag != *copyOut.CopyPartResult.ETag { + return fmt.Errorf("expected part etag to be %v, instead got %v", *copyOut.CopyPartResult.ETag, *res.Parts[0].ETag) + } + + err = teardown(s, srcBucket) + if err != nil { + return err + } + + return nil + }) +} + +func ListParts_incorrect_uploadId(s *S3Conf) { + testName := "ListParts_incorrect_uploadId" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + UploadId: getPtr("invalid uploadId"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + + return nil + }) +} + +func ListParts_incorrect_object_key(s *S3Conf) { + testName := "ListParts_incorrect_object_key" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: getPtr("incorrect-object-key"), + UploadId: out.UploadId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + + return nil + }) +} + +func ListParts_success(s *S3Conf) { + testName := "ListParts_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + parts, err := uploadParts(s3client, 5*1024*1024, 5, bucket, obj, *out.UploadId) + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + }) + cancel() + if err != nil { + return err + } + + if ok := compareParts(parts, res.Parts); !ok { + return fmt.Errorf("expected parts %+v, instead got %+v", parts, res.Parts) + } + + return nil + }) +} + +func ListMultipartUploads_non_existing_bucket(s *S3Conf) { + testName := "ListMultipartUploads_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bucketName := getBucketName() + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucketName, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func ListMultipartUploads_empty_result(s *S3Conf) { + testName := "ListMultipartUploads_empty_result" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + if len(out.Uploads) != 0 { + return fmt.Errorf("expected empty uploads, instead got %+v", out.Uploads) + } + + return nil + }) +} + +func ListMultipartUploads_success(s *S3Conf) { + testName := "ListMultipartUploads_max_uploads" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj1, obj2 := "my-obj-1", "my-obj-2" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out1, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj1, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out2, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj2, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + expected := []types.MultipartUpload{ + { + Key: &obj2, + UploadId: out2.UploadId, + }, + { + Key: &obj1, + UploadId: out1.UploadId, + }, + } + + if len(out.Uploads) != 2 { + return fmt.Errorf("expected 2 upload, instead got %v", len(out.Uploads)) + } + if ok := compareMultipartUploads(out.Uploads, expected); !ok { + return fmt.Errorf("expected uploads %v, instead got %v", expected, out.Uploads) + } + + return nil + }) +} + +func AbortMultipartUpload_non_existing_bucket(s *S3Conf) { + testName := "AbortMultipartUpload_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: getPtr("incorrectBucket"), + Key: getPtr("my-obj"), + UploadId: getPtr("uploadId"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func AbortMultipartUpload_incorrect_uploadId(s *S3Conf) { + testName := "AbortMultipartUpload_incorrect_uploadId" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + UploadId: getPtr("invalid uploadId"), + }) + cancel() + if err := checkSdkApiErr(err, "NoSuchUpload"); err != nil { + return err + } + + return nil + }) +} + +func AbortMultipartUpload_incorrect_object_key(s *S3Conf) { + testName := "AbortMultipartUpload_incorrect_object_key" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: &bucket, + Key: getPtr("incorrect-object-key"), + UploadId: out.UploadId, + }) + cancel() + if err := checkSdkApiErr(err, "NoSuchUpload"); err != nil { + return err + } + + return nil + }) +} + +func AbortMultipartUpload_success(s *S3Conf) { + testName := "AbortMultipartUpload_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if len(res.Uploads) != 0 { + return fmt.Errorf("expected 0 upload, instead got %v", len(res.Uploads)) + } + + return nil + }) +} + +func CompletedMultipartUpload_non_existing_bucket(s *S3Conf) { + testName := "CompletedMultipartUpload_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: getPtr("non-existing-bucket"), + Key: getPtr("some/key"), + UploadId: getPtr("uploadId"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_invalid_part_number(s *S3Conf) { + testName := "CompleteMultipartUpload_invalid_part_number" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + PartNumber: 1, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + { + ETag: res.ETag, + PartNumber: 5, + }, }, }, - }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { + return err + } + + return nil }) - cancel() - if err == nil { - failF("%v: complete multipart expected err", testname) - return - } +} - badEtag := "bogusEtagValue" +func CompleteMultipartUpload_invalid_ETag(s *S3Conf) { + testName := "CompleteMultipartUpload_invalid_ETag" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + PartNumber: 1, + }) + cancel() + if err != nil { + return err + } - // Empty multipart upload - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err == nil { - failF("%v: complete multipart expected err", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - MultipartUpload: &types.CompletedMultipartUpload{ - Parts: []types.CompletedPart{ - { - ETag: mp2.ETag, - PartNumber: 96, - }, - { - ETag: &badEtag, - PartNumber: 42, + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + { + ETag: getPtr("invalidETag"), + PartNumber: 1, + }, }, }, - }, - }) - cancel() - if err == nil { - failF("%v: complete multipart expected err", testname) - return - } + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { + return err + } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - MultipartUpload: &types.CompletedMultipartUpload{ - Parts: []types.CompletedPart{ - { - ETag: mp1.ETag, - PartNumber: 42, + return nil + }) +} + +func CompleteMultipartUpload_success(s *S3Conf) { + testName := "CompleteMultipartUpload_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + objSize := 5 * 1024 * 1024 + parts, err := uploadParts(s3client, objSize, 5, bucket, obj, *out.UploadId) + if err != nil { + return err + } + + compParts := []types.CompletedPart{} + for _, el := range parts { + compParts = append(compParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + }) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err != nil { + return err + } + + if *res.Key != obj { + return fmt.Errorf("expected object key to be %v, instead got %v", obj, *res.Key) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if *resp.ETag != *res.ETag { + return fmt.Errorf("expected the uploaded object etag to be %v, instead got %v", *res.ETag, *resp.ETag) + } + if resp.ContentLength != int64(objSize) { + return fmt.Errorf("expected the uploaded object size to be %v, instead got %v", objSize, resp.ContentLength) + } + + return nil + }) +} + +func PutBucketAcl_non_existing_bucket(s *S3Conf) { + testName := "PutBucketAcl_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: getPtr(getBucketName()), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func PutBucketAcl_invalid_acl_canned_and_acp(s *S3Conf) { + testName := "PutBucketAcl_invalid_acl_canned_and_acp" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + ACL: types.BucketCannedACLPrivate, + GrantRead: getPtr("user1"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRequest)); err != nil { + return err + } + + return nil + }) +} + +func PutBucketAcl_invalid_acl_canned_and_grants(s *S3Conf) { + testName := "PutBucketAcl_invalid_acl_canned_and_grants" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + ACL: types.BucketCannedACLPrivate, + AccessControlPolicy: &types.AccessControlPolicy{ + Grants: []types.Grant{ + { + Grantee: &types.Grantee{ + ID: getPtr("awsID"), + Type: types.TypeCanonicalUser, + }, + }, }, - { - ETag: mp2.ETag, - PartNumber: 96, + Owner: &types.Owner{ + ID: &s.awsID, }, }, - }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRequest)); err != nil { + return err + } + + return nil }) - cancel() - if err != nil { - failF("%v: complete multipart: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - oi, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - failF("%v: head object %v: %v", testname, obj, err) - return - } - - if oi.ContentLength != (int64(size5MB) * 2) { - failF("%v: object len expected %v, got %v", - testname, int64(size5MB)*2, oi.ContentLength) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestIncompleteMultiParts(s *S3Conf) { - testname := "incomplete multipart parts" - runF(testname) +func PutBucketAcl_invalid_acl_acp_and_grants(s *S3Conf) { + testName := "PutBucketAcl_invalid_acl_acp_and_grants" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + GrantFullControl: getPtr("userAccess"), + AccessControlPolicy: &types.AccessControlPolicy{ + Grants: []types.Grant{ + { + Grantee: &types.Grantee{ + ID: getPtr("awsID"), + Type: types.TypeCanonicalUser, + }, + }, + }, + Owner: &types.Owner{ + ID: &s.awsID, + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRequest)); err != nil { + return err + } - bucket := "testbucket9" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - obj := "mympuobject" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, + return nil }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } - - // upload 2 parts - size5MB := 5 * 1024 * 1024 - size1MB := 1024 * 1024 - dr := NewDataReader(size1MB, size1MB) - - datafile := "rand.data" - w, err := os.Create(datafile) - if err != nil { - failF("%v: create %v: %v", testname, datafile, err) - return - } - defer w.Close() - - _, err = io.Copy(w, dr) - if err != nil { - failF("%v: write %v: %v", testname, datafile, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: 42, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), - }) - cancel() - if err == nil { - failF("%v: multipart put short part expected error", testname) - return - } - - // check list parts does not have incomplete part - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err != nil { - failF("%v: list parts: %v", testname, err) - return - } - - if containsPart(42, lp.Parts) { - failF("%v: list parts: found incomplete part", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestIncompletePutObject(s *S3Conf) { - testname := "test incomplete put object" - runF(testname) +func PutBucketAcl_invalid_owner(s *S3Conf) { + testName := "PutBucketAcl_invalid_acl_acp_and_grants" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + AccessControlPolicy: &types.AccessControlPolicy{ + Grants: []types.Grant{ + { + Grantee: &types.Grantee{ + ID: getPtr("awsID"), + Type: types.TypeCanonicalUser, + }, + }, + }, + Owner: &types.Owner{ + ID: getPtr("invalidOwner"), + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } - bucket := "testbucket10" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - // use funny size to prevent accidental alignments - datalen := 1234567 - shortdatalen := 12345 - data := make([]byte, shortdatalen) - rand.Read(data) - r := bytes.NewReader(data) - - name := "myobject" - s3client := s3.NewFromConfig(s.Config()) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &name, - Body: r, - ContentLength: int64(datalen), + return nil }) - cancel() - if err == nil { - failF("%v: expected error for short data put", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &name, - }) - defer cancel() - if err == nil { - failF("%v: expected object not exist", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestRangeGet(s *S3Conf) { - testname := "test range get" - runF(testname) +func PutBucketAcl_success_access_denied(s *S3Conf) { + testName := "PutBucketAcl_success_access_denied" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := createUsers(s, []user{{"grt1", "grt1secret", "user"}}) + if err != nil { + return err + } - bucket := "testbucket11" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + AccessControlPolicy: &types.AccessControlPolicy{ + Grants: []types.Grant{ + { + Grantee: &types.Grantee{ + ID: getPtr("grt1"), + Type: types.TypeCanonicalUser, + }, + Permission: types.PermissionRead, + }, + }, + Owner: &types.Owner{ + ID: &s.awsID, + }, + }, + }) + cancel() + if err != nil { + return err + } - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } + newConf := *s + newConf.awsID = "grt1" + newConf.awsSecret = "grt1secret" + userClient := s3.NewFromConfig(newConf.Config()) - datalen := 10 * 1024 - data := make([]byte, datalen) - rand.Read(data) - r := bytes.NewReader(data) + err = putObjects(userClient, []string{"my-obj"}, bucket) + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } - name := "myobject" - s3client := s3.NewFromConfig(s.Config()) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &name, - Body: r, + return nil }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - lgRange := "bytes=20000000-30000000000" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - Range: &lgRange, - }) - cancel() - if err == nil { - failF("%v: expected range error", testname) - return - } - - // Invalid range - invRange := "bytes=100-asd" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - Range: &invRange, - }) - defer cancel() - if err == nil { - failF("%v: expected range error", testname) - return - } - - rangeString := "bytes=100-200" - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - Range: &rangeString, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - defer out.Body.Close() - - if getString(out.ContentRange) != fmt.Sprintf("bytes 100-200/%v", datalen) { - failF("%v: expected content range: %v, instead got: %v", testname, fmt.Sprintf("bytes 100-200/%v", datalen), getString(out.ContentRange)) - return - } - if getString(out.AcceptRanges) != rangeString { - failF("%v: expected accept range: %v, instead got: %v", testname, rangeString, getString(out.AcceptRanges)) - } - - b, err := io.ReadAll(out.Body) - if err != nil { - failF("%v: read body %v", testname, err) - return - } - - // bytes range is inclusive, go range for second value is not - if !isEqual(b, data[100:201]) { - failF("%v: data mismatch of range", testname) - return - } - - rangeString = "bytes=100-" - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - Range: &rangeString, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - defer out.Body.Close() - - b, err = io.ReadAll(out.Body) - if err != nil { - failF("%v: read body %v", testname, err) - return - } - - // bytes range is inclusive, go range for second value is not - if !isEqual(b, data[100:]) { - failF("%v: data mismatch of range", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestInvalidMultiParts(s *S3Conf) { - testname := "invalid multipart parts" - runF(testname) +func PutBucketAcl_success(s *S3Conf) { + testName := "PutBucketAcl_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := createUsers(s, []user{{"grt1", "grt1secret", "user"}}) + if err != nil { + return err + } - bucket := "bucket12" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + AccessControlPolicy: &types.AccessControlPolicy{ + Grants: []types.Grant{ + { + Grantee: &types.Grantee{ + ID: getPtr("grt1"), + Type: types.TypeCanonicalUser, + }, + Permission: types.PermissionFullControl, + }, + }, + Owner: &types.Owner{ + ID: &s.awsID, + }, + }, + }) + cancel() + if err != nil { + return err + } - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } + newConf := *s + newConf.awsID = "grt1" + newConf.awsSecret = "grt1secret" + userClient := s3.NewFromConfig(newConf.Config()) - s3client := s3.NewFromConfig(s.Config()) + err = putObjects(userClient, []string{"my-obj"}, bucket) + if err != nil { + return err + } - obj := "mympuobject" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, + return nil }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } +} - // upload 2 parts - size5MB := 5 * 1024 * 1024 - dr := NewDataReader(size5MB, size5MB) +func GetBucketAcl_non_existing_bucket(s *S3Conf) { + testName := "GetBucketAcl_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{ + Bucket: getPtr(getBucketName()), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } - datafile := "rand.data" - w, err := os.Create(datafile) - if err != nil { - failF("%v: create %v: %v", testname, datafile, err) - return - } - defer w.Close() - - _, err = io.Copy(w, dr) - if err != nil { - failF("%v: write %v: %v", testname, datafile, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: -1, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), + return nil }) - cancel() - if err == nil { - failF("%v: multipart put part 1 expected error", testname) - return - } +} - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, +func GetBucketAcl_access_denied(s *S3Conf) { + testName := "GetBucketAcl_access_denied" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := createUsers(s, []user{{"grt1", "grt1secret", "user"}}) + if err != nil { + return err + } + + newConf := *s + newConf.awsID = "grt1" + newConf.awsSecret = "grt1secret" + userClient := s3.NewFromConfig(newConf.Config()) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.GetBucketAcl(ctx, &s3.GetBucketAclInput{ + Bucket: &bucket, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } + + return nil }) - cancel() - if err == nil { - failF("%v: head object %v expected error", testname, obj) - return - } +} - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) +func GetBucketAcl_success(s *S3Conf) { + testName := "GetBucketAcl_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := createUsers(s, []user{ + {"grt1", "grt1secret", "user"}, + {"grt2", "grt2secret", "user"}, + {"grt3", "grt3secret", "user"}, + }) + if err != nil { + return err + } + + grants := []types.Grant{ + { + Grantee: &types.Grantee{ + ID: getPtr("grt1"), + Type: types.TypeCanonicalUser, + }, + Permission: types.PermissionFullControl, + }, + { + Grantee: &types.Grantee{ + ID: getPtr("grt2"), + Type: types.TypeCanonicalUser, + }, + Permission: types.PermissionReadAcp, + }, + { + Grantee: &types.Grantee{ + ID: getPtr("grt3"), + Type: types.TypeCanonicalUser, + }, + Permission: types.PermissionWrite, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + AccessControlPolicy: &types.AccessControlPolicy{ + Grants: grants, + Owner: &types.Owner{ + ID: &s.awsID, + }, + }, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if ok := compareGrants(out.Grants, grants); !ok { + return fmt.Errorf("expected grants to be %v, instead got %v", grants, out.Grants) + } + if *out.Owner.ID != s.awsID { + return fmt.Errorf("expected bucket owner to be %v, instead got %v", s.awsID, *out.Owner.ID) + } + + return nil + }) } type prefResult struct { @@ -1360,233 +2497,3 @@ func TestPerformance(s *S3Conf, upload, download bool, files int, objectSize int return nil } - -func TestPutGetRemoveTags(s *S3Conf) { - testname := "test put/get/remove object tags" - runF(testname) - - bucket := "testbucket13" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - obj := "myobject" - s3client := s3.NewFromConfig(s.Config()) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - key1 := "hello1" - key2 := "hello2" - val1 := "world1" - val2 := "world2" - - tagging := types.Tagging{TagSet: []types.Tag{{Key: &key1, Value: &val1}, {Key: &key2, Value: &val2}}} - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - Tagging: &tagging, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Key: &obj, - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ok := areTagsSame(tagging.TagSet, out.TagSet) - if !ok { - failF("%v: expected %v instead got %v", testname, tagging.TagSet, out.TagSet) - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ - Key: &obj, - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Key: &obj, - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - if len(out.TagSet) > 0 { - failF("%v: expected empty tag set instead got %v", testname, out.TagSet) - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -func TestAclActions(s *S3Conf) { - testname := "test put/get acl" - runF(testname) - - bucket := "testbucket14" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - rootAccess := s.awsID - rootSecret := s.awsSecret - - s.awsID = "grt1" - s.awsSecret = "grt1secret" - - userS3Client := s3.NewFromConfig(s.Config()) - - s.awsID = rootAccess - s.awsSecret = rootSecret - - grt1 := "grt1" - - grants := []types.Grant{ - { - Permission: "READ", - Grantee: &types.Grantee{ - ID: &grt1, - Type: "CanonicalUser", - }, - }, - } - - succUsrCrt := "The user has been created successfully" - failUsrCrt := "failed to create a user: update iam data: account already exists" - - out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "create-user", "-a", grt1, "-s", "grt1secret", "-r", "user") - if err != nil { - failF("%v: %v", err) - return - } - if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) { - failF("%v: failed to create user accounts", testname) - return - } - - // Validation error case - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ - Bucket: &bucket, - AccessControlPolicy: &types.AccessControlPolicy{ - Grants: grants, - }, - ACL: "private", - }) - cancel() - if err == nil { - failF("%v: expected validation error", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ - Bucket: &bucket, - AccessControlPolicy: &types.AccessControlPolicy{ - Grants: grants, - Owner: &types.Owner{ID: &s.awsID}, - }, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - acl, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - if *acl.Owner.ID != s.awsID { - failF("%v: expected bucket owner: %v, instead got: %v", testname, s.awsID, *acl.Owner.ID) - return - } - if !checkGrants(acl.Grants, grants) { - failF("%v: expected %v, instead got %v", testname, grants, acl.Grants) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userS3Client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ - Bucket: &bucket, - }) - cancel() - if err == nil { - failF("%v: expected acl access denied error", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -// Full flow test -func TestFullFlow(s *S3Conf) { - // TODO: add more test cases to get 100% coverage - TestMakeBucket(s) - TestPutGetObject(s) - TestPutGetMPObject(s) - TestPutDirObject(s) - TestListObject(s) - TestIncompletePutObject(s) - TestListMultiParts(s) - TestIncompleteMultiParts(s) - TestIncorrectMultiParts(s) - TestListAbortMultiPartObject(s) - TestRangeGet(s) - TestInvalidMultiParts(s) - TestPutGetRemoveTags(s) - TestAclActions(s) -} diff --git a/integration/utils.go b/integration/utils.go index 46ba3db6..42babc1f 100644 --- a/integration/utils.go +++ b/integration/utils.go @@ -1,15 +1,34 @@ package integration import ( + "bytes" "context" + "crypto/rand" + "crypto/sha256" + "errors" "fmt" + "io" + "os" "os/exec" "strings" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/versity/versitygw/s3err" ) +var ( + bcktCount = 0 + succUsrCrt = "The user has been created successfully" + failUsrCrt = "failed to create a user: update iam data: account already exists" +) + +func getBucketName() string { + bcktCount++ + return fmt.Sprintf("test-bucket-%v", bcktCount) +} + func setup(s *S3Conf, bucket string) error { s3client := s3.NewFromConfig(s.Config()) @@ -69,6 +88,89 @@ func teardown(s *S3Conf, bucket string) error { return err } +func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) { + runF(testName) + bucketName := getBucketName() + err := setup(s, bucketName) + if err != nil { + failF("%v: failed to create a bucket: %v", testName, err.Error()) + return + } + client := s3.NewFromConfig(s.Config()) + handlerErr := handler(client, bucketName) + if handlerErr != nil { + failF("%v: %v", testName, handlerErr.Error()) + } + + err = teardown(s, bucketName) + if err != nil { + if handlerErr == nil { + failF("%v: failed to delete the bucket: %v", testName, err.Error()) + } else { + fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err.Error()) + } + } + if handlerErr == nil { + passF(testName) + } +} + +func putObjects(client *s3.Client, objs []string, bucket string) error { + for _, key := range objs { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Key: &key, + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + } + return nil +} + +func checkApiErr(err error, apiErr s3err.APIError) error { + if err == nil { + return fmt.Errorf("expected %v, instead got nil", apiErr.Code) + } + var ae smithy.APIError + if errors.As(err, &ae) { + if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description { + return nil + } + + return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode()) + } else { + return fmt.Errorf("expected aws api error, instead got: %v", err.Error()) + } +} + +func checkSdkApiErr(err error, code string) error { + var ae smithy.APIError + if errors.As(err, &ae) { + if ae.ErrorCode() != code { + return fmt.Errorf("expected %v, instead got %v", ae.ErrorCode(), code) + } + return nil + } + return err +} + +func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) { + data = make([]byte, lgth) + rand.Read(data) + csum = sha256.Sum256(data) + r := bytes.NewReader(data) + input.Body = r + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = client.PutObject(ctx, input) + cancel() + + return +} + func isEqual(a, b []byte) bool { if len(a) != len(b) { return false @@ -83,32 +185,33 @@ func isEqual(a, b []byte) bool { return true } -func contains(name string, list []types.Object) bool { - for _, item := range list { - fmt.Println(*item.Key) - if strings.EqualFold(name, *item.Key) { - return true +func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool { + if len(list1) != len(list2) { + return false + } + for i, item := range list1 { + if *item.Key != *list2[i].Key || *item.UploadId != *list2[i].UploadId { + return false } } - return false + + return true } -func containsUID(name, id string, list []types.MultipartUpload) bool { - for _, item := range list { - if strings.EqualFold(name, *item.Key) && strings.EqualFold(id, *item.UploadId) { - return true - } +func compareParts(parts1, parts2 []types.Part) bool { + if len(parts1) != len(parts2) { + return false } - return false -} -func containsPart(part int32, list []types.Part) bool { - for _, item := range list { - if item.PartNumber == part { - return true + for i, prt := range parts1 { + if prt.PartNumber != parts2[i].PartNumber { + return false + } + if *prt.ETag != *parts2[i].ETag { + return false } } - return false + return true } func areTagsSame(tags1, tags2 []types.Tag) bool { @@ -133,7 +236,7 @@ func containsTag(tag types.Tag, list []types.Tag) bool { return false } -func checkGrants(grts1, grts2 []types.Grant) bool { +func compareGrants(grts1, grts2 []types.Grant) bool { if len(grts1) != len(grts2) { return false } @@ -162,6 +265,10 @@ func getString(str *string) string { return *str } +func getPtr(str string) *string { + return &str +} + func areMapsSame(mp1, mp2 map[string]string) bool { if len(mp1) != len(mp2) { return false @@ -173,3 +280,115 @@ func areMapsSame(mp1, mp2 map[string]string) bool { } return true } + +func compareObjects(list1 []string, list2 []types.Object) bool { + if len(list1) != len(list2) { + return false + } + + elementMap := make(map[string]bool) + + for _, elem := range list1 { + elementMap[elem] = true + } + + for _, elem := range list2 { + if _, found := elementMap[*elem.Key]; !found { + return false + } + } + + return true +} + +func compareDelObjects(list1 []string, list2 []types.DeletedObject) bool { + if len(list1) != len(list2) { + return false + } + + elementMap := make(map[string]bool) + + for _, elem := range list1 { + elementMap[elem] = true + } + + for _, elem := range list2 { + if _, found := elementMap[*elem.Key]; !found { + return false + } + } + + return true +} + +func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId string) (parts []types.Part, err error) { + dr := NewDataReader(size, size) + datafile := "rand.data" + w, err := os.Create(datafile) + if err != nil { + return parts, err + } + defer w.Close() + + _, err = io.Copy(w, dr) + if err != nil { + return parts, err + } + + fileInfo, err := w.Stat() + if err != nil { + return parts, err + } + + partSize := fileInfo.Size() / int64(partCount) + var offset int64 + + for partNumber := int64(1); partNumber <= int64(partCount); partNumber++ { + partStart := (partNumber - 1) * partSize + partEnd := partStart + partSize - 1 + if partEnd > fileInfo.Size()-1 { + partEnd = fileInfo.Size() - 1 + } + partBuffer := make([]byte, partEnd-partStart+1) + _, err := w.ReadAt(partBuffer, partStart) + if err != nil { + return parts, err + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + Body: bytes.NewReader(partBuffer), + PartNumber: int32(partNumber), + }) + cancel() + if err != nil { + return parts, err + } else { + parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)}) + offset += partSize + } + } + + return parts, err +} + +type user struct { + access string + secret string + role string +} + +func createUsers(s *S3Conf, users []user) error { + for _, usr := range users { + out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role) + if err != nil { + return err + } + if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) { + return fmt.Errorf("failed to create a user account") + } + } + return nil +}