From 2cc0c7203c9026337549e6884224f0d305b3f8ed Mon Sep 17 00:00:00 2001 From: jonaustin09 Date: Fri, 11 Aug 2023 02:13:01 +0400 Subject: [PATCH 1/4] feat: Closes #189, added utility functions for testing, restructured tests for CreateBucket, DeleteBucket... and 5 more actions --- cmd/versitygw/test.go | 108 -- integration/action-tests.go | 58 + integration/tests.go | 3076 +++++++++++++++++++---------------- integration/utils.go | 91 ++ 4 files changed, 1790 insertions(+), 1543 deletions(-) create mode 100644 integration/action-tests.go diff --git a/cmd/versitygw/test.go b/cmd/versitygw/test.go index 578b30a4..1bcfcd96 100644 --- a/cmd/versitygw/test.go +++ b/cmd/versitygw/test.go @@ -67,114 +67,6 @@ func initTestFlags() []cli.Flag { func initTestCommands() []*cli.Command { return []*cli.Command{ - { - Name: "bucket-actions", - Usage: "Test bucket creation, checking the existence, deletes it.", - Description: `Calls s3 gateway create-bucket action to create a new bucket, - calls head-bucket action to check the existence, then calls delete-bucket action to delete the bucket.`, - Action: getAction(integration.TestMakeBucket), - }, - { - Name: "object-actions", - Usage: "Test put/get/delete/copy objects.", - Description: `Creates a bucket with s3 gateway action, puts an object in it, - tries to copy into another bucket, that doesn't exist, creates the destination bucket for copying, - copies the object, get's the object to check the length and content, - get's the copied object to check the length and content, deletes all the objects inside the source bucket, - deletes both the objects and buckets.`, - Action: getAction(integration.TestPutGetObject), - }, - { - Name: "put-get-mp-object", - Usage: "Test put & get multipart object.", - Description: `Creates a bucket with s3 gateway action, puts an object in it with multipart upload, - gets the object from the bucket, deletes both the object and bucket.`, - Action: getAction(integration.TestPutGetMPObject), - }, - { - Name: "put-dir-object", - Usage: "Test put directory object.", - Description: `Creates a bucket with s3 gateway action, puts a directory object in it, - lists the bucket's objects, deletes both the objects and bucket.`, - Action: getAction(integration.TestPutDirObject), - }, - { - Name: "list-objects", - Usage: "Test list-objects action.", - Description: `Creates a bucket with s3 gateway action, puts 2 directory objects in it, - lists the bucket's objects, deletes both the objects and bucket.`, - Action: getAction(integration.TestListObject), - }, - { - Name: "abort-mp", - Usage: "Tests abort-multipart-upload action.", - Description: `Creates a bucket with s3 gateway action, creates a multipart upload, - lists the multipart upload, aborts the multipart upload, lists the multipart upload again, - deletes both the objects and bucket.`, - Action: getAction(integration.TestListAbortMultiPartObject), - }, - { - Name: "list-parts", - Usage: "Tests list-parts action.", - Description: `Creates a bucket with s3 gateway action, creates a multipart upload, - lists the upload parts, deletes both the objects and bucket.`, - Action: getAction(integration.TestListMultiParts), - }, - { - Name: "incorrect-mp", - Usage: "Tests incorrect multipart case.", - Description: `Creates a bucket with s3 gateway action, creates a multipart upload, - uploads different parts, completes the multipart upload with incorrect part numbers, - calls the head-object action, compares the content length, removes both the object and bucket`, - Action: getAction(integration.TestIncorrectMultiParts), - }, - { - Name: "incomplete-mp", - Usage: "Tests incomplete multi parts.", - Description: `Creates a bucket with s3 gateway action, creates a multipart upload, - upload a part, lists the parts, checks if the uploaded part is in the list, - removes both the object and the bucket`, - Action: getAction(integration.TestIncompleteMultiParts), - }, - { - Name: "incomplete-put-object", - Usage: "Tests incomplete put objects case.", - Description: `Creates a bucket with s3 gateway action, puts an object in it, - gets the object with head-object action, expects the object to be got, - removes both the object and bucket`, - Action: getAction(integration.TestIncompletePutObject), - }, - { - Name: "get-range", - Usage: "Tests get object by range.", - Description: `Creates a bucket with s3 gateway action, puts an object in it, - gets the object by specifying the object range, compares the range with the original one, - removes both the object and the bucket`, - Action: getAction(integration.TestRangeGet), - }, - { - Name: "invalid-mp", - Usage: "Tests invalid multi part case.", - Description: `Creates a bucket with s3 gateway action, creates a multi part upload, - uploads an invalid part, gets the object with head-object action, expects to get error, - removes both the object and bucket`, - Action: getAction(integration.TestInvalidMultiParts), - }, - { - Name: "object-tag-actions", - Usage: "Tests get/put/delete object tag actions.", - Description: `Creates a bucket with s3 gateway action, puts an object in it, - puts some tags for the object, gets the tags, compares the results, removes the tags, - gets the tags again, checks it to be empty, then removes both the object and bucket`, - Action: getAction(integration.TestPutGetRemoveTags), - }, - { - Name: "bucket-acl-actions", - Usage: "Tests put/get bucket actions.", - Description: `Creates a bucket with s3 gateway action, puts some bucket acls - gets the acl, verifies it, then removes the bucket`, - Action: getAction(integration.TestAclActions), - }, { Name: "full-flow", Usage: "Tests the full flow of gateway.", diff --git a/integration/action-tests.go b/integration/action-tests.go new file mode 100644 index 00000000..f6f2ae4f --- /dev/null +++ b/integration/action-tests.go @@ -0,0 +1,58 @@ +package integration + +func TestCreateBucket(s *S3Conf) { + CreateBucket_invalid_bucket_name(s) + CreateBucket_existing_bucket(s) + CreateDeleteBucket_success(s) +} + +func TestDeleteBucket(s *S3Conf) { + DeleteBucket_non_existing_bucket(s) + DeleteBucket_non_empty_bucket(s) +} + +func TestPutObject(s *S3Conf) { + PutObject_non_existing_bucket(s) + PutObject_special_chars(s) + PutObject_existing_dir_obj(s) + PutObject_obj_parent_is_file(s) + PutObject_success(s) +} + +func TestGetObject(s *S3Conf) { + GetObject_non_existing_key(s) + GetObject_invalid_ranges(s) + GetObject_with_meta(s) + GetObject_success(s) + GetObject_by_range_success(s) +} + +func TestCopyObject(s *S3Conf) { + CopyObject_non_existing_dst_bucket(s) + CopyObject_success(s) +} + +func TestPutObjectTagging(s *S3Conf) { + PutObjectTagging_non_existing_object(s) + PutObjectTagging_success(s) +} + +func TestGetObjectTagging(s *S3Conf) { + GetObjectTagging_non_existing_object(s) + GetObjectTagging_success(s) +} + +func TestDeleteObjectTagging(s *S3Conf) { + DeleteObjectTagging_non_existing_object(s) + DeleteObjectTagging_success(s) +} + +func TestFullFlow(s *S3Conf) { + TestCreateBucket(s) + TestDeleteBucket(s) + TestPutObject(s) + TestGetObject(s) + TestCopyObject(s) + TestPutObjectTagging(s) + TestDeleteObjectTagging(s) +} diff --git a/integration/tests.go b/integration/tests.go index aed74d99..f4c0aa68 100644 --- a/integration/tests.go +++ b/integration/tests.go @@ -1,1295 +1,1731 @@ package integration import ( - "bytes" "context" - "crypto/rand" "crypto/sha256" + "errors" "fmt" "io" "math" - "os" - "strings" "sync" "time" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" ) var ( shortTimeout = 10 * time.Second ) -func TestMakeBucket(s *S3Conf) { - testname := "test make/head/delete bucket" - runF(testname) - - s3client := s3.NewFromConfig(s.Config()) - - invBucket := "aa" - err := setup(s, invBucket) - if err == nil { - failF("%v: expected bucket name validation error", testname) +func CreateBucket_invalid_bucket_name(s *S3Conf) { + testName := "CreateBucket_invalid_bucket_name" + runF(testName) + err := setup(s, "aa") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketName)); err != nil { + failF("%v: %v", testName, err.Error()) return } - bucket := "testbucket" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket}) - cancel() - if err == nil { - failF("%v: expected error, instead got success response", testname) + err = setup(s, ".gitignore") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketName)); err != nil { + failF("%v: %v", testName, err.Error()) return } + err = setup(s, "my-bucket.") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketName)); err != nil { + failF("%v: %v", testName, err.Error()) + return + } + + err = setup(s, "bucket-%") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketName)); err != nil { + failF("%v: %v", testName, err.Error()) + return + } + passF(testName) +} + +func CreateBucket_existing_bucket(s *S3Conf) { + testName := "CreateBucket_existing_bucket" + runF(testName) + bucket := getBucketName() + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testName, err.Error()) + return + } err = setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return + var bne *types.BucketAlreadyExists + if !errors.As(err, &bne) { + failF("%v: %v", testName, err.Error()) } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket}) - cancel() + err = teardown(s, bucket) if err != nil { - failF("%v: %v", testname, err) + failF("%v: %v", err.Error()) + return + } + passF(testName) +} + +func CreateDeleteBucket_success(s *S3Conf) { + testName := "CreateBucket_success" + runF(testName) + bucket := getBucketName() + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", err.Error()) return } err = teardown(s, bucket) if err != nil { - failF("%v: %v", testname, err) + failF("%v: %v", err.Error()) return } - passF(testname) + + passF(testName) } -func TestPutGetObject(s *S3Conf) { - testname := "test put/get/delete/copy objects" - runF(testname) - - bucket := "testbucket1" - dstBucket := "testdstbucket" - obj := "myobject" - obj2 := "myobject2" - obj3 := "myobject%%3" - copySource := bucket + "/" + obj - +func DeleteBucket_non_existing_bucket(s *S3Conf) { + testName := "DeleteBucket_non_existing_bucket" + runF(testName) + bucket := getBucketName() s3client := s3.NewFromConfig(s.Config()) - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - // use funny size to prevent accidental alignments - datalen := 1234567 - data := make([]byte, datalen) - rand.Read(data) - csum := sha256.Sum256(data) - r := bytes.NewReader(data) - meta := map[string]string{ - "key1": "val1", - "key2": "val2", - } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - Body: r, - Metadata: meta, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + _, err := s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ Bucket: &bucket, - Key: &obj3, }) cancel() - if err != nil { - failF("%v: %v", testname, err) + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + failF("%v: %v", testName, err.Error()) return } + passF(testName) +} - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj2, - Body: r, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj3, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - if !areMapsSame(out.Metadata, meta) { - failF("%v: incorrect object metadata", testname) - return - } - defer out.Body.Close() - - if out.ContentLength != int64(datalen) { - failF("%v: content length got %v expected %v", testname, out.ContentLength, datalen) - return - } - - b, err := io.ReadAll(out.Body) - if err != nil { - failF("%v: read body %v", testname, err) - return - } - - newsum := sha256.Sum256(b) - if csum != newsum { - failF("%v: checksum got %x expected %x", testname, newsum, csum) - return - } - - // Expected error: destination bucket doesn't exist - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: &dstBucket, Key: &obj, CopySource: ©Source}) - cancel() - if err == nil { - failF("%v: expect bucket not found error instead got success response", testname) - return - } - - err = setup(s, dstBucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: &dstBucket, Key: &obj, CopySource: ©Source}) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - copyObjOut, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &dstBucket, - Key: &obj, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - defer copyObjOut.Body.Close() - - if copyObjOut.ContentLength != int64(datalen) { - failF("%v: content length got %v expected %v", testname, copyObjOut.ContentLength, datalen) - return - } - - b, err = io.ReadAll(copyObjOut.Body) - if err != nil { - failF("%v: read body %v", testname, err) - return - } - - copysum := sha256.Sum256(b) - if csum != copysum { - failF("%v: copied object checksum got %x expected %x", testname, copysum, csum) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{Bucket: &bucket, Delete: &types.Delete{Objects: []types.ObjectIdentifier{{Key: &obj}, {Key: &obj2}, {Key: &obj3}}}}) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - objCount := 0 - - in := &s3.ListObjectsV2Input{Bucket: &bucket} - for { +func DeleteBucket_non_empty_bucket(s *S3Conf) { + testName := "DeleteBucket_non_empty_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"foo"}, bucket) + if err != nil { + return err + } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectsV2(ctx, in) + _, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: &bucket, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrBucketNotEmpty)); err != nil { + return err + } + + return nil + }) +} + +func PutObject_non_existing_bucket(s *S3Conf) { + testName := "PutObject_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"my-obj"}, "non-existing-bucket") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + return nil + }) +} + +func PutObject_special_chars(s *S3Conf) { + testName := "PutObject_special_chars" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"foo%%", "bar^", "baz**"}, bucket) + if err != nil { + return err + } + return nil + }) +} + +func PutObject_existing_dir_obj(s *S3Conf) { + testName := "PutObject_existing_dir_obj" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"foo/bar", "foo"}, bucket) + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)); err != nil { + return err + } + return nil + }) +} + +func PutObject_obj_parent_is_file(s *S3Conf) { + testName := "PutObject_obj_parent_is_file" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"foo", "foo/bar/"}, bucket) + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectParentIsFile)); err != nil { + return err + } + return nil + }) +} + +func PutObject_success(s *S3Conf) { + testName := "PutObject_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"my-obj"}, bucket) + if err != nil { + return err + } + return nil + }) +} + +func GetObject_non_existing_key(s *S3Conf) { + testName := "GetObject_non_existing_key" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: getPtr("non-existing-key"), + }) + cancel() + var bae *types.NoSuchKey + if !errors.As(err, &bae) { + return err + } + return nil + }) +} + +func GetObject_invalid_ranges(s *S3Conf) { + testName := "GetObject_invalid_ranges" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + + _, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: getPtr("bytes=invalid-range"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: getPtr("bytes=33-10"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: getPtr("bytes=1000000000-999999999999"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { + return err + } + return nil + }) +} + +func GetObject_with_meta(s *S3Conf) { + testName := "GetObject_with_meta" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + meta := map[string]string{ + "key1": "val1", + "key2": "val2", + } + + _, _, err := putObjectWithData(0, &s3.PutObjectInput{Bucket: &bucket, Key: &obj, Metadata: meta}, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + + if !areMapsSame(out.Metadata, meta) { + return fmt.Errorf("incorrect object metadata") + } + + return nil + }) +} + +func GetObject_success(s *S3Conf) { + testName := "GetObject_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + + csum, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + if out.ContentLength != dataLength { + return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) + } + + bdy, err := io.ReadAll(out.Body) + if err != nil { + return err + } + defer out.Body.Close() + outCsum := sha256.Sum256(bdy) + if outCsum != csum { + return fmt.Errorf("invalid object data") + } + return nil + }) +} + +func GetObject_by_range_success(s *S3Conf) { + testName := "GetObject_by_range_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + + _, data, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + rangeString := "bytes=100-200" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: &rangeString, + }) + defer cancel() + if err != nil { + return err + } + defer out.Body.Close() + + if getString(out.ContentRange) != fmt.Sprintf("bytes 100-200/%v", dataLength) { + return fmt.Errorf("expected content range: %v, instead got: %v", fmt.Sprintf("bytes 100-200/%v", dataLength), getString(out.ContentRange)) + } + if getString(out.AcceptRanges) != rangeString { + return fmt.Errorf("expected accept range: %v, instead got: %v", rangeString, getString(out.AcceptRanges)) + } + b, err := io.ReadAll(out.Body) + if err != nil { + return err + } + + // bytes range is inclusive, go range for second value is not + if !isEqual(b, data[100:201]) { + return fmt.Errorf("data mismatch of range") + } + + rangeString = "bytes=100-" + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: &rangeString, + }) + defer cancel() + if err != nil { + return err + } + defer out.Body.Close() + + b, err = io.ReadAll(out.Body) + if err != nil { + return err + } + + // bytes range is inclusive, go range for second value is not + if !isEqual(b, data[100:]) { + return fmt.Errorf("data mismatch of range") + } + return nil + }) +} + +func CopyObject_non_existing_dst_bucket(s *S3Conf) { + testName := "CopyObject_non_existing_dst_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + err := putObjects(s3client, []string{obj}, bucket) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr("bucket/obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + return nil + }) +} + +func CopyObject_success(s *S3Conf) { + testName := "CopyObject_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + dstBucket := getBucketName() + err := setup(s, dstBucket) + if err != nil { + return err + } + + csum, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &dstBucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, obj)), + }) cancel() if err != nil { - failF("%v: %v", testname, err) - return + return err } - objCount += len(out.Contents) - if out.IsTruncated { - in.ContinuationToken = out.ContinuationToken - } else { - break - } - } - if objCount != 0 { - failF("%v: expected object count %v instead got %v", testname, 0, objCount) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - err = teardown(s, dstBucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - passF(testname) -} - -func TestPutGetMPObject(s *S3Conf) { - testname := "test put/get multipart object" - runF(testname) - - bucket := "testbucket2" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - name := "mympuobject" - s3client := s3.NewFromConfig(s.Config()) - - datalen := 10*1024*1024 + 15 - dr := NewDataReader(datalen, 5*1024*1024) - WithPartSize(5 * 1024 * 1024) - s.PartSize = 5 * 1024 * 1024 - err = s.UploadData(dr, bucket, name) - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - defer out.Body.Close() - - if out.ContentLength != int64(datalen) { - failF("%v: content length got %v expected %v", testname, out.ContentLength, datalen) - return - } - - b := make([]byte, 1048576) - h := sha256.New() - for { - n, err := out.Body.Read(b) - if err == io.EOF { - h.Write(b[:n]) - break - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &dstBucket, + Key: &obj, + }) + defer cancel() if err != nil { - failF("%v: read %v", err) - return + return err } - h.Write(b[:n]) - } - - if !isEqual(dr.Sum(), h.Sum(nil)) { - failF("%v: checksum got %x expected %x", testname, h.Sum(nil), dr.Sum()) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -func TestPutDirObject(s *S3Conf) { - testname := "test put directory object" - runF(testname) - - bucket := "testbucket3" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - name := "myobjectdir/" - s3client := s3.NewFromConfig(s.Config()) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &name, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket, MaxKeys: -4}) - cancel() - if err == nil { - failF("%v: expected invalid argument error", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) - cancel() - if err != nil { - failF("failed to list objects: %v", err) - return - } - - if !contains(name, out.Contents) { - failF("directory object not found") - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -func TestListObject(s *S3Conf) { - testname := "list objects" - runF(testname) - - bucket := "testbucket4" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - dir1 := "myobjectdir/" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &dir1, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - obj1 := "myobjectdir/myobject" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj1, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - obj2 := "myobjectdir1/myobject" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj2, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - // put: - // "myobjectdir/" - // "myobjectdir/myobject" - // "myobjectdir1/myobject" - // should return: - // "myobjectdir/myobject" - // "myobjectdir1/myobject" - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) - cancel() - if err != nil { - failF("failed to list objects: %v", err) - return - } - - if !contains(obj1, out.Contents) { - failF("object %v not found", obj1) - return - } - if !contains(obj2, out.Contents) { - failF("object %v not found", obj2) - return - } - if out.KeyCount != 2 { - failF("%v: expected key count: %v, instead got: %v", testname, 2, out.KeyCount) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj1, - }) - cancel() - if err != nil { - failF("failed to delete %v: %v", obj1, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj2, - }) - cancel() - if err != nil { - failF("failed to delete %v: %v", obj2, err) - return - } - - // put: - // "myobjectdir/" - // "myobjectdir/myobject" - // "myobjectdir1/myobject" - // delete: - // "myobjectdir/myobject" - // "myobjectdir1/myobject" - // should return: - // "myobjectdir/" - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) - cancel() - if err != nil { - failF("failed to list objects: %v", err) - return - } - - if !contains(dir1, out.Contents) { - failF("dir %v not found", dir1) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -func TestListAbortMultiPartObject(s *S3Conf) { - testname := "list/abort multipart objects" - runF(testname) - - bucket := "testbucket6" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - obj := "mympuobject" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lmpu, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: list multipart upload: %v", testname, err) - return - } - - //for _, item := range lmpu.Uploads { - // fmt.Println(" -- ", *item.Key, *item.UploadId) - //} - - if !containsUID(obj, *mpu.UploadId, lmpu.Uploads) { - failF("%v: upload %v/%v not found", testname, obj, *mpu.UploadId) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err != nil { - failF("%v: abort multipart upload: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lmpu, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: list multipart upload: %v", testname, err) - return - } - - if len(lmpu.Uploads) != 0 { - for _, item := range lmpu.Uploads { - fmt.Println(" D- ", *item.Key, *item.UploadId) + if out.ContentLength != dataLength { + return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) } - failF("%v: unexpected multipart uploads found", testname) - return - } - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) + bdy, err := io.ReadAll(out.Body) + if err != nil { + return err + } + defer out.Body.Close() + outCsum := sha256.Sum256(bdy) + if outCsum != csum { + return fmt.Errorf("invalid object data") + } + + err = teardown(s, dstBucket) + if err != nil { + return nil + } + + return nil + }) } -func TestListMultiParts(s *S3Conf) { - testname := "list multipart parts" - runF(testname) - - bucket := "testbucket7" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - obj := "mympuobject" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, +func PutObjectTagging_non_existing_object(s *S3Conf) { + testName := "PutObjectTagging_non_existing_object" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + Tagging: &types.Tagging{TagSet: []types.Tag{}}}) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + return nil }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } - - // check list parts of no parts is good - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err != nil { - failF("%v: list parts: %v", testname, err) - return - } - - if len(lp.Parts) != 0 { - failF("%v: list parts: expected no parts, got %v", - testname, len(lp.Parts)) - return - } - - // upload 1 part and check list parts - size5MB := 5 * 1024 * 1024 - dr := NewDataReader(size5MB, size5MB) - - datafile := "rand.data" - w, err := os.Create(datafile) - if err != nil { - failF("%v: create %v: %v", testname, datafile, err) - return - } - defer w.Close() - - _, err = io.Copy(w, dr) - if err != nil { - failF("%v: write %v: %v", testname, datafile, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: 42, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), - }) - cancel() - if err != nil { - failF("%v: multipart put part: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lp, err = s3client.ListParts(ctx, &s3.ListPartsInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err != nil { - failF("%v: list parts: %v", testname, err) - return - } - - //for _, part := range lp.Parts { - // fmt.Println(" -- ", part.PartNumber, part.ETag) - //} - - if len(lp.Parts) != 1 || lp.Parts[0].PartNumber != 42 { - fmt.Printf("%+v, %v, %v\n", lp.Parts, *lp.Key, *lp.UploadId) - failF("%v: list parts: unexpected parts listing", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestIncorrectMultiParts(s *S3Conf) { - testname := "incorrect multipart parts" - runF(testname) +func PutObjectTagging_success(s *S3Conf) { + testName := "PutObjectTagging_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + tagging := types.Tagging{TagSet: []types.Tag{{Key: getPtr("key1"), Value: getPtr("val2")}, {Key: getPtr("key2"), Value: getPtr("val2")}}} + err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } - bucket := "testbucket8" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + Tagging: &tagging}) + cancel() + if err != nil { + return err + } - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - obj := "mympuobject" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, + return nil }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } - - // upload 2 parts - size5MB := 5 * 1024 * 1024 - dr := NewDataReader(size5MB, size5MB) - - datafile := "rand.data" - w, err := os.Create(datafile) - if err != nil { - failF("%v: create %v: %v", testname, datafile, err) - return - } - defer w.Close() - - _, err = io.Copy(w, dr) - if err != nil { - failF("%v: write %v: %v", testname, datafile, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - mp1, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: 42, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), - }) - cancel() - if err != nil { - failF("%v: multipart put part 1: %v", testname, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - mp2, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: 96, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), - }) - cancel() - if err != nil { - failF("%v: multipart put part 2: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - MultipartUpload: &types.CompletedMultipartUpload{ - Parts: []types.CompletedPart{ - { - ETag: mp2.ETag, - PartNumber: 96, - }, - { - ETag: mp1.ETag, - PartNumber: 99, - }, - }, - }, - }) - cancel() - if err == nil { - failF("%v: complete multipart expected err", testname) - return - } - - badEtag := "bogusEtagValue" - - // Empty multipart upload - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err == nil { - failF("%v: complete multipart expected err", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - MultipartUpload: &types.CompletedMultipartUpload{ - Parts: []types.CompletedPart{ - { - ETag: mp2.ETag, - PartNumber: 96, - }, - { - ETag: &badEtag, - PartNumber: 42, - }, - }, - }, - }) - cancel() - if err == nil { - failF("%v: complete multipart expected err", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - MultipartUpload: &types.CompletedMultipartUpload{ - Parts: []types.CompletedPart{ - { - ETag: mp1.ETag, - PartNumber: 42, - }, - { - ETag: mp2.ETag, - PartNumber: 96, - }, - }, - }, - }) - cancel() - if err != nil { - failF("%v: complete multipart: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - oi, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - failF("%v: head object %v: %v", testname, obj, err) - return - } - - if oi.ContentLength != (int64(size5MB) * 2) { - failF("%v: object len expected %v, got %v", - testname, int64(size5MB)*2, oi.ContentLength) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestIncompleteMultiParts(s *S3Conf) { - testname := "incomplete multipart parts" - runF(testname) - - bucket := "testbucket9" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - obj := "mympuobject" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, +func GetObjectTagging_non_existing_object(s *S3Conf) { + testName := "GetObjectTagging_non_existing_object" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + return nil }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } - - // upload 2 parts - size5MB := 5 * 1024 * 1024 - size1MB := 1024 * 1024 - dr := NewDataReader(size1MB, size1MB) - - datafile := "rand.data" - w, err := os.Create(datafile) - if err != nil { - failF("%v: create %v: %v", testname, datafile, err) - return - } - defer w.Close() - - _, err = io.Copy(w, dr) - if err != nil { - failF("%v: write %v: %v", testname, datafile, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: 42, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), - }) - cancel() - if err == nil { - failF("%v: multipart put short part expected error", testname) - return - } - - // check list parts does not have incomplete part - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err != nil { - failF("%v: list parts: %v", testname, err) - return - } - - if containsPart(42, lp.Parts) { - failF("%v: list parts: found incomplete part", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestIncompletePutObject(s *S3Conf) { - testname := "test incomplete put object" - runF(testname) +func GetObjectTagging_success(s *S3Conf) { + testName := "PutObjectTagging_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + tagging := types.Tagging{TagSet: []types.Tag{{Key: getPtr("key1"), Value: getPtr("val2")}, {Key: getPtr("key2"), Value: getPtr("val2")}}} + err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } - bucket := "testbucket10" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + Tagging: &tagging}) + cancel() + if err != nil { + return err + } - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return nil + } - // use funny size to prevent accidental alignments - datalen := 1234567 - shortdatalen := 12345 - data := make([]byte, shortdatalen) - rand.Read(data) - r := bytes.NewReader(data) + if !areTagsSame(out.TagSet, tagging.TagSet) { + return fmt.Errorf("expected %v instead got %v", tagging.TagSet, out.TagSet) + } - name := "myobject" - s3client := s3.NewFromConfig(s.Config()) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &name, - Body: r, - ContentLength: int64(datalen), + return nil }) - cancel() - if err == nil { - failF("%v: expected error for short data put", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &name, - }) - defer cancel() - if err == nil { - failF("%v: expected object not exist", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestRangeGet(s *S3Conf) { - testname := "test range get" - runF(testname) - - bucket := "testbucket11" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - datalen := 10 * 1024 - data := make([]byte, datalen) - rand.Read(data) - r := bytes.NewReader(data) - - name := "myobject" - s3client := s3.NewFromConfig(s.Config()) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &name, - Body: r, +func DeleteObjectTagging_non_existing_object(s *S3Conf) { + testName := "DeleteObjectTagging_non_existing_object" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + return nil }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - lgRange := "bytes=20000000-30000000000" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - Range: &lgRange, - }) - cancel() - if err == nil { - failF("%v: expected range error", testname) - return - } - - // Invalid range - invRange := "bytes=100-asd" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - Range: &invRange, - }) - defer cancel() - if err == nil { - failF("%v: expected range error", testname) - return - } - - rangeString := "bytes=100-200" - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - Range: &rangeString, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - defer out.Body.Close() - - if getString(out.ContentRange) != fmt.Sprintf("bytes 100-200/%v", datalen) { - failF("%v: expected content range: %v, instead got: %v", testname, fmt.Sprintf("bytes 100-200/%v", datalen), getString(out.ContentRange)) - return - } - if getString(out.AcceptRanges) != rangeString { - failF("%v: expected accept range: %v, instead got: %v", testname, rangeString, getString(out.AcceptRanges)) - } - - b, err := io.ReadAll(out.Body) - if err != nil { - failF("%v: read body %v", testname, err) - return - } - - // bytes range is inclusive, go range for second value is not - if !isEqual(b, data[100:201]) { - failF("%v: data mismatch of range", testname) - return - } - - rangeString = "bytes=100-" - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - Range: &rangeString, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - defer out.Body.Close() - - b, err = io.ReadAll(out.Body) - if err != nil { - failF("%v: read body %v", testname, err) - return - } - - // bytes range is inclusive, go range for second value is not - if !isEqual(b, data[100:]) { - failF("%v: data mismatch of range", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestInvalidMultiParts(s *S3Conf) { - testname := "invalid multipart parts" - runF(testname) +func DeleteObjectTagging_success(s *S3Conf) { + testName := "DeleteObjectTagging_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + tagging := types.Tagging{TagSet: []types.Tag{{Key: getPtr("key1"), Value: getPtr("val2")}, {Key: getPtr("key2"), Value: getPtr("val2")}}} + err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } - bucket := "bucket12" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + Tagging: &tagging}) + cancel() + if err != nil { + return err + } - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return nil + } - s3client := s3.NewFromConfig(s.Config()) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return nil + } - obj := "mympuobject" + if len(out.TagSet) > 0 { + return fmt.Errorf("expected empty tag set, instead got %v", out.TagSet) + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, + return nil }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } - - // upload 2 parts - size5MB := 5 * 1024 * 1024 - dr := NewDataReader(size5MB, size5MB) - - datafile := "rand.data" - w, err := os.Create(datafile) - if err != nil { - failF("%v: create %v: %v", testname, datafile, err) - return - } - defer w.Close() - - _, err = io.Copy(w, dr) - if err != nil { - failF("%v: write %v: %v", testname, datafile, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: -1, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), - }) - cancel() - if err == nil { - failF("%v: multipart put part 1 expected error", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err == nil { - failF("%v: head object %v expected error", testname, obj) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } +// func TestPutGetMPObject(s *S3Conf) { +// testname := "test put/get multipart object" +// runF(testname) + +// bucket := "testbucket2" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// name := "mympuobject" +// s3client := s3.NewFromConfig(s.Config()) + +// datalen := 10*1024*1024 + 15 +// dr := NewDataReader(datalen, 5*1024*1024) +// WithPartSize(5 * 1024 * 1024) +// s.PartSize = 5 * 1024 * 1024 +// err = s.UploadData(dr, bucket, name) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ +// Bucket: &bucket, +// Key: &name, +// }) +// defer cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// defer out.Body.Close() + +// if out.ContentLength != int64(datalen) { +// failF("%v: content length got %v expected %v", testname, out.ContentLength, datalen) +// return +// } + +// b := make([]byte, 1048576) +// h := sha256.New() +// for { +// n, err := out.Body.Read(b) +// if err == io.EOF { +// h.Write(b[:n]) +// break +// } +// if err != nil { +// failF("%v: read %v", err) +// return +// } +// h.Write(b[:n]) +// } + +// if !isEqual(dr.Sum(), h.Sum(nil)) { +// failF("%v: checksum got %x expected %x", testname, h.Sum(nil), dr.Sum()) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestPutDirObject(s *S3Conf) { +// testname := "test put directory object" +// runF(testname) + +// bucket := "testbucket3" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// name := "myobjectdir/" +// s3client := s3.NewFromConfig(s.Config()) + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ +// Bucket: &bucket, +// Key: &name, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket, MaxKeys: -4}) +// cancel() +// if err == nil { +// failF("%v: expected invalid argument error", testname) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) +// cancel() +// if err != nil { +// failF("failed to list objects: %v", err) +// return +// } + +// if !contains(name, out.Contents) { +// failF("directory object not found") +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestListObject(s *S3Conf) { +// testname := "list objects" +// runF(testname) + +// bucket := "testbucket4" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// dir1 := "myobjectdir/" +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ +// Bucket: &bucket, +// Key: &dir1, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// obj1 := "myobjectdir/myobject" +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ +// Bucket: &bucket, +// Key: &obj1, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// obj2 := "myobjectdir1/myobject" +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ +// Bucket: &bucket, +// Key: &obj2, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// // put: +// // "myobjectdir/" +// // "myobjectdir/myobject" +// // "myobjectdir1/myobject" +// // should return: +// // "myobjectdir/myobject" +// // "myobjectdir1/myobject" + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) +// cancel() +// if err != nil { +// failF("failed to list objects: %v", err) +// return +// } + +// if !contains(obj1, out.Contents) { +// failF("object %v not found", obj1) +// return +// } +// if !contains(obj2, out.Contents) { +// failF("object %v not found", obj2) +// return +// } +// if out.KeyCount != 2 { +// failF("%v: expected key count: %v, instead got: %v", testname, 2, out.KeyCount) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ +// Bucket: &bucket, +// Key: &obj1, +// }) +// cancel() +// if err != nil { +// failF("failed to delete %v: %v", obj1, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ +// Bucket: &bucket, +// Key: &obj2, +// }) +// cancel() +// if err != nil { +// failF("failed to delete %v: %v", obj2, err) +// return +// } + +// // put: +// // "myobjectdir/" +// // "myobjectdir/myobject" +// // "myobjectdir1/myobject" +// // delete: +// // "myobjectdir/myobject" +// // "myobjectdir1/myobject" +// // should return: +// // "myobjectdir/" + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// out, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) +// cancel() +// if err != nil { +// failF("failed to list objects: %v", err) +// return +// } + +// if !contains(dir1, out.Contents) { +// failF("dir %v not found", dir1) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestListAbortMultiPartObject(s *S3Conf) { +// testname := "list/abort multipart objects" +// runF(testname) + +// bucket := "testbucket6" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// obj := "mympuobject" + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: create multipart upload: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// lmpu, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ +// Bucket: &bucket, +// }) +// cancel() +// if err != nil { +// failF("%v: list multipart upload: %v", testname, err) +// return +// } + +// //for _, item := range lmpu.Uploads { +// // fmt.Println(" -- ", *item.Key, *item.UploadId) +// //} + +// if !containsUID(obj, *mpu.UploadId, lmpu.Uploads) { +// failF("%v: upload %v/%v not found", testname, obj, *mpu.UploadId) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// }) +// cancel() +// if err != nil { +// failF("%v: abort multipart upload: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// lmpu, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ +// Bucket: &bucket, +// }) +// cancel() +// if err != nil { +// failF("%v: list multipart upload: %v", testname, err) +// return +// } + +// if len(lmpu.Uploads) != 0 { +// for _, item := range lmpu.Uploads { +// fmt.Println(" D- ", *item.Key, *item.UploadId) +// } +// failF("%v: unexpected multipart uploads found", testname) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestListMultiParts(s *S3Conf) { +// testname := "list multipart parts" +// runF(testname) + +// bucket := "testbucket7" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// obj := "mympuobject" + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: create multipart upload: %v", testname, err) +// return +// } + +// // check list parts of no parts is good +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// }) +// cancel() +// if err != nil { +// failF("%v: list parts: %v", testname, err) +// return +// } + +// if len(lp.Parts) != 0 { +// failF("%v: list parts: expected no parts, got %v", +// testname, len(lp.Parts)) +// return +// } + +// // upload 1 part and check list parts +// size5MB := 5 * 1024 * 1024 +// dr := NewDataReader(size5MB, size5MB) + +// datafile := "rand.data" +// w, err := os.Create(datafile) +// if err != nil { +// failF("%v: create %v: %v", testname, datafile, err) +// return +// } +// defer w.Close() + +// _, err = io.Copy(w, dr) +// if err != nil { +// failF("%v: write %v: %v", testname, datafile, err) +// return +// } + +// _, err = w.Seek(0, io.SeekStart) +// if err != nil { +// failF("%v: seek %v: %v", testname, datafile, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ +// Bucket: &bucket, +// Key: &obj, +// PartNumber: 42, +// UploadId: mpu.UploadId, +// Body: w, +// ContentLength: int64(size5MB), +// }) +// cancel() +// if err != nil { +// failF("%v: multipart put part: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// lp, err = s3client.ListParts(ctx, &s3.ListPartsInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// }) +// cancel() +// if err != nil { +// failF("%v: list parts: %v", testname, err) +// return +// } + +// //for _, part := range lp.Parts { +// // fmt.Println(" -- ", part.PartNumber, part.ETag) +// //} + +// if len(lp.Parts) != 1 || lp.Parts[0].PartNumber != 42 { +// fmt.Printf("%+v, %v, %v\n", lp.Parts, *lp.Key, *lp.UploadId) +// failF("%v: list parts: unexpected parts listing", testname) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestIncorrectMultiParts(s *S3Conf) { +// testname := "incorrect multipart parts" +// runF(testname) + +// bucket := "testbucket8" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// obj := "mympuobject" + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: create multipart upload: %v", testname, err) +// return +// } + +// // upload 2 parts +// size5MB := 5 * 1024 * 1024 +// dr := NewDataReader(size5MB, size5MB) + +// datafile := "rand.data" +// w, err := os.Create(datafile) +// if err != nil { +// failF("%v: create %v: %v", testname, datafile, err) +// return +// } +// defer w.Close() + +// _, err = io.Copy(w, dr) +// if err != nil { +// failF("%v: write %v: %v", testname, datafile, err) +// return +// } + +// _, err = w.Seek(0, io.SeekStart) +// if err != nil { +// failF("%v: seek %v: %v", testname, datafile, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// mp1, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ +// Bucket: &bucket, +// Key: &obj, +// PartNumber: 42, +// UploadId: mpu.UploadId, +// Body: w, +// ContentLength: int64(size5MB), +// }) +// cancel() +// if err != nil { +// failF("%v: multipart put part 1: %v", testname, err) +// return +// } + +// _, err = w.Seek(0, io.SeekStart) +// if err != nil { +// failF("%v: seek %v: %v", testname, datafile, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// mp2, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ +// Bucket: &bucket, +// Key: &obj, +// PartNumber: 96, +// UploadId: mpu.UploadId, +// Body: w, +// ContentLength: int64(size5MB), +// }) +// cancel() +// if err != nil { +// failF("%v: multipart put part 2: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// MultipartUpload: &types.CompletedMultipartUpload{ +// Parts: []types.CompletedPart{ +// { +// ETag: mp2.ETag, +// PartNumber: 96, +// }, +// { +// ETag: mp1.ETag, +// PartNumber: 99, +// }, +// }, +// }, +// }) +// cancel() +// if err == nil { +// failF("%v: complete multipart expected err", testname) +// return +// } + +// badEtag := "bogusEtagValue" + +// // Empty multipart upload +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// }) +// cancel() +// if err == nil { +// failF("%v: complete multipart expected err", testname) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// MultipartUpload: &types.CompletedMultipartUpload{ +// Parts: []types.CompletedPart{ +// { +// ETag: mp2.ETag, +// PartNumber: 96, +// }, +// { +// ETag: &badEtag, +// PartNumber: 42, +// }, +// }, +// }, +// }) +// cancel() +// if err == nil { +// failF("%v: complete multipart expected err", testname) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// MultipartUpload: &types.CompletedMultipartUpload{ +// Parts: []types.CompletedPart{ +// { +// ETag: mp1.ETag, +// PartNumber: 42, +// }, +// { +// ETag: mp2.ETag, +// PartNumber: 96, +// }, +// }, +// }, +// }) +// cancel() +// if err != nil { +// failF("%v: complete multipart: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// oi, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: head object %v: %v", testname, obj, err) +// return +// } + +// if oi.ContentLength != (int64(size5MB) * 2) { +// failF("%v: object len expected %v, got %v", +// testname, int64(size5MB)*2, oi.ContentLength) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestIncompleteMultiParts(s *S3Conf) { +// testname := "incomplete multipart parts" +// runF(testname) + +// bucket := "testbucket9" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// obj := "mympuobject" + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: create multipart upload: %v", testname, err) +// return +// } + +// // upload 2 parts +// size5MB := 5 * 1024 * 1024 +// size1MB := 1024 * 1024 +// dr := NewDataReader(size1MB, size1MB) + +// datafile := "rand.data" +// w, err := os.Create(datafile) +// if err != nil { +// failF("%v: create %v: %v", testname, datafile, err) +// return +// } +// defer w.Close() + +// _, err = io.Copy(w, dr) +// if err != nil { +// failF("%v: write %v: %v", testname, datafile, err) +// return +// } + +// _, err = w.Seek(0, io.SeekStart) +// if err != nil { +// failF("%v: seek %v: %v", testname, datafile, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ +// Bucket: &bucket, +// Key: &obj, +// PartNumber: 42, +// UploadId: mpu.UploadId, +// Body: w, +// ContentLength: int64(size5MB), +// }) +// cancel() +// if err == nil { +// failF("%v: multipart put short part expected error", testname) +// return +// } + +// // check list parts does not have incomplete part +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// }) +// cancel() +// if err != nil { +// failF("%v: list parts: %v", testname, err) +// return +// } + +// if containsPart(42, lp.Parts) { +// failF("%v: list parts: found incomplete part", testname) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestIncompletePutObject(s *S3Conf) { +// testname := "test incomplete put object" +// runF(testname) + +// bucket := "testbucket10" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// // use funny size to prevent accidental alignments +// datalen := 1234567 +// shortdatalen := 12345 +// data := make([]byte, shortdatalen) +// rand.Read(data) +// r := bytes.NewReader(data) + +// name := "myobject" +// s3client := s3.NewFromConfig(s.Config()) + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ +// Bucket: &bucket, +// Key: &name, +// Body: r, +// ContentLength: int64(datalen), +// }) +// cancel() +// if err == nil { +// failF("%v: expected error for short data put", testname) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ +// Bucket: &bucket, +// Key: &name, +// }) +// defer cancel() +// if err == nil { +// failF("%v: expected object not exist", testname) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestInvalidMultiParts(s *S3Conf) { +// testname := "invalid multipart parts" +// runF(testname) + +// bucket := "bucket12" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// obj := "mympuobject" + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: create multipart upload: %v", testname, err) +// return +// } + +// // upload 2 parts +// size5MB := 5 * 1024 * 1024 +// dr := NewDataReader(size5MB, size5MB) + +// datafile := "rand.data" +// w, err := os.Create(datafile) +// if err != nil { +// failF("%v: create %v: %v", testname, datafile, err) +// return +// } +// defer w.Close() + +// _, err = io.Copy(w, dr) +// if err != nil { +// failF("%v: write %v: %v", testname, datafile, err) +// return +// } + +// _, err = w.Seek(0, io.SeekStart) +// if err != nil { +// failF("%v: seek %v: %v", testname, datafile, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ +// Bucket: &bucket, +// Key: &obj, +// PartNumber: -1, +// UploadId: mpu.UploadId, +// Body: w, +// ContentLength: int64(size5MB), +// }) +// cancel() +// if err == nil { +// failF("%v: multipart put part 1 expected error", testname) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err == nil { +// failF("%v: head object %v expected error", testname, obj) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestPutGetRemoveTags(s *S3Conf) { +// testname := "test put/get/remove object tags" +// runF(testname) + +// bucket := "testbucket13" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// obj := "myobject" +// s3client := s3.NewFromConfig(s.Config()) + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// key1 := "hello1" +// key2 := "hello2" +// val1 := "world1" +// val2 := "world2" + +// tagging := types.Tagging{TagSet: []types.Tag{{Key: &key1, Value: &val1}, {Key: &key2, Value: &val2}}} + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ +// Bucket: &bucket, +// Key: &obj, +// Tagging: &tagging, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ +// Key: &obj, +// Bucket: &bucket, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// ok := areTagsSame(tagging.TagSet, out.TagSet) +// if !ok { +// failF("%v: expected %v instead got %v", testname, tagging.TagSet, out.TagSet) +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ +// Key: &obj, +// Bucket: &bucket, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// out, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ +// Key: &obj, +// Bucket: &bucket, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// if len(out.TagSet) > 0 { +// failF("%v: expected empty tag set instead got %v", testname, out.TagSet) +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestAclActions(s *S3Conf) { +// testname := "test put/get acl" +// runF(testname) + +// bucket := "testbucket14" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// rootAccess := s.awsID +// rootSecret := s.awsSecret + +// s.awsID = "grt1" +// s.awsSecret = "grt1secret" + +// userS3Client := s3.NewFromConfig(s.Config()) + +// s.awsID = rootAccess +// s.awsSecret = rootSecret + +// grt1 := "grt1" + +// grants := []types.Grant{ +// { +// Permission: "READ", +// Grantee: &types.Grantee{ +// ID: &grt1, +// Type: "CanonicalUser", +// }, +// }, +// } + +// succUsrCrt := "The user has been created successfully" +// failUsrCrt := "failed to create a user: update iam data: account already exists" + +// out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "create-user", "-a", grt1, "-s", "grt1secret", "-r", "user") +// if err != nil { +// failF("%v: %v", err) +// return +// } +// if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) { +// failF("%v: failed to create user accounts", testname) +// return +// } + +// // Validation error case +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ +// Bucket: &bucket, +// AccessControlPolicy: &types.AccessControlPolicy{ +// Grants: grants, +// }, +// ACL: "private", +// }) +// cancel() +// if err == nil { +// failF("%v: expected validation error", testname) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ +// Bucket: &bucket, +// AccessControlPolicy: &types.AccessControlPolicy{ +// Grants: grants, +// Owner: &types.Owner{ID: &s.awsID}, +// }, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// acl, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{ +// Bucket: &bucket, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// if *acl.Owner.ID != s.awsID { +// failF("%v: expected bucket owner: %v, instead got: %v", testname, s.awsID, *acl.Owner.ID) +// return +// } +// if !checkGrants(acl.Grants, grants) { +// failF("%v: expected %v, instead got %v", testname, grants, acl.Grants) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = userS3Client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ +// Bucket: &bucket, +// }) +// cancel() +// if err == nil { +// failF("%v: expected acl access denied error", testname) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + type prefResult struct { elapsed time.Duration size int64 @@ -1360,233 +1796,3 @@ func TestPerformance(s *S3Conf, upload, download bool, files int, objectSize int return nil } - -func TestPutGetRemoveTags(s *S3Conf) { - testname := "test put/get/remove object tags" - runF(testname) - - bucket := "testbucket13" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - obj := "myobject" - s3client := s3.NewFromConfig(s.Config()) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - key1 := "hello1" - key2 := "hello2" - val1 := "world1" - val2 := "world2" - - tagging := types.Tagging{TagSet: []types.Tag{{Key: &key1, Value: &val1}, {Key: &key2, Value: &val2}}} - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - Tagging: &tagging, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Key: &obj, - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ok := areTagsSame(tagging.TagSet, out.TagSet) - if !ok { - failF("%v: expected %v instead got %v", testname, tagging.TagSet, out.TagSet) - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ - Key: &obj, - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Key: &obj, - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - if len(out.TagSet) > 0 { - failF("%v: expected empty tag set instead got %v", testname, out.TagSet) - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -func TestAclActions(s *S3Conf) { - testname := "test put/get acl" - runF(testname) - - bucket := "testbucket14" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - rootAccess := s.awsID - rootSecret := s.awsSecret - - s.awsID = "grt1" - s.awsSecret = "grt1secret" - - userS3Client := s3.NewFromConfig(s.Config()) - - s.awsID = rootAccess - s.awsSecret = rootSecret - - grt1 := "grt1" - - grants := []types.Grant{ - { - Permission: "READ", - Grantee: &types.Grantee{ - ID: &grt1, - Type: "CanonicalUser", - }, - }, - } - - succUsrCrt := "The user has been created successfully" - failUsrCrt := "failed to create a user: update iam data: account already exists" - - out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "create-user", "-a", grt1, "-s", "grt1secret", "-r", "user") - if err != nil { - failF("%v: %v", err) - return - } - if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) { - failF("%v: failed to create user accounts", testname) - return - } - - // Validation error case - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ - Bucket: &bucket, - AccessControlPolicy: &types.AccessControlPolicy{ - Grants: grants, - }, - ACL: "private", - }) - cancel() - if err == nil { - failF("%v: expected validation error", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ - Bucket: &bucket, - AccessControlPolicy: &types.AccessControlPolicy{ - Grants: grants, - Owner: &types.Owner{ID: &s.awsID}, - }, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - acl, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - if *acl.Owner.ID != s.awsID { - failF("%v: expected bucket owner: %v, instead got: %v", testname, s.awsID, *acl.Owner.ID) - return - } - if !checkGrants(acl.Grants, grants) { - failF("%v: expected %v, instead got %v", testname, grants, acl.Grants) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userS3Client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ - Bucket: &bucket, - }) - cancel() - if err == nil { - failF("%v: expected acl access denied error", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -// Full flow test -func TestFullFlow(s *S3Conf) { - // TODO: add more test cases to get 100% coverage - TestMakeBucket(s) - TestPutGetObject(s) - TestPutGetMPObject(s) - TestPutDirObject(s) - TestListObject(s) - TestIncompletePutObject(s) - TestListMultiParts(s) - TestIncompleteMultiParts(s) - TestIncorrectMultiParts(s) - TestListAbortMultiPartObject(s) - TestRangeGet(s) - TestInvalidMultiParts(s) - TestPutGetRemoveTags(s) - TestAclActions(s) -} diff --git a/integration/utils.go b/integration/utils.go index 46ba3db6..dff7b6d6 100644 --- a/integration/utils.go +++ b/integration/utils.go @@ -1,15 +1,30 @@ package integration import ( + "bytes" "context" + "crypto/rand" + "crypto/sha256" + "errors" "fmt" "os/exec" "strings" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/versity/versitygw/s3err" ) +var ( + bcktCount = 0 +) + +func getBucketName() string { + bcktCount++ + return fmt.Sprintf("test-bucket-%v", bcktCount) +} + func setup(s *S3Conf, bucket string) error { s3client := s3.NewFromConfig(s.Config()) @@ -69,6 +84,78 @@ func teardown(s *S3Conf, bucket string) error { return err } +func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) { + runF(testName) + bucketName := getBucketName() + err := setup(s, bucketName) + if err != nil { + failF("%v: failed to create a bucket: %v", testName, err.Error()) + return + } + client := s3.NewFromConfig(s.Config()) + handlerErr := handler(client, bucketName) + if handlerErr != nil { + failF("%v: %v", testName, handlerErr.Error()) + } + + err = teardown(s, bucketName) + if err != nil { + if handlerErr == nil { + failF("%v: failed to delete the bucket: %v", testName, err.Error()) + } else { + fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err.Error()) + } + } + if handlerErr == nil { + passF(testName) + } +} + +func putObjects(client *s3.Client, objs []string, bucket string) error { + for _, key := range objs { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Key: &key, + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + } + return nil +} + +func checkApiErr(err error, apiErr s3err.APIError) error { + if err == nil { + return fmt.Errorf("expected %v, instead got nil", apiErr.Code) + } + var ae smithy.APIError + if errors.As(err, &ae) { + if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description { + return nil + } + + return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode()) + } else { + return fmt.Errorf("expected aws api error, instead got: %v", err.Error()) + } +} + +func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) { + data = make([]byte, lgth) + rand.Read(data) + csum = sha256.Sum256(data) + r := bytes.NewReader(data) + input.Body = r + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = client.PutObject(ctx, input) + cancel() + + return +} + func isEqual(a, b []byte) bool { if len(a) != len(b) { return false @@ -162,6 +249,10 @@ func getString(str *string) string { return *str } +func getPtr(str string) *string { + return &str +} + func areMapsSame(mp1, mp2 map[string]string) bool { if len(mp1) != len(mp2) { return false From da3ad55483531444154eb30a804b742934b1d557 Mon Sep 17 00:00:00 2001 From: jonaustin09 Date: Fri, 11 Aug 2023 23:33:50 +0400 Subject: [PATCH 2/4] feat: Added integration test cases for HeadBucket, HeadObject, DeleteObject, DeleteObjects, ListObjects --- integration/action-tests.go | 35 ++ integration/tests.go | 690 +++++++++++++++++++++--------------- integration/utils.go | 52 ++- 3 files changed, 494 insertions(+), 283 deletions(-) diff --git a/integration/action-tests.go b/integration/action-tests.go index f6f2ae4f..d94178b0 100644 --- a/integration/action-tests.go +++ b/integration/action-tests.go @@ -6,6 +6,11 @@ func TestCreateBucket(s *S3Conf) { CreateDeleteBucket_success(s) } +func TestHeadBucket(s *S3Conf) { + HeadBucket_non_existing_bucket(s) + HeadBucket_success(s) +} + func TestDeleteBucket(s *S3Conf) { DeleteBucket_non_existing_bucket(s) DeleteBucket_non_empty_bucket(s) @@ -19,6 +24,11 @@ func TestPutObject(s *S3Conf) { PutObject_success(s) } +func TestHeadObject(s *S3Conf) { + HeadObject_non_existing_object(s) + HeadObject_success(s) +} + func TestGetObject(s *S3Conf) { GetObject_non_existing_key(s) GetObject_invalid_ranges(s) @@ -27,6 +37,26 @@ func TestGetObject(s *S3Conf) { GetObject_by_range_success(s) } +func TestListObjects(s *S3Conf) { + ListObjects_non_existing_bucket(s) + ListObjects_with_prefix(s) + ListObject_truncated(s) + ListObjects_invalid_max_keys(s) + ListObjects_max_keys_0(s) +} + +func TestDeleteObject(s *S3Conf) { + DeleteObject_non_existing_object(s) + DeleteObject_success(s) +} + +func TestDeleteObjects(s *S3Conf) { + DeleteObjects_empty_input(s) + //TODO: Uncomment this after fixing the bug: #195 + // DeleteObjects_non_existing_objects(s) + DeleteObjects_success(s) +} + func TestCopyObject(s *S3Conf) { CopyObject_non_existing_dst_bucket(s) CopyObject_success(s) @@ -49,9 +79,14 @@ func TestDeleteObjectTagging(s *S3Conf) { func TestFullFlow(s *S3Conf) { TestCreateBucket(s) + TestHeadBucket(s) TestDeleteBucket(s) TestPutObject(s) + TestHeadObject(s) TestGetObject(s) + TestListObjects(s) + TestDeleteObject(s) + TestDeleteObjects(s) TestCopyObject(s) TestPutObjectTagging(s) TestDeleteObjectTagging(s) diff --git a/integration/tests.go b/integration/tests.go index f4c0aa68..e7ba7ee9 100644 --- a/integration/tests.go +++ b/integration/tests.go @@ -71,6 +71,38 @@ func CreateBucket_existing_bucket(s *S3Conf) { passF(testName) } +func HeadBucket_non_existing_bucket(s *S3Conf) { + testName := "HeadBucket_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bcktName := getBucketName() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.HeadBucket(ctx, &s3.HeadBucketInput{ + Bucket: &bcktName, + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + return nil + }) +} + +func HeadBucket_success(s *S3Conf) { + testName := "HeadBucket_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.HeadBucket(ctx, &s3.HeadBucketInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + return nil + }) +} + func CreateDeleteBucket_success(s *S3Conf) { testName := "CreateBucket_success" runF(testName) @@ -184,6 +216,57 @@ func PutObject_success(s *S3Conf) { }) } +func HeadObject_non_existing_object(s *S3Conf) { + testName := "HeadObject_non_existing_object" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + }) + cancel() + if err := checkSdkApiErr(err, "NotFound"); err != nil { + return err + } + return nil + }) +} + +func HeadObject_success(s *S3Conf) { + testName := "HeadObject_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, dataLen := "my-obj", int64(1234567) + meta := map[string]string{ + "key1": "val1", + "key2": "val2", + } + + _, _, err := putObjectWithData(dataLen, &s3.PutObjectInput{Bucket: &bucket, Key: &obj, Metadata: meta}, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + + if !areMapsSame(out.Metadata, meta) { + return fmt.Errorf("incorrect object metadata") + } + if out.ContentLength != dataLen { + return fmt.Errorf("expected data length %v, instead got %v", dataLen, out.ContentLength) + } + + return nil + }) +} + func GetObject_non_existing_key(s *S3Conf) { testName := "GetObject_non_existing_key" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -390,6 +473,331 @@ func GetObject_by_range_success(s *S3Conf) { }) } +func ListObjects_non_existing_bucket(s *S3Conf) { + testName := "ListObjects_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bckt := getBucketName() + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bckt, + }) + cancel() + if err := checkSdkApiErr(err, "NoSuchBucket"); err != nil { + return err + } + return nil + }) +} + +func ListObjects_with_prefix(s *S3Conf) { + testName := "ListObjects_with_prefix" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + prefix := "obj" + objWithPrefix := []string{prefix + "/foo", prefix + "/bar", prefix + "/baz/bla"} + err := putObjects(s3client, append(objWithPrefix, []string{"xzy/csf", "hell"}...), bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + Prefix: &prefix, + }) + cancel() + if err != nil { + return err + } + + if *out.Prefix != prefix { + return fmt.Errorf("expected prefix %v, instead got %v", prefix, *out.Prefix) + } + if !compareObjects(objWithPrefix, out.Contents) { + return fmt.Errorf("unexpected output for list objects with prefix") + } + + return nil + }) +} + +func ListObject_truncated(s *S3Conf) { + testName := "ListObject_truncated" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + maxKeys := int32(2) + err := putObjects(s3client, []string{"foo", "bar", "baz"}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + MaxKeys: maxKeys, + }) + cancel() + if err != nil { + return err + } + + if !out.IsTruncated { + return fmt.Errorf("expected output to be truncated") + } + + if out.MaxKeys != maxKeys { + return fmt.Errorf("expected max-keys to be %v, instead got %v", maxKeys, out.MaxKeys) + } + + if !compareObjects([]string{"bar", "baz"}, out.Contents) { + return fmt.Errorf("unexpected output for list objects with max-keys") + } + + //TODO: Add next marker checker after bug-fixing + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err = s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + Marker: out.NextMarker, + }) + cancel() + if err != nil { + return err + } + + if out.IsTruncated { + return fmt.Errorf("expected output not to be truncated") + } + + if !compareObjects([]string{"foo"}, out.Contents) { + return fmt.Errorf("unexpected output for list objects with max-keys") + } + return nil + }) +} + +func ListObjects_invalid_max_keys(s *S3Conf) { + testName := "ListObjects_invalid_max_keys" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + MaxKeys: -5, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidMaxKeys)); err != nil { + return err + } + + return nil + }) +} + +func ListObjects_max_keys_0(s *S3Conf) { + testName := "ListObjects_max_keys_0" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objects := []string{"foo", "bar", "baz"} + err := putObjects(s3client, objects, bucket) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + MaxKeys: 0, + }) + cancel() + if err != nil { + return nil + } + + if !compareObjects(objects, out.Contents) { + return fmt.Errorf("unexpected output for list objects with max-keys 0") + } + + return nil + }) +} + +//TODO: Add a test case for delimiter after buf-fixing, as delimiter doesn't work as intended + +func DeleteObject_non_existing_object(s *S3Conf) { + testName := "DeleteObject_non_existing_object" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + return nil + }) +} + +func DeleteObject_success(s *S3Conf) { + testName := "DeleteObject_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err := checkSdkApiErr(err, "NoSuchKey"); err != nil { + return err + } + return nil + }) +} + +func DeleteObjects_empty_input(s *S3Conf) { + testName := "DeleteObjects_empty_input" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objects := []string{"foo", "bar", "baz"} + err := putObjects(s3client, objects, bucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: []types.ObjectIdentifier{}, + }, + }) + cancel() + if err != nil { + return err + } + + if len(out.Deleted) != 0 { + return fmt.Errorf("expected deleted object count 0, instead got %v", len(out.Deleted)) + } + if len(out.Errors) != 0 { + return fmt.Errorf("expected 0 errors, instead got %v", len(out.Errors)) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(objects, res.Contents) { + return fmt.Errorf("unexpected output for list objects with prefix") + } + + return nil + }) +} + +//TODO: Uncomment the test after fixing the bug: #195 +// func DeleteObjects_non_existing_objects(s *S3Conf) { +// testName := "DeleteObjects_empty_input" +// actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { +// delObjects := []types.ObjectIdentifier{{Key: getPtr("obj1")}, {Key: getPtr("obj2")}} +// +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ +// Bucket: &bucket, +// Delete: &types.Delete{ +// Objects: delObjects, +// }, +// }) +// cancel() +// if err != nil { +// return err +// } + +// if len(out.Deleted) != 0 { +// return fmt.Errorf("expected deleted object count 0, instead got %v", len(out.Deleted)) +// } +// if len(out.Errors) != 2 { +// return fmt.Errorf("expected 2 errors, instead got %v", len(out.Errors)) +// } + +// for _, delErr := range out.Errors { +// if *delErr.Code != "NoSuchKey" { +// return fmt.Errorf("expected NoSuchKey error, instead got %v", *delErr.Code) +// } +// } + +// return nil +// }) +// } + +func DeleteObjects_success(s *S3Conf) { + testName := "DeleteObjects_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + objects, objToDel := []string{"obj1", "obj2", "obj3"}, []string{"foo", "bar", "baz"} + err := putObjects(s3client, append(objToDel, objects...), bucket) + + delObjects := []types.ObjectIdentifier{} + for _, key := range objToDel { + k := key + delObjects = append(delObjects, types.ObjectIdentifier{Key: &k}) + } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &bucket, + Delete: &types.Delete{ + Objects: delObjects, + }, + }) + cancel() + if err != nil { + return err + } + + if len(out.Deleted) != 3 { + return fmt.Errorf("expected deleted object count 3, instead got %v", len(out.Deleted)) + } + if len(out.Errors) != 0 { + return fmt.Errorf("expected 2 errors, instead got %v", len(out.Errors)) + } + + if !compareDelObjects(objToDel, out.Deleted) { + return fmt.Errorf("unexpected deleted output") + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if !compareObjects(objects, res.Contents) { + return fmt.Errorf("unexpected output for list objects with prefix") + } + + return nil + }) +} + func CopyObject_non_existing_dst_bucket(s *S3Conf) { testName := "CopyObject_non_existing_dst_bucket" actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -698,193 +1106,6 @@ func DeleteObjectTagging_success(s *S3Conf) { // passF(testname) // } -// func TestPutDirObject(s *S3Conf) { -// testname := "test put directory object" -// runF(testname) - -// bucket := "testbucket3" - -// err := setup(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// name := "myobjectdir/" -// s3client := s3.NewFromConfig(s.Config()) - -// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ -// Bucket: &bucket, -// Key: &name, -// }) -// cancel() -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket, MaxKeys: -4}) -// cancel() -// if err == nil { -// failF("%v: expected invalid argument error", testname) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) -// cancel() -// if err != nil { -// failF("failed to list objects: %v", err) -// return -// } - -// if !contains(name, out.Contents) { -// failF("directory object not found") -// return -// } - -// err = teardown(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } -// passF(testname) -// } - -// func TestListObject(s *S3Conf) { -// testname := "list objects" -// runF(testname) - -// bucket := "testbucket4" - -// err := setup(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// s3client := s3.NewFromConfig(s.Config()) - -// dir1 := "myobjectdir/" -// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ -// Bucket: &bucket, -// Key: &dir1, -// }) -// cancel() -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// obj1 := "myobjectdir/myobject" -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ -// Bucket: &bucket, -// Key: &obj1, -// }) -// cancel() -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// obj2 := "myobjectdir1/myobject" -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ -// Bucket: &bucket, -// Key: &obj2, -// }) -// cancel() -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// // put: -// // "myobjectdir/" -// // "myobjectdir/myobject" -// // "myobjectdir1/myobject" -// // should return: -// // "myobjectdir/myobject" -// // "myobjectdir1/myobject" - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) -// cancel() -// if err != nil { -// failF("failed to list objects: %v", err) -// return -// } - -// if !contains(obj1, out.Contents) { -// failF("object %v not found", obj1) -// return -// } -// if !contains(obj2, out.Contents) { -// failF("object %v not found", obj2) -// return -// } -// if out.KeyCount != 2 { -// failF("%v: expected key count: %v, instead got: %v", testname, 2, out.KeyCount) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ -// Bucket: &bucket, -// Key: &obj1, -// }) -// cancel() -// if err != nil { -// failF("failed to delete %v: %v", obj1, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ -// Bucket: &bucket, -// Key: &obj2, -// }) -// cancel() -// if err != nil { -// failF("failed to delete %v: %v", obj2, err) -// return -// } - -// // put: -// // "myobjectdir/" -// // "myobjectdir/myobject" -// // "myobjectdir1/myobject" -// // delete: -// // "myobjectdir/myobject" -// // "myobjectdir1/myobject" -// // should return: -// // "myobjectdir/" - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// out, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) -// cancel() -// if err != nil { -// failF("failed to list objects: %v", err) -// return -// } - -// if !contains(dir1, out.Contents) { -// failF("dir %v not found", dir1) -// return -// } - -// err = teardown(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } -// passF(testname) -// } - // func TestListAbortMultiPartObject(s *S3Conf) { // testname := "list/abort multipart objects" // runF(testname) @@ -1515,101 +1736,6 @@ func DeleteObjectTagging_success(s *S3Conf) { // passF(testname) // } -// func TestPutGetRemoveTags(s *S3Conf) { -// testname := "test put/get/remove object tags" -// runF(testname) - -// bucket := "testbucket13" - -// err := setup(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// obj := "myobject" -// s3client := s3.NewFromConfig(s.Config()) - -// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ -// Bucket: &bucket, -// Key: &obj, -// }) -// cancel() -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// key1 := "hello1" -// key2 := "hello2" -// val1 := "world1" -// val2 := "world2" - -// tagging := types.Tagging{TagSet: []types.Tag{{Key: &key1, Value: &val1}, {Key: &key2, Value: &val2}}} - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ -// Bucket: &bucket, -// Key: &obj, -// Tagging: &tagging, -// }) -// cancel() -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ -// Key: &obj, -// Bucket: &bucket, -// }) -// cancel() -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// ok := areTagsSame(tagging.TagSet, out.TagSet) -// if !ok { -// failF("%v: expected %v instead got %v", testname, tagging.TagSet, out.TagSet) -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ -// Key: &obj, -// Bucket: &bucket, -// }) -// cancel() -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// out, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ -// Key: &obj, -// Bucket: &bucket, -// }) -// cancel() -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// if len(out.TagSet) > 0 { -// failF("%v: expected empty tag set instead got %v", testname, out.TagSet) -// } - -// err = teardown(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } -// passF(testname) -// } - // func TestAclActions(s *S3Conf) { // testname := "test put/get acl" // runF(testname) diff --git a/integration/utils.go b/integration/utils.go index dff7b6d6..d76233b8 100644 --- a/integration/utils.go +++ b/integration/utils.go @@ -142,6 +142,17 @@ func checkApiErr(err error, apiErr s3err.APIError) error { } } +func checkSdkApiErr(err error, code string) error { + var ae smithy.APIError + if errors.As(err, &ae) { + if ae.ErrorCode() != code { + return fmt.Errorf("expected %v, instead got %v", ae.ErrorCode(), code) + } + return nil + } + return err +} + func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) { data = make([]byte, lgth) rand.Read(data) @@ -172,7 +183,6 @@ func isEqual(a, b []byte) bool { func contains(name string, list []types.Object) bool { for _, item := range list { - fmt.Println(*item.Key) if strings.EqualFold(name, *item.Key) { return true } @@ -264,3 +274,43 @@ func areMapsSame(mp1, mp2 map[string]string) bool { } return true } + +func compareObjects(list1 []string, list2 []types.Object) bool { + if len(list1) != len(list2) { + return false + } + + elementMap := make(map[string]bool) + + for _, elem := range list1 { + elementMap[elem] = true + } + + for _, elem := range list2 { + if _, found := elementMap[*elem.Key]; !found { + return false + } + } + + return true +} + +func compareDelObjects(list1 []string, list2 []types.DeletedObject) bool { + if len(list1) != len(list2) { + return false + } + + elementMap := make(map[string]bool) + + for _, elem := range list1 { + elementMap[elem] = true + } + + for _, elem := range list2 { + if _, found := elementMap[*elem.Key]; !found { + return false + } + } + + return true +} From 22227c875a28cb7d7653ede0f9f737a3ad6f4ec8 Mon Sep 17 00:00:00 2001 From: jonaustin09 Date: Tue, 22 Aug 2023 20:20:27 +0400 Subject: [PATCH 3/4] feat: Added integration test cases for CreateMultipartUpload, UploadPart, UploadPartCopy, ListParts, ListMultipartUpload, CompleteMultipartUpload --- integration/action-tests.go | 59 ++ integration/tests.go | 1754 +++++++++++++++++++++-------------- integration/utils.go | 89 +- 3 files changed, 1184 insertions(+), 718 deletions(-) diff --git a/integration/action-tests.go b/integration/action-tests.go index d94178b0..59e038d8 100644 --- a/integration/action-tests.go +++ b/integration/action-tests.go @@ -77,6 +77,58 @@ func TestDeleteObjectTagging(s *S3Conf) { DeleteObjectTagging_success(s) } +func TestCreateMultipartUpload(s *S3Conf) { + CreateMultipartUpload_non_existing_bucket(s) + CreateMultipartUpload_success(s) +} + +func TestUploadPart(s *S3Conf) { + UploadPart_non_existing_bucket(s) + UploadPart_invalid_part_number(s) + UploadPart_non_existing_key(s) + UploadPart_non_existing_mp_upload(s) + UploadPart_success(s) +} + +func TestUploadPartCopy(s *S3Conf) { + UploadPartCopy_non_existing_bucket(s) + UploadPartCopy_incorrect_uploadId(s) + UploadPartCopy_incorrect_object_key(s) + UploadPartCopy_invalid_part_number(s) + UploadPartCopy_invalid_copy_source(s) + UploadPartCopy_non_existing_source_bucket(s) + UploadPartCopy_non_existing_source_object_key(s) + UploadPartCopy_success(s) + UploadPartCopy_by_range_invalid_range(s) + UploadPartCopy_by_range_success(s) +} + +func TestListParts(s *S3Conf) { + ListParts_incorrect_uploadId(s) + ListParts_incorrect_object_key(s) + ListParts_success(s) +} + +func TestListMultipartUploads(s *S3Conf) { + ListMultipartUploads_non_existing_bucket(s) + ListMultipartUploads_empty_result(s) + ListMultipartUploads_success(s) +} + +func TestAbortMultipartUpload(s *S3Conf) { + AbortMultipartUpload_non_existing_bucket(s) + AbortMultipartUpload_incorrect_uploadId(s) + AbortMultipartUpload_incorrect_object_key(s) + AbortMultipartUpload_success(s) +} + +func TestCompleteMultipartUpload(s *S3Conf) { + CompletedMultipartUpload_non_existing_bucket(s) + CompleteMultipartUpload_invalid_part_number(s) + CompleteMultipartUpload_invalid_ETag(s) + CompleteMultipartUpload_success(s) +} + func TestFullFlow(s *S3Conf) { TestCreateBucket(s) TestHeadBucket(s) @@ -90,4 +142,11 @@ func TestFullFlow(s *S3Conf) { TestCopyObject(s) TestPutObjectTagging(s) TestDeleteObjectTagging(s) + TestCreateMultipartUpload(s) + TestUploadPart(s) + TestUploadPartCopy(s) + TestListParts(s) + TestListMultipartUploads(s) + TestAbortMultipartUpload(s) + TestCompleteMultipartUpload(s) } diff --git a/integration/tests.go b/integration/tests.go index e7ba7ee9..0c7090c4 100644 --- a/integration/tests.go +++ b/integration/tests.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/google/uuid" "github.com/versity/versitygw/s3err" ) @@ -617,7 +618,7 @@ func ListObjects_max_keys_0(s *S3Conf) { }) } -//TODO: Add a test case for delimiter after buf-fixing, as delimiter doesn't work as intended +//TODO: Add a test case for delimiter after bug-fixing, as delimiter doesn't work as intended func DeleteObject_non_existing_object(s *S3Conf) { testName := "DeleteObject_non_existing_object" @@ -1036,705 +1037,1058 @@ func DeleteObjectTagging_success(s *S3Conf) { }) } -// func TestPutGetMPObject(s *S3Conf) { -// testname := "test put/get multipart object" -// runF(testname) - -// bucket := "testbucket2" - -// err := setup(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// name := "mympuobject" -// s3client := s3.NewFromConfig(s.Config()) - -// datalen := 10*1024*1024 + 15 -// dr := NewDataReader(datalen, 5*1024*1024) -// WithPartSize(5 * 1024 * 1024) -// s.PartSize = 5 * 1024 * 1024 -// err = s.UploadData(dr, bucket, name) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) -// out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ -// Bucket: &bucket, -// Key: &name, -// }) -// defer cancel() -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } -// defer out.Body.Close() - -// if out.ContentLength != int64(datalen) { -// failF("%v: content length got %v expected %v", testname, out.ContentLength, datalen) -// return -// } - -// b := make([]byte, 1048576) -// h := sha256.New() -// for { -// n, err := out.Body.Read(b) -// if err == io.EOF { -// h.Write(b[:n]) -// break -// } -// if err != nil { -// failF("%v: read %v", err) -// return -// } -// h.Write(b[:n]) -// } - -// if !isEqual(dr.Sum(), h.Sum(nil)) { -// failF("%v: checksum got %x expected %x", testname, h.Sum(nil), dr.Sum()) -// return -// } - -// err = teardown(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } -// passF(testname) -// } - -// func TestListAbortMultiPartObject(s *S3Conf) { -// testname := "list/abort multipart objects" -// runF(testname) - -// bucket := "testbucket6" - -// err := setup(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// s3client := s3.NewFromConfig(s.Config()) - -// obj := "mympuobject" - -// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) -// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ -// Bucket: &bucket, -// Key: &obj, -// }) -// cancel() -// if err != nil { -// failF("%v: create multipart upload: %v", testname, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// lmpu, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ -// Bucket: &bucket, -// }) -// cancel() -// if err != nil { -// failF("%v: list multipart upload: %v", testname, err) -// return -// } - -// //for _, item := range lmpu.Uploads { -// // fmt.Println(" -- ", *item.Key, *item.UploadId) -// //} - -// if !containsUID(obj, *mpu.UploadId, lmpu.Uploads) { -// failF("%v: upload %v/%v not found", testname, obj, *mpu.UploadId) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ -// Bucket: &bucket, -// Key: &obj, -// UploadId: mpu.UploadId, -// }) -// cancel() -// if err != nil { -// failF("%v: abort multipart upload: %v", testname, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// lmpu, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ -// Bucket: &bucket, -// }) -// cancel() -// if err != nil { -// failF("%v: list multipart upload: %v", testname, err) -// return -// } - -// if len(lmpu.Uploads) != 0 { -// for _, item := range lmpu.Uploads { -// fmt.Println(" D- ", *item.Key, *item.UploadId) -// } -// failF("%v: unexpected multipart uploads found", testname) -// return -// } - -// err = teardown(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } -// passF(testname) -// } - -// func TestListMultiParts(s *S3Conf) { -// testname := "list multipart parts" -// runF(testname) - -// bucket := "testbucket7" - -// err := setup(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// s3client := s3.NewFromConfig(s.Config()) - -// obj := "mympuobject" - -// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) -// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ -// Bucket: &bucket, -// Key: &obj, -// }) -// cancel() -// if err != nil { -// failF("%v: create multipart upload: %v", testname, err) -// return -// } - -// // check list parts of no parts is good -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ -// Bucket: &bucket, -// Key: &obj, -// UploadId: mpu.UploadId, -// }) -// cancel() -// if err != nil { -// failF("%v: list parts: %v", testname, err) -// return -// } - -// if len(lp.Parts) != 0 { -// failF("%v: list parts: expected no parts, got %v", -// testname, len(lp.Parts)) -// return -// } - -// // upload 1 part and check list parts -// size5MB := 5 * 1024 * 1024 -// dr := NewDataReader(size5MB, size5MB) - -// datafile := "rand.data" -// w, err := os.Create(datafile) -// if err != nil { -// failF("%v: create %v: %v", testname, datafile, err) -// return -// } -// defer w.Close() - -// _, err = io.Copy(w, dr) -// if err != nil { -// failF("%v: write %v: %v", testname, datafile, err) -// return -// } - -// _, err = w.Seek(0, io.SeekStart) -// if err != nil { -// failF("%v: seek %v: %v", testname, datafile, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ -// Bucket: &bucket, -// Key: &obj, -// PartNumber: 42, -// UploadId: mpu.UploadId, -// Body: w, -// ContentLength: int64(size5MB), -// }) -// cancel() -// if err != nil { -// failF("%v: multipart put part: %v", testname, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// lp, err = s3client.ListParts(ctx, &s3.ListPartsInput{ -// Bucket: &bucket, -// Key: &obj, -// UploadId: mpu.UploadId, -// }) -// cancel() -// if err != nil { -// failF("%v: list parts: %v", testname, err) -// return -// } - -// //for _, part := range lp.Parts { -// // fmt.Println(" -- ", part.PartNumber, part.ETag) -// //} - -// if len(lp.Parts) != 1 || lp.Parts[0].PartNumber != 42 { -// fmt.Printf("%+v, %v, %v\n", lp.Parts, *lp.Key, *lp.UploadId) -// failF("%v: list parts: unexpected parts listing", testname) -// return -// } - -// err = teardown(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } -// passF(testname) -// } - -// func TestIncorrectMultiParts(s *S3Conf) { -// testname := "incorrect multipart parts" -// runF(testname) - -// bucket := "testbucket8" - -// err := setup(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// s3client := s3.NewFromConfig(s.Config()) - -// obj := "mympuobject" - -// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) -// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ -// Bucket: &bucket, -// Key: &obj, -// }) -// cancel() -// if err != nil { -// failF("%v: create multipart upload: %v", testname, err) -// return -// } - -// // upload 2 parts -// size5MB := 5 * 1024 * 1024 -// dr := NewDataReader(size5MB, size5MB) - -// datafile := "rand.data" -// w, err := os.Create(datafile) -// if err != nil { -// failF("%v: create %v: %v", testname, datafile, err) -// return -// } -// defer w.Close() - -// _, err = io.Copy(w, dr) -// if err != nil { -// failF("%v: write %v: %v", testname, datafile, err) -// return -// } - -// _, err = w.Seek(0, io.SeekStart) -// if err != nil { -// failF("%v: seek %v: %v", testname, datafile, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// mp1, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ -// Bucket: &bucket, -// Key: &obj, -// PartNumber: 42, -// UploadId: mpu.UploadId, -// Body: w, -// ContentLength: int64(size5MB), -// }) -// cancel() -// if err != nil { -// failF("%v: multipart put part 1: %v", testname, err) -// return -// } - -// _, err = w.Seek(0, io.SeekStart) -// if err != nil { -// failF("%v: seek %v: %v", testname, datafile, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// mp2, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ -// Bucket: &bucket, -// Key: &obj, -// PartNumber: 96, -// UploadId: mpu.UploadId, -// Body: w, -// ContentLength: int64(size5MB), -// }) -// cancel() -// if err != nil { -// failF("%v: multipart put part 2: %v", testname, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ -// Bucket: &bucket, -// Key: &obj, -// UploadId: mpu.UploadId, -// MultipartUpload: &types.CompletedMultipartUpload{ -// Parts: []types.CompletedPart{ -// { -// ETag: mp2.ETag, -// PartNumber: 96, -// }, -// { -// ETag: mp1.ETag, -// PartNumber: 99, -// }, -// }, -// }, -// }) -// cancel() -// if err == nil { -// failF("%v: complete multipart expected err", testname) -// return -// } - -// badEtag := "bogusEtagValue" - -// // Empty multipart upload -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ -// Bucket: &bucket, -// Key: &obj, -// UploadId: mpu.UploadId, -// }) -// cancel() -// if err == nil { -// failF("%v: complete multipart expected err", testname) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ -// Bucket: &bucket, -// Key: &obj, -// UploadId: mpu.UploadId, -// MultipartUpload: &types.CompletedMultipartUpload{ -// Parts: []types.CompletedPart{ -// { -// ETag: mp2.ETag, -// PartNumber: 96, -// }, -// { -// ETag: &badEtag, -// PartNumber: 42, -// }, -// }, -// }, -// }) -// cancel() -// if err == nil { -// failF("%v: complete multipart expected err", testname) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ -// Bucket: &bucket, -// Key: &obj, -// UploadId: mpu.UploadId, -// MultipartUpload: &types.CompletedMultipartUpload{ -// Parts: []types.CompletedPart{ -// { -// ETag: mp1.ETag, -// PartNumber: 42, -// }, -// { -// ETag: mp2.ETag, -// PartNumber: 96, -// }, -// }, -// }, -// }) -// cancel() -// if err != nil { -// failF("%v: complete multipart: %v", testname, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// oi, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ -// Bucket: &bucket, -// Key: &obj, -// }) -// cancel() -// if err != nil { -// failF("%v: head object %v: %v", testname, obj, err) -// return -// } - -// if oi.ContentLength != (int64(size5MB) * 2) { -// failF("%v: object len expected %v, got %v", -// testname, int64(size5MB)*2, oi.ContentLength) -// return -// } - -// err = teardown(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } -// passF(testname) -// } - -// func TestIncompleteMultiParts(s *S3Conf) { -// testname := "incomplete multipart parts" -// runF(testname) - -// bucket := "testbucket9" - -// err := setup(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// s3client := s3.NewFromConfig(s.Config()) - -// obj := "mympuobject" - -// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) -// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ -// Bucket: &bucket, -// Key: &obj, -// }) -// cancel() -// if err != nil { -// failF("%v: create multipart upload: %v", testname, err) -// return -// } - -// // upload 2 parts -// size5MB := 5 * 1024 * 1024 -// size1MB := 1024 * 1024 -// dr := NewDataReader(size1MB, size1MB) - -// datafile := "rand.data" -// w, err := os.Create(datafile) -// if err != nil { -// failF("%v: create %v: %v", testname, datafile, err) -// return -// } -// defer w.Close() - -// _, err = io.Copy(w, dr) -// if err != nil { -// failF("%v: write %v: %v", testname, datafile, err) -// return -// } - -// _, err = w.Seek(0, io.SeekStart) -// if err != nil { -// failF("%v: seek %v: %v", testname, datafile, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ -// Bucket: &bucket, -// Key: &obj, -// PartNumber: 42, -// UploadId: mpu.UploadId, -// Body: w, -// ContentLength: int64(size5MB), -// }) -// cancel() -// if err == nil { -// failF("%v: multipart put short part expected error", testname) -// return -// } - -// // check list parts does not have incomplete part -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ -// Bucket: &bucket, -// Key: &obj, -// UploadId: mpu.UploadId, -// }) -// cancel() -// if err != nil { -// failF("%v: list parts: %v", testname, err) -// return -// } - -// if containsPart(42, lp.Parts) { -// failF("%v: list parts: found incomplete part", testname) -// return -// } - -// err = teardown(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } -// passF(testname) -// } - -// func TestIncompletePutObject(s *S3Conf) { -// testname := "test incomplete put object" -// runF(testname) - -// bucket := "testbucket10" - -// err := setup(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// // use funny size to prevent accidental alignments -// datalen := 1234567 -// shortdatalen := 12345 -// data := make([]byte, shortdatalen) -// rand.Read(data) -// r := bytes.NewReader(data) - -// name := "myobject" -// s3client := s3.NewFromConfig(s.Config()) - -// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ -// Bucket: &bucket, -// Key: &name, -// Body: r, -// ContentLength: int64(datalen), -// }) -// cancel() -// if err == nil { -// failF("%v: expected error for short data put", testname) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ -// Bucket: &bucket, -// Key: &name, -// }) -// defer cancel() -// if err == nil { -// failF("%v: expected object not exist", testname) -// return -// } - -// err = teardown(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } -// passF(testname) -// } - -// func TestInvalidMultiParts(s *S3Conf) { -// testname := "invalid multipart parts" -// runF(testname) - -// bucket := "bucket12" - -// err := setup(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } - -// s3client := s3.NewFromConfig(s.Config()) - -// obj := "mympuobject" - -// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) -// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ -// Bucket: &bucket, -// Key: &obj, -// }) -// cancel() -// if err != nil { -// failF("%v: create multipart upload: %v", testname, err) -// return -// } - -// // upload 2 parts -// size5MB := 5 * 1024 * 1024 -// dr := NewDataReader(size5MB, size5MB) - -// datafile := "rand.data" -// w, err := os.Create(datafile) -// if err != nil { -// failF("%v: create %v: %v", testname, datafile, err) -// return -// } -// defer w.Close() - -// _, err = io.Copy(w, dr) -// if err != nil { -// failF("%v: write %v: %v", testname, datafile, err) -// return -// } - -// _, err = w.Seek(0, io.SeekStart) -// if err != nil { -// failF("%v: seek %v: %v", testname, datafile, err) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ -// Bucket: &bucket, -// Key: &obj, -// PartNumber: -1, -// UploadId: mpu.UploadId, -// Body: w, -// ContentLength: int64(size5MB), -// }) -// cancel() -// if err == nil { -// failF("%v: multipart put part 1 expected error", testname) -// return -// } - -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ -// Bucket: &bucket, -// Key: &obj, -// }) -// cancel() -// if err == nil { -// failF("%v: head object %v expected error", testname, obj) -// return -// } - -// err = teardown(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } -// passF(testname) -// } +func CreateMultipartUpload_non_existing_bucket(s *S3Conf) { + testName := "CreateMultipartUpload_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bucketName := getBucketName() + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucketName, + Key: getPtr("my-obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func CreateMultipartUpload_success(s *S3Conf) { + testName := "CreateMultipartUpload_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if *out.Bucket != bucket { + return fmt.Errorf("expected bucket name %v, instead got %v", bucket, *out.Bucket) + } + if *out.Key != obj { + return fmt.Errorf("expected object name %v, instead got %v", obj, *out.Key) + } + if _, err := uuid.Parse(*out.UploadId); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_non_existing_bucket(s *S3Conf) { + testName := "UploadPart_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bucketName := getBucketName() + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucketName, + Key: getPtr("my-obj"), + UploadId: getPtr("uploadId"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func UploadPart_invalid_part_number(s *S3Conf) { + testName := "UploadPart_invalid_part_number" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + UploadId: getPtr("uploadId"), + PartNumber: -10, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { + return err + } + return nil + }) +} + +func UploadPart_non_existing_mp_upload(s *S3Conf) { + testName := "UploadPart_non_existing_mp_upload" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + UploadId: getPtr("uploadId"), + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + return nil + }) +} + +func UploadPart_non_existing_key(s *S3Conf) { + testName := "UploadPart_non_existing_key" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: getPtr("non-existing-object-key"), + UploadId: out.UploadId, + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + return nil + }) +} + +func UploadPart_success(s *S3Conf) { + testName := "UploadPart_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + PartNumber: 1, + }) + cancel() + if err != nil { + return err + } + if *res.ETag == "" { + return fmt.Errorf("expected a valid etag, instead got empty") + } + return nil + }) +} + +func UploadPartCopy_non_existing_bucket(s *S3Conf) { + testName := "UploadPartCopy_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bucketName := getBucketName() + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucketName, + CopySource: getPtr("Copy-Source"), + UploadId: getPtr("uploadId"), + Key: getPtr("my-obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + return nil + }) +} + +func UploadPartCopy_incorrect_uploadId(s *S3Conf) { + testName := "UploadPartCopy_incorrect_uploadId" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, srcBucket, srcObj := "my-obj", getBucketName(), "src-obj" + err := setup(s, srcBucket) + if err != nil { + return err + } + err = putObjects(s3client, []string{srcObj}, srcBucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr(srcBucket + "/" + srcObj), + UploadId: getPtr("incorrect-upload-id"), + Key: &obj, + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + + err = teardown(s, srcBucket) + if err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_incorrect_object_key(s *S3Conf) { + testName := "UploadPartCopy_incorrect_object_key" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, srcBucket, srcObj := "my-obj", getBucketName(), "src-obj" + err := setup(s, srcBucket) + if err != nil { + return err + } + err = putObjects(s3client, []string{srcObj}, srcBucket) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr(srcBucket + "/" + srcObj), + UploadId: out.UploadId, + Key: getPtr("non-existing-object-key"), + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + + err = teardown(s, srcBucket) + if err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_invalid_part_number(s *S3Conf) { + testName := "UploadPartCopy_invalid_part_number" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr("Copy-Source"), + UploadId: getPtr("uploadId"), + Key: getPtr("non-existing-object-key"), + PartNumber: -10, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_invalid_copy_source(s *S3Conf) { + testName := "UploadPartCopy_invalid_copy_source" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr("invalid-copy-source"), + UploadId: out.UploadId, + Key: &obj, + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidCopySource)); err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_non_existing_source_bucket(s *S3Conf) { + testName := "UploadPartCopy_non_existing_source_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr("src/bucket/src/obj"), + UploadId: out.UploadId, + Key: &obj, + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_non_existing_source_object_key(s *S3Conf) { + testName := "UploadPartCopy_non_existing_source_object_key" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, srcBucket := "my-obj", getBucketName() + + err := setup(s, srcBucket) + if err != nil { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr(srcBucket + "/non/existing/obj/key"), + UploadId: out.UploadId, + Key: &obj, + PartNumber: 1, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_success(s *S3Conf) { + testName := "UploadPartCopy_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, srcBucket, srcObj := "my-obj", getBucketName(), "src-obj" + err := setup(s, srcBucket) + if err != nil { + return err + } + objSize := 5 * 1024 * 1024 + _, _, err = putObjectWithData(int64(objSize), &s3.PutObjectInput{ + Bucket: &srcBucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr(srcBucket + "/" + srcObj), + UploadId: out.UploadId, + Key: &obj, + PartNumber: 1, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + }) + cancel() + if err != nil { + return err + } + + if len(res.Parts) != 1 { + return fmt.Errorf("expected parts to be 1, instead got %v", len(res.Parts)) + } + if res.Parts[0].PartNumber != 1 { + return fmt.Errorf("expected part-number to be 1, instead got %v", res.Parts[0].PartNumber) + } + if res.Parts[0].Size != int64(objSize) { + return fmt.Errorf("expected part size to be %v, instead got %v", objSize, res.Parts[0].Size) + } + if *res.Parts[0].ETag != *copyOut.CopyPartResult.ETag { + return fmt.Errorf("expected part etag to be %v, instead got %v", *copyOut.CopyPartResult.ETag, *res.Parts[0].ETag) + } + + err = teardown(s, srcBucket) + if err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_by_range_invalid_range(s *S3Conf) { + testName := "UploadPartCopy_by_range_invalid_range" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, srcBucket, srcObj := "my-obj", getBucketName(), "src-obj" + err := setup(s, srcBucket) + if err != nil { + return err + } + objSize := 5 * 1024 * 1024 + _, _, err = putObjectWithData(int64(objSize), &s3.PutObjectInput{ + Bucket: &srcBucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr(srcBucket + "/" + srcObj), + UploadId: out.UploadId, + Key: &obj, + PartNumber: 1, + CopySourceRange: getPtr("invalid-range"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { + return err + } + + err = teardown(s, srcBucket) + if err != nil { + return err + } + + return nil + }) +} + +func UploadPartCopy_by_range_success(s *S3Conf) { + testName := "UploadPartCopy_by_range_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, srcBucket, srcObj := "my-obj", getBucketName(), "src-obj" + err := setup(s, srcBucket) + if err != nil { + return err + } + objSize := 5 * 1024 * 1024 + _, _, err = putObjectWithData(int64(objSize), &s3.PutObjectInput{ + Bucket: &srcBucket, + Key: &srcObj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + copyOut, err := s3client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, + CopySource: getPtr(srcBucket + "/" + srcObj), + CopySourceRange: getPtr("bytes=100-200"), + UploadId: out.UploadId, + Key: &obj, + PartNumber: 1, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + }) + cancel() + if err != nil { + return err + } + + if len(res.Parts) != 1 { + return fmt.Errorf("expected parts to be 1, instead got %v", len(res.Parts)) + } + if res.Parts[0].PartNumber != 1 { + return fmt.Errorf("expected part-number to be 1, instead got %v", res.Parts[0].PartNumber) + } + if res.Parts[0].Size != 101 { + return fmt.Errorf("expected part size to be %v, instead got %v", 101, res.Parts[0].Size) + } + if *res.Parts[0].ETag != *copyOut.CopyPartResult.ETag { + return fmt.Errorf("expected part etag to be %v, instead got %v", *copyOut.CopyPartResult.ETag, *res.Parts[0].ETag) + } + + err = teardown(s, srcBucket) + if err != nil { + return err + } + + return nil + }) +} + +func ListParts_incorrect_uploadId(s *S3Conf) { + testName := "ListParts_incorrect_uploadId" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + UploadId: getPtr("invalid uploadId"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + + return nil + }) +} + +func ListParts_incorrect_object_key(s *S3Conf) { + testName := "ListParts_incorrect_object_key" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: getPtr("incorrect-object-key"), + UploadId: out.UploadId, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchUpload)); err != nil { + return err + } + + return nil + }) +} + +func ListParts_success(s *S3Conf) { + testName := "ListParts_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + parts, err := uploadParts(s3client, 5*1024*1024, 5, bucket, obj, *out.UploadId) + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListParts(ctx, &s3.ListPartsInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + }) + cancel() + if err != nil { + return err + } + + if ok := compareParts(parts, res.Parts); !ok { + return fmt.Errorf("expected parts %+v, instead got %+v", parts, res.Parts) + } + + return nil + }) +} + +func ListMultipartUploads_non_existing_bucket(s *S3Conf) { + testName := "ListMultipartUploads_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + bucketName := getBucketName() + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucketName, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func ListMultipartUploads_empty_result(s *S3Conf) { + testName := "ListMultipartUploads_empty_result" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + if len(out.Uploads) != 0 { + return fmt.Errorf("expected empty uploads, instead got %+v", out.Uploads) + } + + return nil + }) +} + +func ListMultipartUploads_success(s *S3Conf) { + testName := "ListMultipartUploads_max_uploads" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj1, obj2 := "my-obj-1", "my-obj-2" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out1, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj1, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out2, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj2, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + expected := []types.MultipartUpload{ + { + Key: &obj2, + UploadId: out2.UploadId, + }, + { + Key: &obj1, + UploadId: out1.UploadId, + }, + } + + if len(out.Uploads) != 2 { + return fmt.Errorf("expected 2 upload, instead got %v", len(out.Uploads)) + } + if ok := compareMultipartUploads(out.Uploads, expected); !ok { + return fmt.Errorf("expected uploads %v, instead got %v", expected, out.Uploads) + } + + return nil + }) +} + +func AbortMultipartUpload_non_existing_bucket(s *S3Conf) { + testName := "AbortMultipartUpload_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: getPtr("incorrectBucket"), + Key: getPtr("my-obj"), + UploadId: getPtr("uploadId"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func AbortMultipartUpload_incorrect_uploadId(s *S3Conf) { + testName := "AbortMultipartUpload_incorrect_uploadId" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + UploadId: getPtr("invalid uploadId"), + }) + cancel() + if err := checkSdkApiErr(err, "NoSuchUpload"); err != nil { + return err + } + + return nil + }) +} + +func AbortMultipartUpload_incorrect_object_key(s *S3Conf) { + testName := "AbortMultipartUpload_incorrect_object_key" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: &bucket, + Key: getPtr("incorrect-object-key"), + UploadId: out.UploadId, + }) + cancel() + if err := checkSdkApiErr(err, "NoSuchUpload"); err != nil { + return err + } + + return nil + }) +} + +func AbortMultipartUpload_success(s *S3Conf) { + testName := "AbortMultipartUpload_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if len(res.Uploads) != 0 { + return fmt.Errorf("expected 0 upload, instead got %v", len(res.Uploads)) + } + + return nil + }) +} + +func CompletedMultipartUpload_non_existing_bucket(s *S3Conf) { + testName := "CompletedMultipartUpload_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: getPtr("non-existing-bucket"), + Key: getPtr("some/key"), + UploadId: getPtr("uploadId"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_invalid_part_number(s *S3Conf) { + testName := "CompleteMultipartUpload_invalid_part_number" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + PartNumber: 1, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + { + ETag: res.ETag, + PartNumber: 5, + }, + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_invalid_ETag(s *S3Conf) { + testName := "CompleteMultipartUpload_invalid_ETag" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + PartNumber: 1, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{ + { + ETag: getPtr("invalidETag"), + PartNumber: 1, + }, + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_success(s *S3Conf) { + testName := "CompleteMultipartUpload_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + objSize := 5 * 1024 * 1024 + parts, err := uploadParts(s3client, objSize, 5, bucket, obj, *out.UploadId) + if err != nil { + return err + } + + compParts := []types.CompletedPart{} + for _, el := range parts { + compParts = append(compParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + }) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: out.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: compParts, + }, + }) + cancel() + if err != nil { + return err + } + + if *res.Key != obj { + return fmt.Errorf("expected object key to be %v, instead got %v", obj, *res.Key) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + resp, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return err + } + + if *resp.ETag != *res.ETag { + return fmt.Errorf("expected the uploaded object etag to be %v, instead got %v", *res.ETag, *resp.ETag) + } + if resp.ContentLength != int64(objSize) { + return fmt.Errorf("expected the uploaded object size to be %v, instead got %v", objSize, resp.ContentLength) + } + + return nil + }) +} // func TestAclActions(s *S3Conf) { // testname := "test put/get acl" diff --git a/integration/utils.go b/integration/utils.go index d76233b8..694015fa 100644 --- a/integration/utils.go +++ b/integration/utils.go @@ -7,8 +7,9 @@ import ( "crypto/sha256" "errors" "fmt" + "io" + "os" "os/exec" - "strings" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" @@ -181,31 +182,33 @@ func isEqual(a, b []byte) bool { return true } -func contains(name string, list []types.Object) bool { - for _, item := range list { - if strings.EqualFold(name, *item.Key) { - return true +func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool { + if len(list1) != len(list2) { + return false + } + for i, item := range list1 { + if *item.Key != *list2[i].Key || *item.UploadId != *list2[i].UploadId { + return false } } - return false + + return true } -func containsUID(name, id string, list []types.MultipartUpload) bool { - for _, item := range list { - if strings.EqualFold(name, *item.Key) && strings.EqualFold(id, *item.UploadId) { - return true - } +func compareParts(parts1, parts2 []types.Part) bool { + if len(parts1) != len(parts2) { + return false } - return false -} -func containsPart(part int32, list []types.Part) bool { - for _, item := range list { - if item.PartNumber == part { - return true + for i, prt := range parts1 { + if prt.PartNumber != parts2[i].PartNumber { + return false + } + if *prt.ETag != *parts2[i].ETag { + return false } } - return false + return true } func areTagsSame(tags1, tags2 []types.Tag) bool { @@ -314,3 +317,53 @@ func compareDelObjects(list1 []string, list2 []types.DeletedObject) bool { return true } + +func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId string) (parts []types.Part, err error) { + dr := NewDataReader(size, size) + datafile := "rand.data" + w, err := os.Create(datafile) + if err != nil { + return parts, err + } + defer w.Close() + + _, err = io.Copy(w, dr) + if err != nil { + return parts, err + } + + fileInfo, err := w.Stat() + if err != nil { + return parts, err + } + + partSize := fileInfo.Size() / int64(partCount) + var offset int64 + + for partNumber := int64(1); partNumber <= int64(partCount); partNumber++ { + partStart := (partNumber - 1) * partSize + partEnd := partStart + partSize - 1 + if partEnd > fileInfo.Size()-1 { + partEnd = fileInfo.Size() - 1 + } + partBuffer := make([]byte, partEnd-partStart+1) + _, err := w.ReadAt(partBuffer, partStart) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := client.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, + Key: &key, + UploadId: &uploadId, + Body: bytes.NewReader(partBuffer), + PartNumber: int32(partNumber), + }) + cancel() + if err != nil { + return parts, err + } else { + parts = append(parts, types.Part{ETag: out.ETag, PartNumber: int32(partNumber)}) + offset += partSize + } + } + + return parts, err +} From a05179b14fbd23346e87c02256c7c7d4f75b864e Mon Sep 17 00:00:00 2001 From: jonaustin09 Date: Wed, 23 Aug 2023 17:24:16 +0400 Subject: [PATCH 4/4] feat: Added integration test cases for PutBucketAcl, GetBucketAcl actions --- integration/action-tests.go | 18 ++ integration/tests.go | 417 +++++++++++++++++++++++++++--------- integration/utils.go | 29 ++- 3 files changed, 364 insertions(+), 100 deletions(-) diff --git a/integration/action-tests.go b/integration/action-tests.go index 59e038d8..0cb7699e 100644 --- a/integration/action-tests.go +++ b/integration/action-tests.go @@ -129,6 +129,22 @@ func TestCompleteMultipartUpload(s *S3Conf) { CompleteMultipartUpload_success(s) } +func TestPutBucketAcl(s *S3Conf) { + PutBucketAcl_non_existing_bucket(s) + PutBucketAcl_invalid_acl_canned_and_acp(s) + PutBucketAcl_invalid_acl_canned_and_grants(s) + PutBucketAcl_invalid_acl_acp_and_grants(s) + PutBucketAcl_invalid_owner(s) + PutBucketAcl_success_access_denied(s) + PutBucketAcl_success(s) +} + +func TestGetBucketAcl(s *S3Conf) { + GetBucketAcl_non_existing_bucket(s) + GetBucketAcl_access_denied(s) + GetBucketAcl_success(s) +} + func TestFullFlow(s *S3Conf) { TestCreateBucket(s) TestHeadBucket(s) @@ -149,4 +165,6 @@ func TestFullFlow(s *S3Conf) { TestListMultipartUploads(s) TestAbortMultipartUpload(s) TestCompleteMultipartUpload(s) + TestPutBucketAcl(s) + TestGetBucketAcl(s) } diff --git a/integration/tests.go b/integration/tests.go index 0c7090c4..5bb23f9c 100644 --- a/integration/tests.go +++ b/integration/tests.go @@ -753,6 +753,9 @@ func DeleteObjects_success(s *S3Conf) { actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { objects, objToDel := []string{"obj1", "obj2", "obj3"}, []string{"foo", "bar", "baz"} err := putObjects(s3client, append(objToDel, objects...), bucket) + if err != nil { + return err + } delObjects := []types.ObjectIdentifier{} for _, key := range objToDel { @@ -804,6 +807,9 @@ func CopyObject_non_existing_dst_bucket(s *S3Conf) { actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ Bucket: &bucket, @@ -1425,6 +1431,11 @@ func UploadPartCopy_non_existing_source_object_key(s *S3Conf) { return err } + err = teardown(s, srcBucket) + if err != nil { + return err + } + return nil }) } @@ -2090,121 +2101,331 @@ func CompleteMultipartUpload_success(s *S3Conf) { }) } -// func TestAclActions(s *S3Conf) { -// testname := "test put/get acl" -// runF(testname) +func PutBucketAcl_non_existing_bucket(s *S3Conf) { + testName := "PutBucketAcl_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: getPtr(getBucketName()), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } -// bucket := "testbucket14" + return nil + }) +} -// err := setup(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } +func PutBucketAcl_invalid_acl_canned_and_acp(s *S3Conf) { + testName := "PutBucketAcl_invalid_acl_canned_and_acp" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + ACL: types.BucketCannedACLPrivate, + GrantRead: getPtr("user1"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRequest)); err != nil { + return err + } -// s3client := s3.NewFromConfig(s.Config()) + return nil + }) +} -// rootAccess := s.awsID -// rootSecret := s.awsSecret +func PutBucketAcl_invalid_acl_canned_and_grants(s *S3Conf) { + testName := "PutBucketAcl_invalid_acl_canned_and_grants" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + ACL: types.BucketCannedACLPrivate, + AccessControlPolicy: &types.AccessControlPolicy{ + Grants: []types.Grant{ + { + Grantee: &types.Grantee{ + ID: getPtr("awsID"), + Type: types.TypeCanonicalUser, + }, + }, + }, + Owner: &types.Owner{ + ID: &s.awsID, + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRequest)); err != nil { + return err + } -// s.awsID = "grt1" -// s.awsSecret = "grt1secret" + return nil + }) +} -// userS3Client := s3.NewFromConfig(s.Config()) +func PutBucketAcl_invalid_acl_acp_and_grants(s *S3Conf) { + testName := "PutBucketAcl_invalid_acl_acp_and_grants" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + GrantFullControl: getPtr("userAccess"), + AccessControlPolicy: &types.AccessControlPolicy{ + Grants: []types.Grant{ + { + Grantee: &types.Grantee{ + ID: getPtr("awsID"), + Type: types.TypeCanonicalUser, + }, + }, + }, + Owner: &types.Owner{ + ID: &s.awsID, + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRequest)); err != nil { + return err + } -// s.awsID = rootAccess -// s.awsSecret = rootSecret + return nil + }) +} -// grt1 := "grt1" +func PutBucketAcl_invalid_owner(s *S3Conf) { + testName := "PutBucketAcl_invalid_acl_acp_and_grants" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + AccessControlPolicy: &types.AccessControlPolicy{ + Grants: []types.Grant{ + { + Grantee: &types.Grantee{ + ID: getPtr("awsID"), + Type: types.TypeCanonicalUser, + }, + }, + }, + Owner: &types.Owner{ + ID: getPtr("invalidOwner"), + }, + }, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } -// grants := []types.Grant{ -// { -// Permission: "READ", -// Grantee: &types.Grantee{ -// ID: &grt1, -// Type: "CanonicalUser", -// }, -// }, -// } + return nil + }) +} -// succUsrCrt := "The user has been created successfully" -// failUsrCrt := "failed to create a user: update iam data: account already exists" +func PutBucketAcl_success_access_denied(s *S3Conf) { + testName := "PutBucketAcl_success_access_denied" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := createUsers(s, []user{{"grt1", "grt1secret", "user"}}) + if err != nil { + return err + } -// out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "create-user", "-a", grt1, "-s", "grt1secret", "-r", "user") -// if err != nil { -// failF("%v: %v", err) -// return -// } -// if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) { -// failF("%v: failed to create user accounts", testname) -// return -// } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + AccessControlPolicy: &types.AccessControlPolicy{ + Grants: []types.Grant{ + { + Grantee: &types.Grantee{ + ID: getPtr("grt1"), + Type: types.TypeCanonicalUser, + }, + Permission: types.PermissionRead, + }, + }, + Owner: &types.Owner{ + ID: &s.awsID, + }, + }, + }) + cancel() + if err != nil { + return err + } -// // Validation error case -// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ -// Bucket: &bucket, -// AccessControlPolicy: &types.AccessControlPolicy{ -// Grants: grants, -// }, -// ACL: "private", -// }) -// cancel() -// if err == nil { -// failF("%v: expected validation error", testname) -// return -// } + newConf := *s + newConf.awsID = "grt1" + newConf.awsSecret = "grt1secret" + userClient := s3.NewFromConfig(newConf.Config()) -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ -// Bucket: &bucket, -// AccessControlPolicy: &types.AccessControlPolicy{ -// Grants: grants, -// Owner: &types.Owner{ID: &s.awsID}, -// }, -// }) -// cancel() -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } + err = putObjects(userClient, []string{"my-obj"}, bucket) + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// acl, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{ -// Bucket: &bucket, -// }) -// cancel() -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } + return nil + }) +} -// if *acl.Owner.ID != s.awsID { -// failF("%v: expected bucket owner: %v, instead got: %v", testname, s.awsID, *acl.Owner.ID) -// return -// } -// if !checkGrants(acl.Grants, grants) { -// failF("%v: expected %v, instead got %v", testname, grants, acl.Grants) -// return -// } +func PutBucketAcl_success(s *S3Conf) { + testName := "PutBucketAcl_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := createUsers(s, []user{{"grt1", "grt1secret", "user"}}) + if err != nil { + return err + } -// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) -// _, err = userS3Client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ -// Bucket: &bucket, -// }) -// cancel() -// if err == nil { -// failF("%v: expected acl access denied error", testname) -// return -// } + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + AccessControlPolicy: &types.AccessControlPolicy{ + Grants: []types.Grant{ + { + Grantee: &types.Grantee{ + ID: getPtr("grt1"), + Type: types.TypeCanonicalUser, + }, + Permission: types.PermissionFullControl, + }, + }, + Owner: &types.Owner{ + ID: &s.awsID, + }, + }, + }) + cancel() + if err != nil { + return err + } -// err = teardown(s, bucket) -// if err != nil { -// failF("%v: %v", testname, err) -// return -// } -// passF(testname) -// } + newConf := *s + newConf.awsID = "grt1" + newConf.awsSecret = "grt1secret" + userClient := s3.NewFromConfig(newConf.Config()) + + err = putObjects(userClient, []string{"my-obj"}, bucket) + if err != nil { + return err + } + + return nil + }) +} + +func GetBucketAcl_non_existing_bucket(s *S3Conf) { + testName := "GetBucketAcl_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{ + Bucket: getPtr(getBucketName()), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + + return nil + }) +} + +func GetBucketAcl_access_denied(s *S3Conf) { + testName := "GetBucketAcl_access_denied" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := createUsers(s, []user{{"grt1", "grt1secret", "user"}}) + if err != nil { + return err + } + + newConf := *s + newConf.awsID = "grt1" + newConf.awsSecret = "grt1secret" + userClient := s3.NewFromConfig(newConf.Config()) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = userClient.GetBucketAcl(ctx, &s3.GetBucketAclInput{ + Bucket: &bucket, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrAccessDenied)); err != nil { + return err + } + + return nil + }) +} + +func GetBucketAcl_success(s *S3Conf) { + testName := "GetBucketAcl_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := createUsers(s, []user{ + {"grt1", "grt1secret", "user"}, + {"grt2", "grt2secret", "user"}, + {"grt3", "grt3secret", "user"}, + }) + if err != nil { + return err + } + + grants := []types.Grant{ + { + Grantee: &types.Grantee{ + ID: getPtr("grt1"), + Type: types.TypeCanonicalUser, + }, + Permission: types.PermissionFullControl, + }, + { + Grantee: &types.Grantee{ + ID: getPtr("grt2"), + Type: types.TypeCanonicalUser, + }, + Permission: types.PermissionReadAcp, + }, + { + Grantee: &types.Grantee{ + ID: getPtr("grt3"), + Type: types.TypeCanonicalUser, + }, + Permission: types.PermissionWrite, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ + Bucket: &bucket, + AccessControlPolicy: &types.AccessControlPolicy{ + Grants: grants, + Owner: &types.Owner{ + ID: &s.awsID, + }, + }, + }) + cancel() + if err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{ + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + + if ok := compareGrants(out.Grants, grants); !ok { + return fmt.Errorf("expected grants to be %v, instead got %v", grants, out.Grants) + } + if *out.Owner.ID != s.awsID { + return fmt.Errorf("expected bucket owner to be %v, instead got %v", s.awsID, *out.Owner.ID) + } + + return nil + }) +} type prefResult struct { elapsed time.Duration diff --git a/integration/utils.go b/integration/utils.go index 694015fa..42babc1f 100644 --- a/integration/utils.go +++ b/integration/utils.go @@ -10,6 +10,7 @@ import ( "io" "os" "os/exec" + "strings" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" @@ -18,7 +19,9 @@ import ( ) var ( - bcktCount = 0 + bcktCount = 0 + succUsrCrt = "The user has been created successfully" + failUsrCrt = "failed to create a user: update iam data: account already exists" ) func getBucketName() string { @@ -233,7 +236,7 @@ func containsTag(tag types.Tag, list []types.Tag) bool { return false } -func checkGrants(grts1, grts2 []types.Grant) bool { +func compareGrants(grts1, grts2 []types.Grant) bool { if len(grts1) != len(grts2) { return false } @@ -348,6 +351,9 @@ func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId s } partBuffer := make([]byte, partEnd-partStart+1) _, err := w.ReadAt(partBuffer, partStart) + if err != nil { + return parts, err + } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := client.UploadPart(ctx, &s3.UploadPartInput{ Bucket: &bucket, @@ -367,3 +373,22 @@ func uploadParts(client *s3.Client, size, partCount int, bucket, key, uploadId s return parts, err } + +type user struct { + access string + secret string + role string +} + +func createUsers(s *S3Conf, users []user) error { + for _, usr := range users { + out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "create-user", "-a", usr.access, "-s", usr.secret, "-r", usr.role) + if err != nil { + return err + } + if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) { + return fmt.Errorf("failed to create a user account") + } + } + return nil +}