From 2cc0c7203c9026337549e6884224f0d305b3f8ed Mon Sep 17 00:00:00 2001 From: jonaustin09 Date: Fri, 11 Aug 2023 02:13:01 +0400 Subject: [PATCH] feat: Closes #189, added utility functions for testing, restructured tests for CreateBucket, DeleteBucket... and 5 more actions --- cmd/versitygw/test.go | 108 -- integration/action-tests.go | 58 + integration/tests.go | 3076 +++++++++++++++++++---------------- integration/utils.go | 91 ++ 4 files changed, 1790 insertions(+), 1543 deletions(-) create mode 100644 integration/action-tests.go diff --git a/cmd/versitygw/test.go b/cmd/versitygw/test.go index 578b30a..1bcfcd9 100644 --- a/cmd/versitygw/test.go +++ b/cmd/versitygw/test.go @@ -67,114 +67,6 @@ func initTestFlags() []cli.Flag { func initTestCommands() []*cli.Command { return []*cli.Command{ - { - Name: "bucket-actions", - Usage: "Test bucket creation, checking the existence, deletes it.", - Description: `Calls s3 gateway create-bucket action to create a new bucket, - calls head-bucket action to check the existence, then calls delete-bucket action to delete the bucket.`, - Action: getAction(integration.TestMakeBucket), - }, - { - Name: "object-actions", - Usage: "Test put/get/delete/copy objects.", - Description: `Creates a bucket with s3 gateway action, puts an object in it, - tries to copy into another bucket, that doesn't exist, creates the destination bucket for copying, - copies the object, get's the object to check the length and content, - get's the copied object to check the length and content, deletes all the objects inside the source bucket, - deletes both the objects and buckets.`, - Action: getAction(integration.TestPutGetObject), - }, - { - Name: "put-get-mp-object", - Usage: "Test put & get multipart object.", - Description: `Creates a bucket with s3 gateway action, puts an object in it with multipart upload, - gets the object from the bucket, deletes both the object and bucket.`, - Action: getAction(integration.TestPutGetMPObject), - }, - { - Name: "put-dir-object", - Usage: "Test put directory object.", - Description: `Creates a bucket with s3 gateway action, puts a directory object in it, - lists the bucket's objects, deletes both the objects and bucket.`, - Action: getAction(integration.TestPutDirObject), - }, - { - Name: "list-objects", - Usage: "Test list-objects action.", - Description: `Creates a bucket with s3 gateway action, puts 2 directory objects in it, - lists the bucket's objects, deletes both the objects and bucket.`, - Action: getAction(integration.TestListObject), - }, - { - Name: "abort-mp", - Usage: "Tests abort-multipart-upload action.", - Description: `Creates a bucket with s3 gateway action, creates a multipart upload, - lists the multipart upload, aborts the multipart upload, lists the multipart upload again, - deletes both the objects and bucket.`, - Action: getAction(integration.TestListAbortMultiPartObject), - }, - { - Name: "list-parts", - Usage: "Tests list-parts action.", - Description: `Creates a bucket with s3 gateway action, creates a multipart upload, - lists the upload parts, deletes both the objects and bucket.`, - Action: getAction(integration.TestListMultiParts), - }, - { - Name: "incorrect-mp", - Usage: "Tests incorrect multipart case.", - Description: `Creates a bucket with s3 gateway action, creates a multipart upload, - uploads different parts, completes the multipart upload with incorrect part numbers, - calls the head-object action, compares the content length, removes both the object and bucket`, - Action: getAction(integration.TestIncorrectMultiParts), - }, - { - Name: "incomplete-mp", - Usage: "Tests incomplete multi parts.", - Description: `Creates a bucket with s3 gateway action, creates a multipart upload, - upload a part, lists the parts, checks if the uploaded part is in the list, - removes both the object and the bucket`, - Action: getAction(integration.TestIncompleteMultiParts), - }, - { - Name: "incomplete-put-object", - Usage: "Tests incomplete put objects case.", - Description: `Creates a bucket with s3 gateway action, puts an object in it, - gets the object with head-object action, expects the object to be got, - removes both the object and bucket`, - Action: getAction(integration.TestIncompletePutObject), - }, - { - Name: "get-range", - Usage: "Tests get object by range.", - Description: `Creates a bucket with s3 gateway action, puts an object in it, - gets the object by specifying the object range, compares the range with the original one, - removes both the object and the bucket`, - Action: getAction(integration.TestRangeGet), - }, - { - Name: "invalid-mp", - Usage: "Tests invalid multi part case.", - Description: `Creates a bucket with s3 gateway action, creates a multi part upload, - uploads an invalid part, gets the object with head-object action, expects to get error, - removes both the object and bucket`, - Action: getAction(integration.TestInvalidMultiParts), - }, - { - Name: "object-tag-actions", - Usage: "Tests get/put/delete object tag actions.", - Description: `Creates a bucket with s3 gateway action, puts an object in it, - puts some tags for the object, gets the tags, compares the results, removes the tags, - gets the tags again, checks it to be empty, then removes both the object and bucket`, - Action: getAction(integration.TestPutGetRemoveTags), - }, - { - Name: "bucket-acl-actions", - Usage: "Tests put/get bucket actions.", - Description: `Creates a bucket with s3 gateway action, puts some bucket acls - gets the acl, verifies it, then removes the bucket`, - Action: getAction(integration.TestAclActions), - }, { Name: "full-flow", Usage: "Tests the full flow of gateway.", diff --git a/integration/action-tests.go b/integration/action-tests.go new file mode 100644 index 0000000..f6f2ae4 --- /dev/null +++ b/integration/action-tests.go @@ -0,0 +1,58 @@ +package integration + +func TestCreateBucket(s *S3Conf) { + CreateBucket_invalid_bucket_name(s) + CreateBucket_existing_bucket(s) + CreateDeleteBucket_success(s) +} + +func TestDeleteBucket(s *S3Conf) { + DeleteBucket_non_existing_bucket(s) + DeleteBucket_non_empty_bucket(s) +} + +func TestPutObject(s *S3Conf) { + PutObject_non_existing_bucket(s) + PutObject_special_chars(s) + PutObject_existing_dir_obj(s) + PutObject_obj_parent_is_file(s) + PutObject_success(s) +} + +func TestGetObject(s *S3Conf) { + GetObject_non_existing_key(s) + GetObject_invalid_ranges(s) + GetObject_with_meta(s) + GetObject_success(s) + GetObject_by_range_success(s) +} + +func TestCopyObject(s *S3Conf) { + CopyObject_non_existing_dst_bucket(s) + CopyObject_success(s) +} + +func TestPutObjectTagging(s *S3Conf) { + PutObjectTagging_non_existing_object(s) + PutObjectTagging_success(s) +} + +func TestGetObjectTagging(s *S3Conf) { + GetObjectTagging_non_existing_object(s) + GetObjectTagging_success(s) +} + +func TestDeleteObjectTagging(s *S3Conf) { + DeleteObjectTagging_non_existing_object(s) + DeleteObjectTagging_success(s) +} + +func TestFullFlow(s *S3Conf) { + TestCreateBucket(s) + TestDeleteBucket(s) + TestPutObject(s) + TestGetObject(s) + TestCopyObject(s) + TestPutObjectTagging(s) + TestDeleteObjectTagging(s) +} diff --git a/integration/tests.go b/integration/tests.go index aed74d9..f4c0aa6 100644 --- a/integration/tests.go +++ b/integration/tests.go @@ -1,1295 +1,1731 @@ package integration import ( - "bytes" "context" - "crypto/rand" "crypto/sha256" + "errors" "fmt" "io" "math" - "os" - "strings" "sync" "time" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3err" ) var ( shortTimeout = 10 * time.Second ) -func TestMakeBucket(s *S3Conf) { - testname := "test make/head/delete bucket" - runF(testname) - - s3client := s3.NewFromConfig(s.Config()) - - invBucket := "aa" - err := setup(s, invBucket) - if err == nil { - failF("%v: expected bucket name validation error", testname) +func CreateBucket_invalid_bucket_name(s *S3Conf) { + testName := "CreateBucket_invalid_bucket_name" + runF(testName) + err := setup(s, "aa") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketName)); err != nil { + failF("%v: %v", testName, err.Error()) return } - bucket := "testbucket" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket}) - cancel() - if err == nil { - failF("%v: expected error, instead got success response", testname) + err = setup(s, ".gitignore") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketName)); err != nil { + failF("%v: %v", testName, err.Error()) return } + err = setup(s, "my-bucket.") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketName)); err != nil { + failF("%v: %v", testName, err.Error()) + return + } + + err = setup(s, "bucket-%") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketName)); err != nil { + failF("%v: %v", testName, err.Error()) + return + } + passF(testName) +} + +func CreateBucket_existing_bucket(s *S3Conf) { + testName := "CreateBucket_existing_bucket" + runF(testName) + bucket := getBucketName() + err := setup(s, bucket) + if err != nil { + failF("%v: %v", testName, err.Error()) + return + } err = setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return + var bne *types.BucketAlreadyExists + if !errors.As(err, &bne) { + failF("%v: %v", testName, err.Error()) } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket}) - cancel() + err = teardown(s, bucket) if err != nil { - failF("%v: %v", testname, err) + failF("%v: %v", err.Error()) + return + } + passF(testName) +} + +func CreateDeleteBucket_success(s *S3Conf) { + testName := "CreateBucket_success" + runF(testName) + bucket := getBucketName() + + err := setup(s, bucket) + if err != nil { + failF("%v: %v", err.Error()) return } err = teardown(s, bucket) if err != nil { - failF("%v: %v", testname, err) + failF("%v: %v", err.Error()) return } - passF(testname) + + passF(testName) } -func TestPutGetObject(s *S3Conf) { - testname := "test put/get/delete/copy objects" - runF(testname) - - bucket := "testbucket1" - dstBucket := "testdstbucket" - obj := "myobject" - obj2 := "myobject2" - obj3 := "myobject%%3" - copySource := bucket + "/" + obj - +func DeleteBucket_non_existing_bucket(s *S3Conf) { + testName := "DeleteBucket_non_existing_bucket" + runF(testName) + bucket := getBucketName() s3client := s3.NewFromConfig(s.Config()) - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - // use funny size to prevent accidental alignments - datalen := 1234567 - data := make([]byte, datalen) - rand.Read(data) - csum := sha256.Sum256(data) - r := bytes.NewReader(data) - meta := map[string]string{ - "key1": "val1", - "key2": "val2", - } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - Body: r, - Metadata: meta, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ + _, err := s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ Bucket: &bucket, - Key: &obj3, }) cancel() - if err != nil { - failF("%v: %v", testname, err) + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + failF("%v: %v", testName, err.Error()) return } + passF(testName) +} - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj2, - Body: r, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj3, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - if !areMapsSame(out.Metadata, meta) { - failF("%v: incorrect object metadata", testname) - return - } - defer out.Body.Close() - - if out.ContentLength != int64(datalen) { - failF("%v: content length got %v expected %v", testname, out.ContentLength, datalen) - return - } - - b, err := io.ReadAll(out.Body) - if err != nil { - failF("%v: read body %v", testname, err) - return - } - - newsum := sha256.Sum256(b) - if csum != newsum { - failF("%v: checksum got %x expected %x", testname, newsum, csum) - return - } - - // Expected error: destination bucket doesn't exist - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: &dstBucket, Key: &obj, CopySource: ©Source}) - cancel() - if err == nil { - failF("%v: expect bucket not found error instead got success response", testname) - return - } - - err = setup(s, dstBucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: &dstBucket, Key: &obj, CopySource: ©Source}) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - copyObjOut, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &dstBucket, - Key: &obj, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - defer copyObjOut.Body.Close() - - if copyObjOut.ContentLength != int64(datalen) { - failF("%v: content length got %v expected %v", testname, copyObjOut.ContentLength, datalen) - return - } - - b, err = io.ReadAll(copyObjOut.Body) - if err != nil { - failF("%v: read body %v", testname, err) - return - } - - copysum := sha256.Sum256(b) - if csum != copysum { - failF("%v: copied object checksum got %x expected %x", testname, copysum, csum) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{Bucket: &bucket, Delete: &types.Delete{Objects: []types.ObjectIdentifier{{Key: &obj}, {Key: &obj2}, {Key: &obj3}}}}) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - objCount := 0 - - in := &s3.ListObjectsV2Input{Bucket: &bucket} - for { +func DeleteBucket_non_empty_bucket(s *S3Conf) { + testName := "DeleteBucket_non_empty_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"foo"}, bucket) + if err != nil { + return err + } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectsV2(ctx, in) + _, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: &bucket, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrBucketNotEmpty)); err != nil { + return err + } + + return nil + }) +} + +func PutObject_non_existing_bucket(s *S3Conf) { + testName := "PutObject_non_existing_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"my-obj"}, "non-existing-bucket") + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + return nil + }) +} + +func PutObject_special_chars(s *S3Conf) { + testName := "PutObject_special_chars" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"foo%%", "bar^", "baz**"}, bucket) + if err != nil { + return err + } + return nil + }) +} + +func PutObject_existing_dir_obj(s *S3Conf) { + testName := "PutObject_existing_dir_obj" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"foo/bar", "foo"}, bucket) + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrExistingObjectIsDirectory)); err != nil { + return err + } + return nil + }) +} + +func PutObject_obj_parent_is_file(s *S3Conf) { + testName := "PutObject_obj_parent_is_file" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"foo", "foo/bar/"}, bucket) + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectParentIsFile)); err != nil { + return err + } + return nil + }) +} + +func PutObject_success(s *S3Conf) { + testName := "PutObject_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + err := putObjects(s3client, []string{"my-obj"}, bucket) + if err != nil { + return err + } + return nil + }) +} + +func GetObject_non_existing_key(s *S3Conf) { + testName := "GetObject_non_existing_key" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: getPtr("non-existing-key"), + }) + cancel() + var bae *types.NoSuchKey + if !errors.As(err, &bae) { + return err + } + return nil + }) +} + +func GetObject_invalid_ranges(s *S3Conf) { + testName := "GetObject_invalid_ranges" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + + _, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: getPtr("bytes=invalid-range"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: getPtr("bytes=33-10"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { + return err + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: getPtr("bytes=1000000000-999999999999"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidRange)); err != nil { + return err + } + return nil + }) +} + +func GetObject_with_meta(s *S3Conf) { + testName := "GetObject_with_meta" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + meta := map[string]string{ + "key1": "val1", + "key2": "val2", + } + + _, _, err := putObjectWithData(0, &s3.PutObjectInput{Bucket: &bucket, Key: &obj, Metadata: meta}, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + + if !areMapsSame(out.Metadata, meta) { + return fmt.Errorf("incorrect object metadata") + } + + return nil + }) +} + +func GetObject_success(s *S3Conf) { + testName := "GetObject_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + + csum, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + if out.ContentLength != dataLength { + return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) + } + + bdy, err := io.ReadAll(out.Body) + if err != nil { + return err + } + defer out.Body.Close() + outCsum := sha256.Sum256(bdy) + if outCsum != csum { + return fmt.Errorf("invalid object data") + } + return nil + }) +} + +func GetObject_by_range_success(s *S3Conf) { + testName := "GetObject_by_range_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + + _, data, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + rangeString := "bytes=100-200" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: &rangeString, + }) + defer cancel() + if err != nil { + return err + } + defer out.Body.Close() + + if getString(out.ContentRange) != fmt.Sprintf("bytes 100-200/%v", dataLength) { + return fmt.Errorf("expected content range: %v, instead got: %v", fmt.Sprintf("bytes 100-200/%v", dataLength), getString(out.ContentRange)) + } + if getString(out.AcceptRanges) != rangeString { + return fmt.Errorf("expected accept range: %v, instead got: %v", rangeString, getString(out.AcceptRanges)) + } + b, err := io.ReadAll(out.Body) + if err != nil { + return err + } + + // bytes range is inclusive, go range for second value is not + if !isEqual(b, data[100:201]) { + return fmt.Errorf("data mismatch of range") + } + + rangeString = "bytes=100-" + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err = s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + Range: &rangeString, + }) + defer cancel() + if err != nil { + return err + } + defer out.Body.Close() + + b, err = io.ReadAll(out.Body) + if err != nil { + return err + } + + // bytes range is inclusive, go range for second value is not + if !isEqual(b, data[100:]) { + return fmt.Errorf("data mismatch of range") + } + return nil + }) +} + +func CopyObject_non_existing_dst_bucket(s *S3Conf) { + testName := "CopyObject_non_existing_dst_bucket" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + err := putObjects(s3client, []string{obj}, bucket) + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &bucket, + Key: &obj, + CopySource: getPtr("bucket/obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil { + return err + } + return nil + }) +} + +func CopyObject_success(s *S3Conf) { + testName := "CopyObject_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(1234567), "my-obj" + dstBucket := getBucketName() + err := setup(s, dstBucket) + if err != nil { + return err + } + + csum, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: &dstBucket, + Key: &obj, + CopySource: getPtr(fmt.Sprintf("%v/%v", bucket, obj)), + }) cancel() if err != nil { - failF("%v: %v", testname, err) - return + return err } - objCount += len(out.Contents) - if out.IsTruncated { - in.ContinuationToken = out.ContinuationToken - } else { - break - } - } - if objCount != 0 { - failF("%v: expected object count %v instead got %v", testname, 0, objCount) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - err = teardown(s, dstBucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - passF(testname) -} - -func TestPutGetMPObject(s *S3Conf) { - testname := "test put/get multipart object" - runF(testname) - - bucket := "testbucket2" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - name := "mympuobject" - s3client := s3.NewFromConfig(s.Config()) - - datalen := 10*1024*1024 + 15 - dr := NewDataReader(datalen, 5*1024*1024) - WithPartSize(5 * 1024 * 1024) - s.PartSize = 5 * 1024 * 1024 - err = s.UploadData(dr, bucket, name) - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - defer out.Body.Close() - - if out.ContentLength != int64(datalen) { - failF("%v: content length got %v expected %v", testname, out.ContentLength, datalen) - return - } - - b := make([]byte, 1048576) - h := sha256.New() - for { - n, err := out.Body.Read(b) - if err == io.EOF { - h.Write(b[:n]) - break - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &dstBucket, + Key: &obj, + }) + defer cancel() if err != nil { - failF("%v: read %v", err) - return + return err } - h.Write(b[:n]) - } - - if !isEqual(dr.Sum(), h.Sum(nil)) { - failF("%v: checksum got %x expected %x", testname, h.Sum(nil), dr.Sum()) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -func TestPutDirObject(s *S3Conf) { - testname := "test put directory object" - runF(testname) - - bucket := "testbucket3" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - name := "myobjectdir/" - s3client := s3.NewFromConfig(s.Config()) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &name, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket, MaxKeys: -4}) - cancel() - if err == nil { - failF("%v: expected invalid argument error", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) - cancel() - if err != nil { - failF("failed to list objects: %v", err) - return - } - - if !contains(name, out.Contents) { - failF("directory object not found") - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -func TestListObject(s *S3Conf) { - testname := "list objects" - runF(testname) - - bucket := "testbucket4" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - dir1 := "myobjectdir/" - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &dir1, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - obj1 := "myobjectdir/myobject" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj1, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - obj2 := "myobjectdir1/myobject" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj2, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - // put: - // "myobjectdir/" - // "myobjectdir/myobject" - // "myobjectdir1/myobject" - // should return: - // "myobjectdir/myobject" - // "myobjectdir1/myobject" - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) - cancel() - if err != nil { - failF("failed to list objects: %v", err) - return - } - - if !contains(obj1, out.Contents) { - failF("object %v not found", obj1) - return - } - if !contains(obj2, out.Contents) { - failF("object %v not found", obj2) - return - } - if out.KeyCount != 2 { - failF("%v: expected key count: %v, instead got: %v", testname, 2, out.KeyCount) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj1, - }) - cancel() - if err != nil { - failF("failed to delete %v: %v", obj1, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &obj2, - }) - cancel() - if err != nil { - failF("failed to delete %v: %v", obj2, err) - return - } - - // put: - // "myobjectdir/" - // "myobjectdir/myobject" - // "myobjectdir1/myobject" - // delete: - // "myobjectdir/myobject" - // "myobjectdir1/myobject" - // should return: - // "myobjectdir/" - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) - cancel() - if err != nil { - failF("failed to list objects: %v", err) - return - } - - if !contains(dir1, out.Contents) { - failF("dir %v not found", dir1) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -func TestListAbortMultiPartObject(s *S3Conf) { - testname := "list/abort multipart objects" - runF(testname) - - bucket := "testbucket6" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - obj := "mympuobject" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lmpu, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: list multipart upload: %v", testname, err) - return - } - - //for _, item := range lmpu.Uploads { - // fmt.Println(" -- ", *item.Key, *item.UploadId) - //} - - if !containsUID(obj, *mpu.UploadId, lmpu.Uploads) { - failF("%v: upload %v/%v not found", testname, obj, *mpu.UploadId) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err != nil { - failF("%v: abort multipart upload: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lmpu, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: list multipart upload: %v", testname, err) - return - } - - if len(lmpu.Uploads) != 0 { - for _, item := range lmpu.Uploads { - fmt.Println(" D- ", *item.Key, *item.UploadId) + if out.ContentLength != dataLength { + return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) } - failF("%v: unexpected multipart uploads found", testname) - return - } - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) + bdy, err := io.ReadAll(out.Body) + if err != nil { + return err + } + defer out.Body.Close() + outCsum := sha256.Sum256(bdy) + if outCsum != csum { + return fmt.Errorf("invalid object data") + } + + err = teardown(s, dstBucket) + if err != nil { + return nil + } + + return nil + }) } -func TestListMultiParts(s *S3Conf) { - testname := "list multipart parts" - runF(testname) - - bucket := "testbucket7" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - obj := "mympuobject" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, +func PutObjectTagging_non_existing_object(s *S3Conf) { + testName := "PutObjectTagging_non_existing_object" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + Tagging: &types.Tagging{TagSet: []types.Tag{}}}) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + return nil }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } - - // check list parts of no parts is good - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err != nil { - failF("%v: list parts: %v", testname, err) - return - } - - if len(lp.Parts) != 0 { - failF("%v: list parts: expected no parts, got %v", - testname, len(lp.Parts)) - return - } - - // upload 1 part and check list parts - size5MB := 5 * 1024 * 1024 - dr := NewDataReader(size5MB, size5MB) - - datafile := "rand.data" - w, err := os.Create(datafile) - if err != nil { - failF("%v: create %v: %v", testname, datafile, err) - return - } - defer w.Close() - - _, err = io.Copy(w, dr) - if err != nil { - failF("%v: write %v: %v", testname, datafile, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: 42, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), - }) - cancel() - if err != nil { - failF("%v: multipart put part: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lp, err = s3client.ListParts(ctx, &s3.ListPartsInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err != nil { - failF("%v: list parts: %v", testname, err) - return - } - - //for _, part := range lp.Parts { - // fmt.Println(" -- ", part.PartNumber, part.ETag) - //} - - if len(lp.Parts) != 1 || lp.Parts[0].PartNumber != 42 { - fmt.Printf("%+v, %v, %v\n", lp.Parts, *lp.Key, *lp.UploadId) - failF("%v: list parts: unexpected parts listing", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestIncorrectMultiParts(s *S3Conf) { - testname := "incorrect multipart parts" - runF(testname) +func PutObjectTagging_success(s *S3Conf) { + testName := "PutObjectTagging_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + tagging := types.Tagging{TagSet: []types.Tag{{Key: getPtr("key1"), Value: getPtr("val2")}, {Key: getPtr("key2"), Value: getPtr("val2")}}} + err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } - bucket := "testbucket8" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + Tagging: &tagging}) + cancel() + if err != nil { + return err + } - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - obj := "mympuobject" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, + return nil }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } - - // upload 2 parts - size5MB := 5 * 1024 * 1024 - dr := NewDataReader(size5MB, size5MB) - - datafile := "rand.data" - w, err := os.Create(datafile) - if err != nil { - failF("%v: create %v: %v", testname, datafile, err) - return - } - defer w.Close() - - _, err = io.Copy(w, dr) - if err != nil { - failF("%v: write %v: %v", testname, datafile, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - mp1, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: 42, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), - }) - cancel() - if err != nil { - failF("%v: multipart put part 1: %v", testname, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - mp2, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: 96, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), - }) - cancel() - if err != nil { - failF("%v: multipart put part 2: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - MultipartUpload: &types.CompletedMultipartUpload{ - Parts: []types.CompletedPart{ - { - ETag: mp2.ETag, - PartNumber: 96, - }, - { - ETag: mp1.ETag, - PartNumber: 99, - }, - }, - }, - }) - cancel() - if err == nil { - failF("%v: complete multipart expected err", testname) - return - } - - badEtag := "bogusEtagValue" - - // Empty multipart upload - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err == nil { - failF("%v: complete multipart expected err", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - MultipartUpload: &types.CompletedMultipartUpload{ - Parts: []types.CompletedPart{ - { - ETag: mp2.ETag, - PartNumber: 96, - }, - { - ETag: &badEtag, - PartNumber: 42, - }, - }, - }, - }) - cancel() - if err == nil { - failF("%v: complete multipart expected err", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - MultipartUpload: &types.CompletedMultipartUpload{ - Parts: []types.CompletedPart{ - { - ETag: mp1.ETag, - PartNumber: 42, - }, - { - ETag: mp2.ETag, - PartNumber: 96, - }, - }, - }, - }) - cancel() - if err != nil { - failF("%v: complete multipart: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - oi, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - failF("%v: head object %v: %v", testname, obj, err) - return - } - - if oi.ContentLength != (int64(size5MB) * 2) { - failF("%v: object len expected %v, got %v", - testname, int64(size5MB)*2, oi.ContentLength) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestIncompleteMultiParts(s *S3Conf) { - testname := "incomplete multipart parts" - runF(testname) - - bucket := "testbucket9" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - obj := "mympuobject" - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, +func GetObjectTagging_non_existing_object(s *S3Conf) { + testName := "GetObjectTagging_non_existing_object" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + return nil }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } - - // upload 2 parts - size5MB := 5 * 1024 * 1024 - size1MB := 1024 * 1024 - dr := NewDataReader(size1MB, size1MB) - - datafile := "rand.data" - w, err := os.Create(datafile) - if err != nil { - failF("%v: create %v: %v", testname, datafile, err) - return - } - defer w.Close() - - _, err = io.Copy(w, dr) - if err != nil { - failF("%v: write %v: %v", testname, datafile, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: 42, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), - }) - cancel() - if err == nil { - failF("%v: multipart put short part expected error", testname) - return - } - - // check list parts does not have incomplete part - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mpu.UploadId, - }) - cancel() - if err != nil { - failF("%v: list parts: %v", testname, err) - return - } - - if containsPart(42, lp.Parts) { - failF("%v: list parts: found incomplete part", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestIncompletePutObject(s *S3Conf) { - testname := "test incomplete put object" - runF(testname) +func GetObjectTagging_success(s *S3Conf) { + testName := "PutObjectTagging_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + tagging := types.Tagging{TagSet: []types.Tag{{Key: getPtr("key1"), Value: getPtr("val2")}, {Key: getPtr("key2"), Value: getPtr("val2")}}} + err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } - bucket := "testbucket10" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + Tagging: &tagging}) + cancel() + if err != nil { + return err + } - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return nil + } - // use funny size to prevent accidental alignments - datalen := 1234567 - shortdatalen := 12345 - data := make([]byte, shortdatalen) - rand.Read(data) - r := bytes.NewReader(data) + if !areTagsSame(out.TagSet, tagging.TagSet) { + return fmt.Errorf("expected %v instead got %v", tagging.TagSet, out.TagSet) + } - name := "myobject" - s3client := s3.NewFromConfig(s.Config()) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &name, - Body: r, - ContentLength: int64(datalen), + return nil }) - cancel() - if err == nil { - failF("%v: expected error for short data put", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &name, - }) - defer cancel() - if err == nil { - failF("%v: expected object not exist", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestRangeGet(s *S3Conf) { - testname := "test range get" - runF(testname) - - bucket := "testbucket11" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - datalen := 10 * 1024 - data := make([]byte, datalen) - rand.Read(data) - r := bytes.NewReader(data) - - name := "myobject" - s3client := s3.NewFromConfig(s.Config()) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &name, - Body: r, +func DeleteObjectTagging_non_existing_object(s *S3Conf) { + testName := "DeleteObjectTagging_non_existing_object" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: getPtr("my-obj"), + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchKey)); err != nil { + return err + } + return nil }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - lgRange := "bytes=20000000-30000000000" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - Range: &lgRange, - }) - cancel() - if err == nil { - failF("%v: expected range error", testname) - return - } - - // Invalid range - invRange := "bytes=100-asd" - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - Range: &invRange, - }) - defer cancel() - if err == nil { - failF("%v: expected range error", testname) - return - } - - rangeString := "bytes=100-200" - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - Range: &rangeString, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - defer out.Body.Close() - - if getString(out.ContentRange) != fmt.Sprintf("bytes 100-200/%v", datalen) { - failF("%v: expected content range: %v, instead got: %v", testname, fmt.Sprintf("bytes 100-200/%v", datalen), getString(out.ContentRange)) - return - } - if getString(out.AcceptRanges) != rangeString { - failF("%v: expected accept range: %v, instead got: %v", testname, rangeString, getString(out.AcceptRanges)) - } - - b, err := io.ReadAll(out.Body) - if err != nil { - failF("%v: read body %v", testname, err) - return - } - - // bytes range is inclusive, go range for second value is not - if !isEqual(b, data[100:201]) { - failF("%v: data mismatch of range", testname) - return - } - - rangeString = "bytes=100-" - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err = s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &name, - Range: &rangeString, - }) - defer cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - defer out.Body.Close() - - b, err = io.ReadAll(out.Body) - if err != nil { - failF("%v: read body %v", testname, err) - return - } - - // bytes range is inclusive, go range for second value is not - if !isEqual(b, data[100:]) { - failF("%v: data mismatch of range", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } -func TestInvalidMultiParts(s *S3Conf) { - testname := "invalid multipart parts" - runF(testname) +func DeleteObjectTagging_success(s *S3Conf) { + testName := "DeleteObjectTagging_success" + actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + tagging := types.Tagging{TagSet: []types.Tag{{Key: getPtr("key1"), Value: getPtr("val2")}, {Key: getPtr("key2"), Value: getPtr("val2")}}} + err := putObjects(s3client, []string{obj}, bucket) + if err != nil { + return err + } - bucket := "bucket12" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + Tagging: &tagging}) + cancel() + if err != nil { + return err + } - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return nil + } - s3client := s3.NewFromConfig(s.Config()) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &obj, + }) + cancel() + if err != nil { + return nil + } - obj := "mympuobject" + if len(out.TagSet) > 0 { + return fmt.Errorf("expected empty tag set, instead got %v", out.TagSet) + } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, + return nil }) - cancel() - if err != nil { - failF("%v: create multipart upload: %v", testname, err) - return - } - - // upload 2 parts - size5MB := 5 * 1024 * 1024 - dr := NewDataReader(size5MB, size5MB) - - datafile := "rand.data" - w, err := os.Create(datafile) - if err != nil { - failF("%v: create %v: %v", testname, datafile, err) - return - } - defer w.Close() - - _, err = io.Copy(w, dr) - if err != nil { - failF("%v: write %v: %v", testname, datafile, err) - return - } - - _, err = w.Seek(0, io.SeekStart) - if err != nil { - failF("%v: seek %v: %v", testname, datafile, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - PartNumber: -1, - UploadId: mpu.UploadId, - Body: w, - ContentLength: int64(size5MB), - }) - cancel() - if err == nil { - failF("%v: multipart put part 1 expected error", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err == nil { - failF("%v: head object %v expected error", testname, obj) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) } +// func TestPutGetMPObject(s *S3Conf) { +// testname := "test put/get multipart object" +// runF(testname) + +// bucket := "testbucket2" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// name := "mympuobject" +// s3client := s3.NewFromConfig(s.Config()) + +// datalen := 10*1024*1024 + 15 +// dr := NewDataReader(datalen, 5*1024*1024) +// WithPartSize(5 * 1024 * 1024) +// s.PartSize = 5 * 1024 * 1024 +// err = s.UploadData(dr, bucket, name) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ +// Bucket: &bucket, +// Key: &name, +// }) +// defer cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// defer out.Body.Close() + +// if out.ContentLength != int64(datalen) { +// failF("%v: content length got %v expected %v", testname, out.ContentLength, datalen) +// return +// } + +// b := make([]byte, 1048576) +// h := sha256.New() +// for { +// n, err := out.Body.Read(b) +// if err == io.EOF { +// h.Write(b[:n]) +// break +// } +// if err != nil { +// failF("%v: read %v", err) +// return +// } +// h.Write(b[:n]) +// } + +// if !isEqual(dr.Sum(), h.Sum(nil)) { +// failF("%v: checksum got %x expected %x", testname, h.Sum(nil), dr.Sum()) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestPutDirObject(s *S3Conf) { +// testname := "test put directory object" +// runF(testname) + +// bucket := "testbucket3" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// name := "myobjectdir/" +// s3client := s3.NewFromConfig(s.Config()) + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ +// Bucket: &bucket, +// Key: &name, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket, MaxKeys: -4}) +// cancel() +// if err == nil { +// failF("%v: expected invalid argument error", testname) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) +// cancel() +// if err != nil { +// failF("failed to list objects: %v", err) +// return +// } + +// if !contains(name, out.Contents) { +// failF("directory object not found") +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestListObject(s *S3Conf) { +// testname := "list objects" +// runF(testname) + +// bucket := "testbucket4" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// dir1 := "myobjectdir/" +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ +// Bucket: &bucket, +// Key: &dir1, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// obj1 := "myobjectdir/myobject" +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ +// Bucket: &bucket, +// Key: &obj1, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// obj2 := "myobjectdir1/myobject" +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ +// Bucket: &bucket, +// Key: &obj2, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// // put: +// // "myobjectdir/" +// // "myobjectdir/myobject" +// // "myobjectdir1/myobject" +// // should return: +// // "myobjectdir/myobject" +// // "myobjectdir1/myobject" + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) +// cancel() +// if err != nil { +// failF("failed to list objects: %v", err) +// return +// } + +// if !contains(obj1, out.Contents) { +// failF("object %v not found", obj1) +// return +// } +// if !contains(obj2, out.Contents) { +// failF("object %v not found", obj2) +// return +// } +// if out.KeyCount != 2 { +// failF("%v: expected key count: %v, instead got: %v", testname, 2, out.KeyCount) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ +// Bucket: &bucket, +// Key: &obj1, +// }) +// cancel() +// if err != nil { +// failF("failed to delete %v: %v", obj1, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.DeleteObject(ctx, &s3.DeleteObjectInput{ +// Bucket: &bucket, +// Key: &obj2, +// }) +// cancel() +// if err != nil { +// failF("failed to delete %v: %v", obj2, err) +// return +// } + +// // put: +// // "myobjectdir/" +// // "myobjectdir/myobject" +// // "myobjectdir1/myobject" +// // delete: +// // "myobjectdir/myobject" +// // "myobjectdir1/myobject" +// // should return: +// // "myobjectdir/" + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// out, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket}) +// cancel() +// if err != nil { +// failF("failed to list objects: %v", err) +// return +// } + +// if !contains(dir1, out.Contents) { +// failF("dir %v not found", dir1) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestListAbortMultiPartObject(s *S3Conf) { +// testname := "list/abort multipart objects" +// runF(testname) + +// bucket := "testbucket6" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// obj := "mympuobject" + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: create multipart upload: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// lmpu, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ +// Bucket: &bucket, +// }) +// cancel() +// if err != nil { +// failF("%v: list multipart upload: %v", testname, err) +// return +// } + +// //for _, item := range lmpu.Uploads { +// // fmt.Println(" -- ", *item.Key, *item.UploadId) +// //} + +// if !containsUID(obj, *mpu.UploadId, lmpu.Uploads) { +// failF("%v: upload %v/%v not found", testname, obj, *mpu.UploadId) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// }) +// cancel() +// if err != nil { +// failF("%v: abort multipart upload: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// lmpu, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ +// Bucket: &bucket, +// }) +// cancel() +// if err != nil { +// failF("%v: list multipart upload: %v", testname, err) +// return +// } + +// if len(lmpu.Uploads) != 0 { +// for _, item := range lmpu.Uploads { +// fmt.Println(" D- ", *item.Key, *item.UploadId) +// } +// failF("%v: unexpected multipart uploads found", testname) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestListMultiParts(s *S3Conf) { +// testname := "list multipart parts" +// runF(testname) + +// bucket := "testbucket7" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// obj := "mympuobject" + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: create multipart upload: %v", testname, err) +// return +// } + +// // check list parts of no parts is good +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// }) +// cancel() +// if err != nil { +// failF("%v: list parts: %v", testname, err) +// return +// } + +// if len(lp.Parts) != 0 { +// failF("%v: list parts: expected no parts, got %v", +// testname, len(lp.Parts)) +// return +// } + +// // upload 1 part and check list parts +// size5MB := 5 * 1024 * 1024 +// dr := NewDataReader(size5MB, size5MB) + +// datafile := "rand.data" +// w, err := os.Create(datafile) +// if err != nil { +// failF("%v: create %v: %v", testname, datafile, err) +// return +// } +// defer w.Close() + +// _, err = io.Copy(w, dr) +// if err != nil { +// failF("%v: write %v: %v", testname, datafile, err) +// return +// } + +// _, err = w.Seek(0, io.SeekStart) +// if err != nil { +// failF("%v: seek %v: %v", testname, datafile, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ +// Bucket: &bucket, +// Key: &obj, +// PartNumber: 42, +// UploadId: mpu.UploadId, +// Body: w, +// ContentLength: int64(size5MB), +// }) +// cancel() +// if err != nil { +// failF("%v: multipart put part: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// lp, err = s3client.ListParts(ctx, &s3.ListPartsInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// }) +// cancel() +// if err != nil { +// failF("%v: list parts: %v", testname, err) +// return +// } + +// //for _, part := range lp.Parts { +// // fmt.Println(" -- ", part.PartNumber, part.ETag) +// //} + +// if len(lp.Parts) != 1 || lp.Parts[0].PartNumber != 42 { +// fmt.Printf("%+v, %v, %v\n", lp.Parts, *lp.Key, *lp.UploadId) +// failF("%v: list parts: unexpected parts listing", testname) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestIncorrectMultiParts(s *S3Conf) { +// testname := "incorrect multipart parts" +// runF(testname) + +// bucket := "testbucket8" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// obj := "mympuobject" + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: create multipart upload: %v", testname, err) +// return +// } + +// // upload 2 parts +// size5MB := 5 * 1024 * 1024 +// dr := NewDataReader(size5MB, size5MB) + +// datafile := "rand.data" +// w, err := os.Create(datafile) +// if err != nil { +// failF("%v: create %v: %v", testname, datafile, err) +// return +// } +// defer w.Close() + +// _, err = io.Copy(w, dr) +// if err != nil { +// failF("%v: write %v: %v", testname, datafile, err) +// return +// } + +// _, err = w.Seek(0, io.SeekStart) +// if err != nil { +// failF("%v: seek %v: %v", testname, datafile, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// mp1, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ +// Bucket: &bucket, +// Key: &obj, +// PartNumber: 42, +// UploadId: mpu.UploadId, +// Body: w, +// ContentLength: int64(size5MB), +// }) +// cancel() +// if err != nil { +// failF("%v: multipart put part 1: %v", testname, err) +// return +// } + +// _, err = w.Seek(0, io.SeekStart) +// if err != nil { +// failF("%v: seek %v: %v", testname, datafile, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// mp2, err := s3client.UploadPart(ctx, &s3.UploadPartInput{ +// Bucket: &bucket, +// Key: &obj, +// PartNumber: 96, +// UploadId: mpu.UploadId, +// Body: w, +// ContentLength: int64(size5MB), +// }) +// cancel() +// if err != nil { +// failF("%v: multipart put part 2: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// MultipartUpload: &types.CompletedMultipartUpload{ +// Parts: []types.CompletedPart{ +// { +// ETag: mp2.ETag, +// PartNumber: 96, +// }, +// { +// ETag: mp1.ETag, +// PartNumber: 99, +// }, +// }, +// }, +// }) +// cancel() +// if err == nil { +// failF("%v: complete multipart expected err", testname) +// return +// } + +// badEtag := "bogusEtagValue" + +// // Empty multipart upload +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// }) +// cancel() +// if err == nil { +// failF("%v: complete multipart expected err", testname) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// MultipartUpload: &types.CompletedMultipartUpload{ +// Parts: []types.CompletedPart{ +// { +// ETag: mp2.ETag, +// PartNumber: 96, +// }, +// { +// ETag: &badEtag, +// PartNumber: 42, +// }, +// }, +// }, +// }) +// cancel() +// if err == nil { +// failF("%v: complete multipart expected err", testname) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// MultipartUpload: &types.CompletedMultipartUpload{ +// Parts: []types.CompletedPart{ +// { +// ETag: mp1.ETag, +// PartNumber: 42, +// }, +// { +// ETag: mp2.ETag, +// PartNumber: 96, +// }, +// }, +// }, +// }) +// cancel() +// if err != nil { +// failF("%v: complete multipart: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// oi, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: head object %v: %v", testname, obj, err) +// return +// } + +// if oi.ContentLength != (int64(size5MB) * 2) { +// failF("%v: object len expected %v, got %v", +// testname, int64(size5MB)*2, oi.ContentLength) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestIncompleteMultiParts(s *S3Conf) { +// testname := "incomplete multipart parts" +// runF(testname) + +// bucket := "testbucket9" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// obj := "mympuobject" + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: create multipart upload: %v", testname, err) +// return +// } + +// // upload 2 parts +// size5MB := 5 * 1024 * 1024 +// size1MB := 1024 * 1024 +// dr := NewDataReader(size1MB, size1MB) + +// datafile := "rand.data" +// w, err := os.Create(datafile) +// if err != nil { +// failF("%v: create %v: %v", testname, datafile, err) +// return +// } +// defer w.Close() + +// _, err = io.Copy(w, dr) +// if err != nil { +// failF("%v: write %v: %v", testname, datafile, err) +// return +// } + +// _, err = w.Seek(0, io.SeekStart) +// if err != nil { +// failF("%v: seek %v: %v", testname, datafile, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ +// Bucket: &bucket, +// Key: &obj, +// PartNumber: 42, +// UploadId: mpu.UploadId, +// Body: w, +// ContentLength: int64(size5MB), +// }) +// cancel() +// if err == nil { +// failF("%v: multipart put short part expected error", testname) +// return +// } + +// // check list parts does not have incomplete part +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// lp, err := s3client.ListParts(ctx, &s3.ListPartsInput{ +// Bucket: &bucket, +// Key: &obj, +// UploadId: mpu.UploadId, +// }) +// cancel() +// if err != nil { +// failF("%v: list parts: %v", testname, err) +// return +// } + +// if containsPart(42, lp.Parts) { +// failF("%v: list parts: found incomplete part", testname) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestIncompletePutObject(s *S3Conf) { +// testname := "test incomplete put object" +// runF(testname) + +// bucket := "testbucket10" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// // use funny size to prevent accidental alignments +// datalen := 1234567 +// shortdatalen := 12345 +// data := make([]byte, shortdatalen) +// rand.Read(data) +// r := bytes.NewReader(data) + +// name := "myobject" +// s3client := s3.NewFromConfig(s.Config()) + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ +// Bucket: &bucket, +// Key: &name, +// Body: r, +// ContentLength: int64(datalen), +// }) +// cancel() +// if err == nil { +// failF("%v: expected error for short data put", testname) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ +// Bucket: &bucket, +// Key: &name, +// }) +// defer cancel() +// if err == nil { +// failF("%v: expected object not exist", testname) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestInvalidMultiParts(s *S3Conf) { +// testname := "invalid multipart parts" +// runF(testname) + +// bucket := "bucket12" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// obj := "mympuobject" + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// mpu, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: create multipart upload: %v", testname, err) +// return +// } + +// // upload 2 parts +// size5MB := 5 * 1024 * 1024 +// dr := NewDataReader(size5MB, size5MB) + +// datafile := "rand.data" +// w, err := os.Create(datafile) +// if err != nil { +// failF("%v: create %v: %v", testname, datafile, err) +// return +// } +// defer w.Close() + +// _, err = io.Copy(w, dr) +// if err != nil { +// failF("%v: write %v: %v", testname, datafile, err) +// return +// } + +// _, err = w.Seek(0, io.SeekStart) +// if err != nil { +// failF("%v: seek %v: %v", testname, datafile, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ +// Bucket: &bucket, +// Key: &obj, +// PartNumber: -1, +// UploadId: mpu.UploadId, +// Body: w, +// ContentLength: int64(size5MB), +// }) +// cancel() +// if err == nil { +// failF("%v: multipart put part 1 expected error", testname) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.HeadObject(ctx, &s3.HeadObjectInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err == nil { +// failF("%v: head object %v expected error", testname, obj) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestPutGetRemoveTags(s *S3Conf) { +// testname := "test put/get/remove object tags" +// runF(testname) + +// bucket := "testbucket13" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// obj := "myobject" +// s3client := s3.NewFromConfig(s.Config()) + +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ +// Bucket: &bucket, +// Key: &obj, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// key1 := "hello1" +// key2 := "hello2" +// val1 := "world1" +// val2 := "world2" + +// tagging := types.Tagging{TagSet: []types.Tag{{Key: &key1, Value: &val1}, {Key: &key2, Value: &val2}}} + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ +// Bucket: &bucket, +// Key: &obj, +// Tagging: &tagging, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ +// Key: &obj, +// Bucket: &bucket, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// ok := areTagsSame(tagging.TagSet, out.TagSet) +// if !ok { +// failF("%v: expected %v instead got %v", testname, tagging.TagSet, out.TagSet) +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ +// Key: &obj, +// Bucket: &bucket, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// out, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ +// Key: &obj, +// Bucket: &bucket, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// if len(out.TagSet) > 0 { +// failF("%v: expected empty tag set instead got %v", testname, out.TagSet) +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + +// func TestAclActions(s *S3Conf) { +// testname := "test put/get acl" +// runF(testname) + +// bucket := "testbucket14" + +// err := setup(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// s3client := s3.NewFromConfig(s.Config()) + +// rootAccess := s.awsID +// rootSecret := s.awsSecret + +// s.awsID = "grt1" +// s.awsSecret = "grt1secret" + +// userS3Client := s3.NewFromConfig(s.Config()) + +// s.awsID = rootAccess +// s.awsSecret = rootSecret + +// grt1 := "grt1" + +// grants := []types.Grant{ +// { +// Permission: "READ", +// Grantee: &types.Grantee{ +// ID: &grt1, +// Type: "CanonicalUser", +// }, +// }, +// } + +// succUsrCrt := "The user has been created successfully" +// failUsrCrt := "failed to create a user: update iam data: account already exists" + +// out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "create-user", "-a", grt1, "-s", "grt1secret", "-r", "user") +// if err != nil { +// failF("%v: %v", err) +// return +// } +// if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) { +// failF("%v: failed to create user accounts", testname) +// return +// } + +// // Validation error case +// ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ +// Bucket: &bucket, +// AccessControlPolicy: &types.AccessControlPolicy{ +// Grants: grants, +// }, +// ACL: "private", +// }) +// cancel() +// if err == nil { +// failF("%v: expected validation error", testname) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ +// Bucket: &bucket, +// AccessControlPolicy: &types.AccessControlPolicy{ +// Grants: grants, +// Owner: &types.Owner{ID: &s.awsID}, +// }, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// acl, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{ +// Bucket: &bucket, +// }) +// cancel() +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } + +// if *acl.Owner.ID != s.awsID { +// failF("%v: expected bucket owner: %v, instead got: %v", testname, s.awsID, *acl.Owner.ID) +// return +// } +// if !checkGrants(acl.Grants, grants) { +// failF("%v: expected %v, instead got %v", testname, grants, acl.Grants) +// return +// } + +// ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) +// _, err = userS3Client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ +// Bucket: &bucket, +// }) +// cancel() +// if err == nil { +// failF("%v: expected acl access denied error", testname) +// return +// } + +// err = teardown(s, bucket) +// if err != nil { +// failF("%v: %v", testname, err) +// return +// } +// passF(testname) +// } + type prefResult struct { elapsed time.Duration size int64 @@ -1360,233 +1796,3 @@ func TestPerformance(s *S3Conf, upload, download bool, files int, objectSize int return nil } - -func TestPutGetRemoveTags(s *S3Conf) { - testname := "test put/get/remove object tags" - runF(testname) - - bucket := "testbucket13" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - obj := "myobject" - s3client := s3.NewFromConfig(s.Config()) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - key1 := "hello1" - key2 := "hello2" - val1 := "world1" - val2 := "world2" - - tagging := types.Tagging{TagSet: []types.Tag{{Key: &key1, Value: &val1}, {Key: &key2, Value: &val2}}} - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ - Bucket: &bucket, - Key: &obj, - Tagging: &tagging, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Key: &obj, - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ok := areTagsSame(tagging.TagSet, out.TagSet) - if !ok { - failF("%v: expected %v instead got %v", testname, tagging.TagSet, out.TagSet) - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.DeleteObjectTagging(ctx, &s3.DeleteObjectTaggingInput{ - Key: &obj, - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - out, err = s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ - Key: &obj, - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - if len(out.TagSet) > 0 { - failF("%v: expected empty tag set instead got %v", testname, out.TagSet) - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -func TestAclActions(s *S3Conf) { - testname := "test put/get acl" - runF(testname) - - bucket := "testbucket14" - - err := setup(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - - s3client := s3.NewFromConfig(s.Config()) - - rootAccess := s.awsID - rootSecret := s.awsSecret - - s.awsID = "grt1" - s.awsSecret = "grt1secret" - - userS3Client := s3.NewFromConfig(s.Config()) - - s.awsID = rootAccess - s.awsSecret = rootSecret - - grt1 := "grt1" - - grants := []types.Grant{ - { - Permission: "READ", - Grantee: &types.Grantee{ - ID: &grt1, - Type: "CanonicalUser", - }, - }, - } - - succUsrCrt := "The user has been created successfully" - failUsrCrt := "failed to create a user: update iam data: account already exists" - - out, err := execCommand("admin", "-a", s.awsID, "-s", s.awsSecret, "create-user", "-a", grt1, "-s", "grt1secret", "-r", "user") - if err != nil { - failF("%v: %v", err) - return - } - if !strings.Contains(string(out), succUsrCrt) && !strings.Contains(string(out), failUsrCrt) { - failF("%v: failed to create user accounts", testname) - return - } - - // Validation error case - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ - Bucket: &bucket, - AccessControlPolicy: &types.AccessControlPolicy{ - Grants: grants, - }, - ACL: "private", - }) - cancel() - if err == nil { - failF("%v: expected validation error", testname) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ - Bucket: &bucket, - AccessControlPolicy: &types.AccessControlPolicy{ - Grants: grants, - Owner: &types.Owner{ID: &s.awsID}, - }, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - acl, err := s3client.GetBucketAcl(ctx, &s3.GetBucketAclInput{ - Bucket: &bucket, - }) - cancel() - if err != nil { - failF("%v: %v", testname, err) - return - } - - if *acl.Owner.ID != s.awsID { - failF("%v: expected bucket owner: %v, instead got: %v", testname, s.awsID, *acl.Owner.ID) - return - } - if !checkGrants(acl.Grants, grants) { - failF("%v: expected %v, instead got %v", testname, grants, acl.Grants) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) - _, err = userS3Client.PutBucketAcl(ctx, &s3.PutBucketAclInput{ - Bucket: &bucket, - }) - cancel() - if err == nil { - failF("%v: expected acl access denied error", testname) - return - } - - err = teardown(s, bucket) - if err != nil { - failF("%v: %v", testname, err) - return - } - passF(testname) -} - -// Full flow test -func TestFullFlow(s *S3Conf) { - // TODO: add more test cases to get 100% coverage - TestMakeBucket(s) - TestPutGetObject(s) - TestPutGetMPObject(s) - TestPutDirObject(s) - TestListObject(s) - TestIncompletePutObject(s) - TestListMultiParts(s) - TestIncompleteMultiParts(s) - TestIncorrectMultiParts(s) - TestListAbortMultiPartObject(s) - TestRangeGet(s) - TestInvalidMultiParts(s) - TestPutGetRemoveTags(s) - TestAclActions(s) -} diff --git a/integration/utils.go b/integration/utils.go index 46ba3db..dff7b6d 100644 --- a/integration/utils.go +++ b/integration/utils.go @@ -1,15 +1,30 @@ package integration import ( + "bytes" "context" + "crypto/rand" + "crypto/sha256" + "errors" "fmt" "os/exec" "strings" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/versity/versitygw/s3err" ) +var ( + bcktCount = 0 +) + +func getBucketName() string { + bcktCount++ + return fmt.Sprintf("test-bucket-%v", bcktCount) +} + func setup(s *S3Conf, bucket string) error { s3client := s3.NewFromConfig(s.Config()) @@ -69,6 +84,78 @@ func teardown(s *S3Conf, bucket string) error { return err } +func actionHandler(s *S3Conf, testName string, handler func(s3client *s3.Client, bucket string) error) { + runF(testName) + bucketName := getBucketName() + err := setup(s, bucketName) + if err != nil { + failF("%v: failed to create a bucket: %v", testName, err.Error()) + return + } + client := s3.NewFromConfig(s.Config()) + handlerErr := handler(client, bucketName) + if handlerErr != nil { + failF("%v: %v", testName, handlerErr.Error()) + } + + err = teardown(s, bucketName) + if err != nil { + if handlerErr == nil { + failF("%v: failed to delete the bucket: %v", testName, err.Error()) + } else { + fmt.Printf(colorRed+"%v: failed to delete the bucket: %v", testName, err.Error()) + } + } + if handlerErr == nil { + passF(testName) + } +} + +func putObjects(client *s3.Client, objs []string, bucket string) error { + for _, key := range objs { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Key: &key, + Bucket: &bucket, + }) + cancel() + if err != nil { + return err + } + } + return nil +} + +func checkApiErr(err error, apiErr s3err.APIError) error { + if err == nil { + return fmt.Errorf("expected %v, instead got nil", apiErr.Code) + } + var ae smithy.APIError + if errors.As(err, &ae) { + if ae.ErrorCode() == apiErr.Code && ae.ErrorMessage() == apiErr.Description { + return nil + } + + return fmt.Errorf("expected %v, instead got %v", apiErr.Code, ae.ErrorCode()) + } else { + return fmt.Errorf("expected aws api error, instead got: %v", err.Error()) + } +} + +func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) (csum [32]byte, data []byte, err error) { + data = make([]byte, lgth) + rand.Read(data) + csum = sha256.Sum256(data) + r := bytes.NewReader(data) + input.Body = r + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = client.PutObject(ctx, input) + cancel() + + return +} + func isEqual(a, b []byte) bool { if len(a) != len(b) { return false @@ -162,6 +249,10 @@ func getString(str *string) string { return *str } +func getPtr(str string) *string { + return &str +} + func areMapsSame(mp1, mp2 map[string]string) bool { if len(mp1) != len(mp2) { return false