Files
versitygw/tests/integration/CreateMultipartUpload.go
niksis02 d05f25f277 feat: refactoring of the integration tests
All the integration tests used to be in a single file, which had become large, messy, and difficult to maintain. These changes split `tests.go` into multiple files, organized by logical test groups.
2025-10-31 20:53:55 +04:00

582 lines
20 KiB
Go

// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package integration
import (
"context"
"fmt"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
)
func CreateMultipartUpload_non_existing_bucket(s *S3Conf) error {
testName := "CreateMultipartUpload_non_existing_bucket"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
bucketName := getBucketName()
_, err := createMp(s3client, bucketName, "my-obj")
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
return err
}
return nil
})
}
func CreateMultipartUpload_with_metadata(s *S3Conf) error {
testName := "CreateMultipartUpload_with_metadata"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
meta := map[string]string{
"prop1": "val1",
"prop2": "val2",
}
cType, cEnc, cDesp, cLang := "application/text", "testenc", "testdesp", "sp"
cacheControl, expires := "no-cache", time.Now().Add(time.Hour*5)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
Metadata: meta,
ContentType: &cType,
ContentEncoding: &cEnc,
ContentDisposition: &cDesp,
ContentLanguage: &cLang,
CacheControl: &cacheControl,
Expires: &expires,
})
cancel()
if err != nil {
return err
}
parts, _, err := uploadParts(s3client, 100, 1, bucket, obj, *out.UploadId)
if err != nil {
return err
}
compParts := []types.CompletedPart{}
for _, el := range parts {
compParts = append(compParts, types.CompletedPart{
ETag: el.ETag,
PartNumber: el.PartNumber,
})
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: out.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: compParts,
},
})
cancel()
if err != nil {
return err
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
resp, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
})
cancel()
if err != nil {
return err
}
if !areMapsSame(resp.Metadata, meta) {
return fmt.Errorf("expected uploaded object metadata to be %v, instead got %v",
meta, resp.Metadata)
}
if getString(resp.ContentType) != cType {
return fmt.Errorf("expected uploaded object content-type to be %v, instead got %v",
cType, getString(resp.ContentType))
}
if getString(resp.ContentEncoding) != cEnc {
return fmt.Errorf("expected uploaded object content-encoding to be %v, instead got %v",
cEnc, getString(resp.ContentEncoding))
}
if getString(resp.ContentLanguage) != cLang {
return fmt.Errorf("expected uploaded object content-language to be %v, instead got %v",
cLang, getString(resp.ContentLanguage))
}
if getString(resp.ContentDisposition) != cDesp {
return fmt.Errorf("expected uploaded object content-disposition to be %v, instead got %v",
cDesp, getString(resp.ContentDisposition))
}
if getString(resp.CacheControl) != cacheControl {
return fmt.Errorf("expected uploaded object cache-control to be %v, instead got %v",
cacheControl, getString(resp.CacheControl))
}
if getString(resp.ExpiresString) != expires.UTC().Format(timefmt) {
return fmt.Errorf("expected uploaded object content-encoding to be %v, instead got %v",
expires.UTC().Format(timefmt), getString(resp.ExpiresString))
}
return nil
})
}
func CreateMultipartUpload_with_object_lock(s *S3Conf) error {
testName := "CreateMultipartUpload_with_object_lock"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
retainUntilDate := time.Now().Add(24 * time.Hour)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatusOn,
ObjectLockMode: types.ObjectLockModeGovernance,
ObjectLockRetainUntilDate: &retainUntilDate,
})
cancel()
if err != nil {
return err
}
parts, _, err := uploadParts(s3client, 100, 1, bucket, obj, *out.UploadId)
if err != nil {
return err
}
compParts := []types.CompletedPart{}
for _, el := range parts {
compParts = append(compParts, types.CompletedPart{
ETag: el.ETag,
PartNumber: el.PartNumber,
})
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: out.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: compParts,
},
})
cancel()
if err != nil {
return err
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
resp, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &bucket,
Key: &obj,
})
cancel()
if err != nil {
return err
}
if resp.ObjectLockLegalHoldStatus != types.ObjectLockLegalHoldStatusOn {
return fmt.Errorf("expected uploaded object legal hold status to be %v, instead got %v",
types.ObjectLockLegalHoldStatusOn, resp.ObjectLockLegalHoldStatus)
}
if resp.ObjectLockMode != types.ObjectLockModeGovernance {
return fmt.Errorf("expected uploaded object lock mode to be %v, instead got %v",
types.ObjectLockModeGovernance, resp.ObjectLockMode)
}
return cleanupLockedObjects(s3client, bucket, []objToDelete{{key: obj, removeLegalHold: true}})
}, withLock())
}
func CreateMultipartUpload_with_object_lock_not_enabled(s *S3Conf) error {
testName := "CreateMultipartUpload_with_object_lock_not_enabled"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatusOn,
})
cancel()
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidBucketObjectLockConfiguration)); err != nil {
return err
}
return nil
})
}
func CreateMultipartUpload_with_object_lock_invalid_retention(s *S3Conf) error {
testName := "CreateMultipartUpload_with_object_lock_invalid_retention"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
retentionDate := time.Now().Add(24 * time.Hour)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
ObjectLockMode: types.ObjectLockModeGovernance,
})
cancel()
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders)); err != nil {
return err
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
ObjectLockRetainUntilDate: &retentionDate,
})
cancel()
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrObjectLockInvalidHeaders)); err != nil {
return err
}
return nil
})
}
func CreateMultipartUpload_past_retain_until_date(s *S3Conf) error {
testName := "CreateMultipartUpload_past_retain_until_date"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
rDate := time.Now().Add(-5 * time.Hour)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
ObjectLockMode: types.ObjectLockModeGovernance,
ObjectLockRetainUntilDate: &rDate,
})
cancel()
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrPastObjectLockRetainDate)); err != nil {
return err
}
return nil
})
}
func CreateMultipartUpload_invalid_legal_hold(s *S3Conf) error {
testName := "CreateMultipartUpload_invalid_legal_hold"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: getPtr("foo"),
ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatus("invalid_status"),
})
cancel()
return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidLegalHoldStatus))
}, withLock())
}
func CreateMultipartUpload_invalid_object_lock_mode(s *S3Conf) error {
testName := "CreateMultipartUpload_invalid_object_lock_mode"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
rDate := time.Now().Add(time.Hour * 10)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: getPtr("foo"),
ObjectLockMode: types.ObjectLockMode("invalid_mode"),
ObjectLockRetainUntilDate: &rDate,
})
cancel()
return checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidObjectLockMode))
}, withLock())
}
func CreateMultipartUpload_invalid_checksum_algorithm(s *S3Conf) error {
testName := "CreateMultipartUpload_invalid_checksum_algorithm"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: getPtr("my-obj"),
ChecksumAlgorithm: types.ChecksumAlgorithm("invalid_checksum_algorithm"),
})
cancel()
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm)); err != nil {
return err
}
return nil
})
}
func CreateMultipartUpload_invalid_checksum_type(s *S3Conf) error {
testName := "CreateMultipartUpload_invalid_checksum_type"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
_, err := createMp(s3client, bucket, "my-mp", withChecksumType(types.ChecksumType("invalid_checksum_type")))
if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type")); err != nil {
return err
}
return nil
})
}
func CreateMultipartUpload_empty_checksum_algorithm_with_checksum_type(s *S3Conf) error {
testName := "CreateMultipartUpload_empty_checksum_algorithm_with_checksum_type"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
for _, el := range types.ChecksumTypeComposite.Values() {
_, err := createMp(s3client, bucket, "my-mp", withChecksumType(el))
if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrChecksumTypeWithAlgo)); err != nil {
return err
}
}
return nil
})
}
func CreateMultipartUpload_type_algo_mismatch(s *S3Conf) error {
testName := "CreateMultipartUpload_type_algo_mismatch"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
for i, test := range []struct {
chType types.ChecksumType
algo types.ChecksumAlgorithm
}{
{types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc64nvme},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha1},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmSha256},
} {
_, err := createMp(s3client, bucket, "my-obj", withChecksum(test.algo), withChecksumType(test.chType))
if err := checkApiErr(err, s3err.GetChecksumSchemaMismatchErr(test.algo, test.chType)); err != nil {
return fmt.Errorf("test %v failed: %w", i, err)
}
}
return nil
})
}
func CreateMultipartUpload_valid_algo_type(s *S3Conf) error {
testName := "CreateMultipartUpload_valid_algo_type"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
for _, test := range []struct {
chType types.ChecksumType
chAlgo types.ChecksumAlgorithm
}{
// composite type
{types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc32},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmCrc32c},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha1},
{types.ChecksumTypeComposite, types.ChecksumAlgorithmSha256},
// full object type
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc64nvme},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc32},
{types.ChecksumTypeFullObject, types.ChecksumAlgorithmCrc32c},
} {
randChType := types.ChecksumType(randomizeCase(string(test.chType)))
randChAlgo := types.ChecksumAlgorithm(randomizeCase(string(test.chAlgo)))
out, err := createMp(s3client, bucket, obj, withChecksum(randChAlgo), withChecksumType(randChType))
if err != nil {
return err
}
if out.ChecksumAlgorithm != test.chAlgo {
return fmt.Errorf("expected the checksum algorithm to be %v, instead got %v", test.chAlgo, out.ChecksumAlgorithm)
}
if out.ChecksumType != test.chType {
return fmt.Errorf("expected the checksum type to be %v, instead got %v", test.chType, out.ChecksumType)
}
}
return nil
})
}
func CreateMultipartUpload_with_tagging(s *S3Conf) error {
testName := "CreateMultipartUpload_with_tagging"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
testTagging := func(tagging string, result map[string]string, expectedErr error) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
mp, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
Tagging: &tagging,
})
cancel()
if err == nil && expectedErr != nil {
return fmt.Errorf("expected err %w, instead got nil", expectedErr)
}
if err != nil {
if expectedErr == nil {
return err
}
switch eErr := expectedErr.(type) {
case s3err.APIError:
return checkApiErr(err, eErr)
default:
return fmt.Errorf("invalid err provided: %w", expectedErr)
}
}
parts, _, err := uploadParts(s3client, 5*1024*1024, 1, bucket, obj, *mp.UploadId)
if err != nil {
return err
}
cParts := []types.CompletedPart{
{
ETag: parts[0].ETag,
PartNumber: parts[0].PartNumber,
ChecksumCRC32: parts[0].ChecksumCRC32,
},
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
_, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Key: &obj,
UploadId: mp.UploadId,
MultipartUpload: &types.CompletedMultipartUpload{
Parts: cParts,
},
})
cancel()
if err != nil {
return err
}
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{
Bucket: &bucket,
Key: &obj,
})
cancel()
if err != nil {
return err
}
if len(res.TagSet) != len(result) {
return fmt.Errorf("tag lengths are not equal: (expected): %v, (got): %v",
len(result), len(res.TagSet))
}
for _, tag := range res.TagSet {
val, ok := result[getString(tag.Key)]
if !ok {
return fmt.Errorf("tag key not found: %v", getString(tag.Key))
}
if val != getString(tag.Value) {
return fmt.Errorf("expected the %v tag value to be %v, instead got %v",
getString(tag.Key), val, getString(tag.Value))
}
}
return nil
}
for i, el := range []struct {
tagging string
result map[string]string
expectedErr error
}{
// success cases
{"&", map[string]string{}, nil},
{"&&&", map[string]string{}, nil},
{"key", map[string]string{"key": ""}, nil},
{"key&", map[string]string{"key": ""}, nil},
{"key=&", map[string]string{"key": ""}, nil},
{"key=val&", map[string]string{"key": "val"}, nil},
{"key1&key2", map[string]string{"key1": "", "key2": ""}, nil},
{"key1=val1&key2=val2", map[string]string{"key1": "val1", "key2": "val2"}, nil},
{"key@=val@", map[string]string{"key@": "val@"}, nil},
// invalid url-encoded
{"=", nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)},
{"key%", nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)},
// duplicate keys
{"key=val&key=val", nil, s3err.GetAPIError(s3err.ErrInvalidURLEncodedTagging)},
// invalid tag keys
{"key?=val", nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)},
{"key(=val", nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)},
{"key*=val", nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)},
{"key$=val", nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)},
{"key#=val", nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)},
{"key!=val", nil, s3err.GetAPIError(s3err.ErrInvalidTagKey)},
// invalid tag values
{"key=val?", nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)},
{"key=val(", nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)},
{"key=val*", nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)},
{"key=val$", nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)},
{"key=val#", nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)},
{"key=val!", nil, s3err.GetAPIError(s3err.ErrInvalidTagValue)},
// success special chars
{"key-key_key.key/key=value-value_value.value/value",
map[string]string{"key-key_key.key/key": "value-value_value.value/value"},
nil},
// should handle supported encoded characters
{"key%2E=value%2F", map[string]string{"key.": "value/"}, nil},
{"key%2D=value%2B", map[string]string{"key-": "value+"}, nil},
{"key++key=value++value", map[string]string{"key key": "value value"}, nil},
{"key%20key=value%20value", map[string]string{"key key": "value value"}, nil},
{"key%5Fkey=value%5Fvalue", map[string]string{"key_key": "value_value"}, nil},
} {
if s.azureTests {
// azure doesn't support '@' character
if strings.Contains(el.tagging, "@") {
continue
}
}
err := testTagging(el.tagging, el.result, el.expectedErr)
if err != nil {
return fmt.Errorf("test case %v faild: %w", i+1, err)
}
}
return nil
})
}
func CreateMultipartUpload_success(s *S3Conf) error {
testName := "CreateMultipartUpload_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
obj := "my-obj"
out, err := createMp(s3client, bucket, obj)
if err != nil {
return err
}
if out.Bucket == nil {
return fmt.Errorf("expected bucket name to be not nil")
}
if out.Key == nil {
return fmt.Errorf("expected object name to be not nil")
}
if *out.Bucket != bucket {
return fmt.Errorf("expected bucket name %v, instead got %v",
bucket, *out.Bucket)
}
if *out.Key != obj {
return fmt.Errorf("expected object name %v, instead got %v",
obj, *out.Key)
}
return nil
})
}