mirror of
https://github.com/versity/versitygw.git
synced 2026-09-20 06:54:47 +00:00
fix: fix azure multipart upload objects masking
As multipart uploads are translated to blobs in azure blob storage, they were visible in ListObjects(V2) as complete objects. Now the blobs with multipart prefix are filtered out during listing. The listing logic is rewritten client-side to implement proper S3 semantics: flat blob enumeration with manual delimiter handling, correct truncation (IsTruncated only set when more items genuinely exist beyond maxKeys), and StartAfter/Marker/ContinuationToken applied via the lexicographic max of both constraints in ListObjectsV2. For the same reason bucket deletion was not allowed. Now multipart objects are explicitly checked on bucket deletion and any pending multipart upload doesn't block the bucket deletion anymore.
This commit is contained in:
niksis02
@@ -585,3 +585,205 @@ func ListObjects_non_truncated_common_prefixes(s *S3Conf) error {
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// ListObjects should not list any pending multipart uploads
|
||||
// and no pending mp should block the bucket from deletion
|
||||
func ListObjects_should_not_list_pending_mps(s *S3Conf) error {
|
||||
testName := "ListObjects_should_not_list_pending_mps"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
for i := range 5 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: &bucket,
|
||||
Key: getPtr(fmt.Sprintf("obj-%d", i)),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
res, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(res.Contents) != 0 {
|
||||
return fmt.Errorf("expected empty object list result, instead got %v", res.Contents)
|
||||
}
|
||||
if len(res.CommonPrefixes) != 0 {
|
||||
return fmt.Errorf("expected empty object common prefixes result, instead got %v", res.CommonPrefixes)
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
cancel()
|
||||
return err
|
||||
}, withSkipTearDown())
|
||||
}
|
||||
|
||||
// ListObjects with a marker should not surface pending multipart uploads
|
||||
// even when real objects are interleaved with the marker boundary.
|
||||
func ListObjects_mp_masking_with_marker(s *S3Conf) error {
|
||||
testName := "ListObjects_mp_masking_with_marker"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
// Create pending multipart uploads with keys that sort after all real objects
|
||||
for i := range 3 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: &bucket,
|
||||
Key: getPtr(fmt.Sprintf("zzz-mp-%d", i+1)),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
contents, err := putObjects(s3client, []string{"aaa", "bbb", "ccc"}, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Marker: getPtr("aaa"),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Expect only bbb and ccc (after marker "aaa"), no multipart upload objects
|
||||
if !compareObjects(contents[1:], out.Contents) {
|
||||
return fmt.Errorf("expected objects %v, instead got %v",
|
||||
contents[1:], out.Contents)
|
||||
}
|
||||
if out.IsTruncated == nil || *out.IsTruncated {
|
||||
return fmt.Errorf("expected non-truncated result")
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// ListObjects truncation should count only real objects, not pending multipart uploads.
|
||||
func ListObjects_mp_masking_truncation(s *S3Conf) error {
|
||||
testName := "ListObjects_mp_masking_truncation"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
// Create pending multipart uploads with keys that sort after real objects
|
||||
for i := range 2 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: &bucket,
|
||||
Key: getPtr(fmt.Sprintf("zzz-mp-%d", i+1)),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
contents, err := putObjects(s3client, []string{"obj-a", "obj-b", "obj-c", "obj-d"}, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
maxKeys := int32(2)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out1, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{
|
||||
Bucket: &bucket,
|
||||
MaxKeys: &maxKeys,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if out1.IsTruncated == nil || !*out1.IsTruncated {
|
||||
return fmt.Errorf("expected first page to be truncated")
|
||||
}
|
||||
if !compareObjects(contents[:2], out1.Contents) {
|
||||
return fmt.Errorf("expected first page objects %v, instead got %v",
|
||||
contents[:2], out1.Contents)
|
||||
}
|
||||
if out1.NextMarker == nil || *out1.NextMarker == "" {
|
||||
return fmt.Errorf("expected non-empty NextMarker")
|
||||
}
|
||||
if *out1.NextMarker != "obj-b" {
|
||||
return fmt.Errorf("expected NextMarker to be obj-b, instead got %v", *out1.NextMarker)
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
out2, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Marker: out1.NextMarker,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if out2.IsTruncated == nil || *out2.IsTruncated {
|
||||
return fmt.Errorf("expected second page to not be truncated")
|
||||
}
|
||||
if !compareObjects(contents[2:], out2.Contents) {
|
||||
return fmt.Errorf("expected second page objects %v, instead got %v",
|
||||
contents[2:], out2.Contents)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// ListObjects with a delimiter should not include the .sgwtmp/ multipart prefix
|
||||
// in common prefixes, even when pending multipart uploads exist.
|
||||
func ListObjects_mp_masking_delimiter(s *S3Conf) error {
|
||||
testName := "ListObjects_mp_masking_delimiter"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
// Create pending multipart uploads
|
||||
for i := range 2 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: &bucket,
|
||||
Key: getPtr(fmt.Sprintf("zzz-mp-%d", i+1)),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
_, err := putObjects(s3client, []string{"dir1/file1", "dir2/file2"}, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.ListObjects(ctx, &s3.ListObjectsInput{
|
||||
Bucket: &bucket,
|
||||
Delimiter: getPtr("/"),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(out.Contents) != 0 {
|
||||
return fmt.Errorf("expected empty Contents, instead got %v", out.Contents)
|
||||
}
|
||||
if !comparePrefixes([]string{"dir1/", "dir2/"}, out.CommonPrefixes) {
|
||||
return fmt.Errorf("expected common prefixes [dir1/ dir2/], instead got %v",
|
||||
sprintPrefixes(out.CommonPrefixes))
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
@@ -658,3 +658,203 @@ func ListObjectsV2_invalid_parent_prefix(s *S3Conf) error {
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// ListObjects should not list any pending multipart uploads
|
||||
// and no pending mp should block the bucket from deletion
|
||||
func ListObjectsV2_should_not_list_pending_mps(s *S3Conf) error {
|
||||
testName := "ListObjectsV2_should_not_list_pending_mps"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
for i := range 5 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: &bucket,
|
||||
Key: getPtr(fmt.Sprintf("obj-%d", i)),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(res.Contents) != 0 {
|
||||
return fmt.Errorf("expected empty object list result, instead got %v", res.Contents)
|
||||
}
|
||||
if len(res.CommonPrefixes) != 0 {
|
||||
return fmt.Errorf("expected empty object common prefixes result, instead got %v", res.CommonPrefixes)
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{
|
||||
Bucket: &bucket,
|
||||
})
|
||||
cancel()
|
||||
return err
|
||||
}, withSkipTearDown())
|
||||
}
|
||||
|
||||
// ListObjectsV2 with startAfter should not surface pending multipart uploads
|
||||
// even when real objects are interleaved with the startAfter boundary.
|
||||
func ListObjectsV2_mp_masking_start_after(s *S3Conf) error {
|
||||
testName := "ListObjectsV2_mp_masking_start_after"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
// Create pending multipart uploads with keys that sort after all real objects
|
||||
for i := range 2 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: &bucket,
|
||||
Key: getPtr(fmt.Sprintf("zzz-mp-%d", i+1)),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
contents, err := putObjects(s3client, []string{"alpha", "beta", "gamma"}, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: &bucket,
|
||||
StartAfter: getPtr("alpha"),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Expect only beta and gamma (after startAfter "alpha"), no multipart upload objects
|
||||
if !compareObjects(contents[1:], out.Contents) {
|
||||
return fmt.Errorf("expected objects %v, instead got %v",
|
||||
contents[1:], out.Contents)
|
||||
}
|
||||
if out.IsTruncated == nil || *out.IsTruncated {
|
||||
return fmt.Errorf("expected non-truncated result")
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// ListObjectsV2 truncation should count only real objects, not pending multipart uploads,
|
||||
// and the continuation token should allow correct pagination.
|
||||
func ListObjectsV2_mp_masking_truncation(s *S3Conf) error {
|
||||
testName := "ListObjectsV2_mp_masking_truncation"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
// Create pending multipart uploads with keys that sort after real objects
|
||||
for i := range 2 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: &bucket,
|
||||
Key: getPtr(fmt.Sprintf("zzz-mp-%d", i+1)),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
contents, err := putObjects(s3client, []string{"obj-a", "obj-b", "obj-c", "obj-d"}, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
maxKeys := int32(2)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out1, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: &bucket,
|
||||
MaxKeys: &maxKeys,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if out1.IsTruncated == nil || !*out1.IsTruncated {
|
||||
return fmt.Errorf("expected first page to be truncated")
|
||||
}
|
||||
if !compareObjects(contents[:2], out1.Contents) {
|
||||
return fmt.Errorf("expected first page objects %v, instead got %v",
|
||||
contents[:2], out1.Contents)
|
||||
}
|
||||
if out1.NextContinuationToken == nil || *out1.NextContinuationToken == "" {
|
||||
return fmt.Errorf("expected non-empty NextContinuationToken")
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
out2, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: &bucket,
|
||||
ContinuationToken: out1.NextContinuationToken,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if out2.IsTruncated == nil || *out2.IsTruncated {
|
||||
return fmt.Errorf("expected second page to not be truncated")
|
||||
}
|
||||
if !compareObjects(contents[2:], out2.Contents) {
|
||||
return fmt.Errorf("expected second page objects %v, instead got %v",
|
||||
contents[2:], out2.Contents)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// ListObjectsV2 with a delimiter should not include the .sgwtmp/ multipart prefix
|
||||
// in common prefixes, even when pending multipart uploads exist.
|
||||
func ListObjectsV2_mp_masking_delimiter(s *S3Conf) error {
|
||||
testName := "ListObjectsV2_mp_masking_delimiter"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
// Create pending multipart uploads
|
||||
for i := range 2 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: &bucket,
|
||||
Key: getPtr(fmt.Sprintf("zzz-mp-%d", i+1)),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
_, err := putObjects(s3client, []string{"dir1/file1", "dir2/file2"}, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: &bucket,
|
||||
Delimiter: getPtr("/"),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(out.Contents) != 0 {
|
||||
return fmt.Errorf("expected empty Contents, instead got %v", out.Contents)
|
||||
}
|
||||
if !comparePrefixes([]string{"dir1/", "dir2/"}, out.CommonPrefixes) {
|
||||
return fmt.Errorf("expected common prefixes [dir1/ dir2/], instead got %v",
|
||||
sprintPrefixes(out.CommonPrefixes))
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
@@ -274,6 +274,10 @@ func TestListObjects(ts *TestState) {
|
||||
ts.Run(ListObjects_nested_dir_file_objs)
|
||||
ts.Run(ListObjects_check_owner)
|
||||
ts.Run(ListObjects_non_truncated_common_prefixes)
|
||||
ts.Run(ListObjects_should_not_list_pending_mps)
|
||||
ts.Run(ListObjects_mp_masking_with_marker)
|
||||
ts.Run(ListObjects_mp_masking_truncation)
|
||||
ts.Run(ListObjects_mp_masking_delimiter)
|
||||
//TODO: remove the condition after implementing checksums in azure
|
||||
if !ts.conf.azureTests {
|
||||
ts.Run(ListObjects_with_checksum)
|
||||
@@ -282,10 +286,7 @@ func TestListObjects(ts *TestState) {
|
||||
|
||||
func TestListObjectsV2(ts *TestState) {
|
||||
ts.Run(ListObjectsV2_start_after)
|
||||
// posix continuation token not compatible with azure
|
||||
if !ts.conf.azureTests {
|
||||
ts.Run(ListObjectsV2_both_start_after_and_continuation_token)
|
||||
}
|
||||
ts.Run(ListObjectsV2_both_start_after_and_continuation_token)
|
||||
ts.Run(ListObjectsV2_start_after_not_in_list)
|
||||
ts.Run(ListObjectsV2_start_after_empty_result)
|
||||
ts.Run(ListObjectsV2_both_delimiter_and_prefix)
|
||||
@@ -301,6 +302,10 @@ func TestListObjectsV2(ts *TestState) {
|
||||
ts.Run(ListObjectsV2_with_checksum)
|
||||
}
|
||||
ts.Run(ListObjectsV2_invalid_parent_prefix)
|
||||
ts.Run(ListObjectsV2_should_not_list_pending_mps)
|
||||
ts.Run(ListObjectsV2_mp_masking_start_after)
|
||||
ts.Run(ListObjectsV2_mp_masking_truncation)
|
||||
ts.Run(ListObjectsV2_mp_masking_delimiter)
|
||||
}
|
||||
|
||||
// VD stands for Versioning Disabled
|
||||
@@ -1433,7 +1438,16 @@ func GetIntTests() IntTests {
|
||||
"ListObjects_nested_dir_file_objs": ListObjects_nested_dir_file_objs,
|
||||
"ListObjects_check_owner": ListObjects_check_owner,
|
||||
"ListObjects_non_truncated_common_prefixes": ListObjects_non_truncated_common_prefixes,
|
||||
"ListObjects_should_not_list_pending_mps": ListObjects_should_not_list_pending_mps,
|
||||
"ListObjects_mp_masking_with_marker": ListObjects_mp_masking_with_marker,
|
||||
"ListObjects_mp_masking_truncation": ListObjects_mp_masking_truncation,
|
||||
"ListObjects_mp_masking_delimiter": ListObjects_mp_masking_delimiter,
|
||||
"ListObjectsV2_non_truncated_common_prefixes": ListObjectsV2_non_truncated_common_prefixes,
|
||||
"ListObjectsV2_invalid_parent_prefix": ListObjectsV2_invalid_parent_prefix,
|
||||
"ListObjectsV2_should_not_list_pending_mps": ListObjectsV2_should_not_list_pending_mps,
|
||||
"ListObjectsV2_mp_masking_start_after": ListObjectsV2_mp_masking_start_after,
|
||||
"ListObjectsV2_mp_masking_truncation": ListObjectsV2_mp_masking_truncation,
|
||||
"ListObjectsV2_mp_masking_delimiter": ListObjectsV2_mp_masking_delimiter,
|
||||
"ListObjects_with_checksum": ListObjects_with_checksum,
|
||||
"ListObjectsV2_start_after": ListObjectsV2_start_after,
|
||||
"ListObjectsV2_both_start_after_and_continuation_token": ListObjectsV2_both_start_after_and_continuation_token,
|
||||
|
||||
Reference in New Issue
Block a user