mirror of
https://github.com/versity/versitygw.git
synced 2026-08-29 20:26:56 +00:00
Nils LegerandGitHub
c638503b90
* feat(azure): carry Azure blob marker in continuation token for pagination Azure blob markers are opaque values only Azure may mint, so an S3 object key can never be passed back to Azure as a listing marker. Paging the underlying Azure listing with an S3 key therefore failed or re-scanned the whole prefix on every page. Introduce azMarkerToken, an opaque S3 continuation token that carries the Azure marker alongside the last returned key: the Azure marker resumes the blob listing where it stopped, and the last key filters out already-returned entries. Tokens are versioned with a "vgw1." prefix; anything without it is treated as a plain key, so tokens from older versions and hand-crafted markers keep working. The shared listBlobs helper now backs both ListObjects and ListObjectsV2, applying the S3 marker and delimiter client side. ListObjectsV2 pages efficiently via the token; ListObjects (v1) has no token to carry state and walks the prefix from the start each page. Add unit tests (token round-trip, marker-resumed pagination, delimiter and common-prefix handling, multipart filtering) driven by a fake Azure container, and integration tests covering full and delimited pagination. Signed-off-by: Nils Leger <nils.leger@getflip.com> * fix: token is now bound to delimiter too Signed-off-by: Nils Leger <nils.leger@getflip.com> --------- Signed-off-by: Nils Leger <nils.leger@getflip.com>
1025 lines
30 KiB
Go
1025 lines
30 KiB
Go
// Copyright 2023 Versity Software
|
|
// This file is licensed under the Apache License, Version 2.0
|
|
// (the "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
package integration
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
)
|
|
|
|
func ListObjectsV2_start_after(s *S3Conf) error {
|
|
testName := "ListObjectsV2_start_after"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
contents, err := putObjects(s3client, []string{"foo", "bar", "baz"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
startAfter := "bar"
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
StartAfter: &startAfter,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if getString(out.StartAfter) != startAfter {
|
|
return fmt.Errorf("expected StartAfter to be %v, insted got %v",
|
|
startAfter, getString(out.StartAfter))
|
|
}
|
|
if !compareObjects(contents[1:], out.Contents) {
|
|
return fmt.Errorf("expected the output to be %v, instead got %v",
|
|
contents, out.Contents)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListObjectsV2_both_start_after_and_continuation_token(s *S3Conf) error {
|
|
testName := "ListObjectsV2_both_start_after_and_continuation_token"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
contents, err := putObjects(s3client, []string{"foo", "bar", "baz", "quxx"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var maxKeys int32 = 1
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
MaxKeys: &maxKeys,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if out.IsTruncated == nil || !*out.IsTruncated {
|
|
return fmt.Errorf("expected output to be truncated")
|
|
}
|
|
|
|
if out.MaxKeys == nil {
|
|
return fmt.Errorf("expected non nil max-keys")
|
|
}
|
|
|
|
if *out.MaxKeys != maxKeys {
|
|
return fmt.Errorf("expected max-keys to be %v, instead got %v",
|
|
maxKeys, out.MaxKeys)
|
|
}
|
|
|
|
// the azure backend returns an opaque token, as it has to carry the
|
|
// azure blob listing marker along with the last key
|
|
if s.azureTests {
|
|
if getString(out.NextContinuationToken) == "" {
|
|
return fmt.Errorf("expected non-empty NextContinuationToken")
|
|
}
|
|
} else if getString(out.NextContinuationToken) != "bar" {
|
|
return fmt.Errorf("expected next-marker to be baz, instead got %v",
|
|
getString(out.NextContinuationToken))
|
|
}
|
|
|
|
if !compareObjects(contents[:1], out.Contents) {
|
|
return fmt.Errorf("expected the output to be %v, instead got %v",
|
|
contents[:1], out.Contents)
|
|
}
|
|
|
|
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
|
resp, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
ContinuationToken: out.NextContinuationToken,
|
|
StartAfter: getPtr("baz"),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !compareObjects(contents[2:], resp.Contents) {
|
|
return fmt.Errorf("expected the output to be %v, instead got %v",
|
|
contents[2:], resp.Contents)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListObjectsV2_start_after_not_in_list(s *S3Conf) error {
|
|
testName := "ListObjectsV2_start_after_not_in_list"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
contents, err := putObjects(s3client, []string{"foo", "bar", "baz", "quxx"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
StartAfter: getPtr("blah"),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !compareObjects(contents[2:], out.Contents) {
|
|
return fmt.Errorf("expected the output to be %v, instead got %v",
|
|
contents[2:], out.Contents)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListObjectsV2_start_after_empty_result(s *S3Conf) error {
|
|
testName := "ListObjectsV2_start_after_empty_result"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
_, err := putObjects(s3client, []string{"foo", "bar", "baz", "quxx"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
StartAfter: getPtr("zzz"),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(out.Contents) != 0 {
|
|
return fmt.Errorf("expected empty output instead got %v", out.Contents)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListObjectsV2_both_delimiter_and_prefix(s *S3Conf) error {
|
|
testName := "ListObjectsV2_both_delimiter_and_prefix"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
_, err := putObjects(s3client, []string{
|
|
"sample.jpg",
|
|
"photos/2006/January/sample.jpg",
|
|
"photos/2006/February/sample2.jpg",
|
|
"photos/2006/February/sample3.jpg",
|
|
"photos/2006/February/sample4.jpg",
|
|
}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
delim, prefix := "/", "photos/2006/"
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
Delimiter: &delim,
|
|
Prefix: &prefix,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if res.Delimiter == nil || *res.Delimiter != delim {
|
|
return fmt.Errorf("expected the delimiter to be %v", delim)
|
|
}
|
|
if res.Prefix == nil || *res.Prefix != prefix {
|
|
return fmt.Errorf("expected the prefix to be %v", prefix)
|
|
}
|
|
if !comparePrefixes([]string{"photos/2006/February/", "photos/2006/January/"},
|
|
res.CommonPrefixes) {
|
|
return fmt.Errorf("expected the common prefixes to be %v, instead got %v",
|
|
[]string{"photos/2006/February/", "photos/2006/January/"}, res.CommonPrefixes)
|
|
}
|
|
if len(res.Contents) != 0 {
|
|
return fmt.Errorf("expected empty objects list, instead got %v", res.Contents)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListObjectsV2_single_dir_object_with_delim_and_prefix(s *S3Conf) error {
|
|
testName := "ListObjectsV2_single_dir_object_with_delim_and_prefix"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
contents, err := putObjects(s3client, []string{"a/"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
delim, prefix := "/", "a"
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
Delimiter: &delim,
|
|
Prefix: &prefix,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !comparePrefixes([]string{"a/"}, res.CommonPrefixes) {
|
|
return fmt.Errorf("expected the common prefixes to be %v, instead got %v",
|
|
[]string{"a/"}, res.CommonPrefixes)
|
|
}
|
|
if len(res.Contents) != 0 {
|
|
return fmt.Errorf("expected empty objects list, instead got %v",
|
|
res.Contents)
|
|
}
|
|
|
|
prefix = "a/"
|
|
|
|
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
|
res, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
Delimiter: &delim,
|
|
Prefix: &prefix,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !compareObjects(contents, res.Contents) {
|
|
return fmt.Errorf("expected the object list to be %v, instead got %v",
|
|
[]string{"a/"}, res.Contents)
|
|
}
|
|
if len(res.CommonPrefixes) != 0 {
|
|
return fmt.Errorf("expected empty common prefixes, instead got %v",
|
|
res.CommonPrefixes)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListObjectsV2_truncated_common_prefixes(s *S3Conf) error {
|
|
testName := "ListObjectsV2_truncated_common_prefixes"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
_, err := putObjects(s3client, []string{"d1/f1", "d2/f2", "d3/f3", "d4/f4"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
delim, maxKeys := "/", int32(3)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
Delimiter: &delim,
|
|
MaxKeys: &maxKeys,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !comparePrefixes([]string{"d1/", "d2/", "d3/"}, out.CommonPrefixes) {
|
|
return fmt.Errorf("expected the common prefixes to be %v, instead got %v",
|
|
[]string{"d1/", "d2/", "d3/"}, sprintPrefixes(out.CommonPrefixes))
|
|
}
|
|
|
|
if out.MaxKeys == nil {
|
|
return fmt.Errorf("expected non nil max-keys")
|
|
}
|
|
if *out.MaxKeys != maxKeys {
|
|
return fmt.Errorf("expected the max-keys to be %v, instead got %v",
|
|
maxKeys, *out.MaxKeys)
|
|
}
|
|
if getString(out.Delimiter) != delim {
|
|
return fmt.Errorf("expected the delimiter to be %v, instead got %v",
|
|
delim, getString(out.Delimiter))
|
|
}
|
|
|
|
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
Delimiter: &delim,
|
|
ContinuationToken: out.NextContinuationToken,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !comparePrefixes([]string{"d4/"}, out.CommonPrefixes) {
|
|
return fmt.Errorf("expected the common prefixes to be %v, instead got %v",
|
|
[]string{"d4/"}, sprintPrefixes(out.CommonPrefixes))
|
|
}
|
|
if getString(out.Delimiter) != delim {
|
|
return fmt.Errorf("expected the delimiter to be %v, instead got %v",
|
|
delim, getString(out.Delimiter))
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListObjectsV2_non_truncated_common_prefixes(s *S3Conf) error {
|
|
testName := "ListObjectsV2_non_truncated_common_prefixes"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
_, err := putObjects(s3client, []string{"asdf", "boo/bar", "boo/baz/xyzzy", "cquux/thud", "cquux/bla"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
delim, marker, maxKeys := "/", "boo/", int32(1)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
StartAfter: &marker,
|
|
Delimiter: &delim,
|
|
MaxKeys: &maxKeys,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if res.IsTruncated == nil {
|
|
return fmt.Errorf("expected non-nil istruncated")
|
|
}
|
|
if *res.IsTruncated {
|
|
return fmt.Errorf("expected non-truncated result")
|
|
}
|
|
if res.MaxKeys == nil {
|
|
return fmt.Errorf("expected non nil max-keys")
|
|
}
|
|
if *res.MaxKeys != maxKeys {
|
|
return fmt.Errorf("expected max-keys to be %v, instead got %v",
|
|
maxKeys, *res.MaxKeys)
|
|
}
|
|
if getString(res.Delimiter) != delim {
|
|
return fmt.Errorf("expected delimiter to be %v, instead got %v",
|
|
delim, getString(res.Delimiter))
|
|
}
|
|
if len(res.Contents) != 0 {
|
|
return fmt.Errorf("expected empty contents, instead got %+v",
|
|
res.Contents)
|
|
}
|
|
cPrefs := []string{"cquux/"}
|
|
if !comparePrefixes(cPrefs, res.CommonPrefixes) {
|
|
return fmt.Errorf("expected common prefixes to be %v, instead got %+v",
|
|
cPrefs, sprintPrefixes(res.CommonPrefixes))
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListObjectsV2_all_objs_max_keys(s *S3Conf) error {
|
|
testName := "ListObjectsV2_all_objs_max_keys"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
contents, err := putObjects(s3client, []string{"bar", "baz", "foo"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
maxKeys := int32(3)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
MaxKeys: &maxKeys,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if out.IsTruncated == nil || *out.IsTruncated {
|
|
return fmt.Errorf("expected the output not to be truncated")
|
|
}
|
|
if getString(out.NextContinuationToken) != "" {
|
|
return fmt.Errorf("expected empty NextContinuationToken, instead got %v",
|
|
getString(out.NextContinuationToken))
|
|
}
|
|
if out.MaxKeys == nil {
|
|
return fmt.Errorf("expected non nil max-keys")
|
|
}
|
|
if *out.MaxKeys != maxKeys {
|
|
return fmt.Errorf("expected the max-keys to be %v, instead got %v",
|
|
maxKeys, *out.MaxKeys)
|
|
}
|
|
|
|
if !compareObjects(contents, out.Contents) {
|
|
return fmt.Errorf("expected the objects list to be %v, instead got %v",
|
|
contents, out.Contents)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListObjectsV2_exceeding_max_keys(s *S3Conf) error {
|
|
testName := "ListObjectsV2_exceeding_max_keys"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
maxKeys := int32(233453333)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
MaxKeys: &maxKeys,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
if out.MaxKeys == nil {
|
|
return fmt.Errorf("unexpected nil max-keys")
|
|
}
|
|
if *out.MaxKeys != 1000 {
|
|
return fmt.Errorf("expected the max-keys to be %v, instaed got %v",
|
|
1000, *out.MaxKeys)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListObjectsV2_list_all_objs(s *S3Conf) error {
|
|
testName := "ListObjectsV2_list_all_objs"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
contents, err := putObjects(s3client, []string{"a", "aa", "aaa", "aaaa", "bar", "baz", "foo", "obj1", "hello/world", "xyzz/quxx"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Test 1: List all objects without pagination
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if out.StartAfter != nil {
|
|
return fmt.Errorf("expected the StartAfter to be nil, instead got %v",
|
|
*out.StartAfter)
|
|
}
|
|
if out.ContinuationToken != nil {
|
|
return fmt.Errorf("expected the ContinuationToken to be nil, instead got %v",
|
|
*out.ContinuationToken)
|
|
}
|
|
if out.NextContinuationToken != nil {
|
|
return fmt.Errorf("expected the NextContinuationToken to be nil, instead got %v",
|
|
*out.NextContinuationToken)
|
|
}
|
|
if out.Delimiter != nil {
|
|
return fmt.Errorf("expected the Delimiter to be nil, instead got %v",
|
|
*out.Delimiter)
|
|
}
|
|
if out.Prefix != nil {
|
|
return fmt.Errorf("expected the Prefix to be nil, instead got %v",
|
|
*out.Prefix)
|
|
}
|
|
|
|
if !compareObjects(contents, out.Contents) {
|
|
return fmt.Errorf("expected the contents to be %v, instead got %v",
|
|
contents, out.Contents)
|
|
}
|
|
|
|
// Test 2: List all objects with pagination using ListObjectsV2
|
|
var continuationToken *string
|
|
var allObjects []types.Object
|
|
maxKeys := int32(2)
|
|
|
|
for {
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
MaxKeys: &maxKeys,
|
|
ContinuationToken: continuationToken,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
allObjects = append(allObjects, out.Contents...)
|
|
|
|
if out.NextContinuationToken == nil || !*out.IsTruncated {
|
|
break
|
|
}
|
|
continuationToken = out.NextContinuationToken
|
|
}
|
|
|
|
if !compareObjects(contents, allObjects) {
|
|
return fmt.Errorf("expected the paginated contents to be %v, instead got %v",
|
|
contents, allObjects)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListObjectsV2_with_owner(s *S3Conf) error {
|
|
testName := "ListObjectsV2_with_owner"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
objs, err := putObjects(s3client, []string{"foo", "bar/baz", "quxx/xyz/eee", "abc/", "bcc"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
FetchOwner: getBoolPtr(true),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for i := range res.Contents {
|
|
res.Contents[i].Owner = &types.Owner{
|
|
ID: &s.awsID,
|
|
}
|
|
}
|
|
|
|
if !compareObjects(objs, res.Contents) {
|
|
return fmt.Errorf("expected the contents to be %v, instead got %v",
|
|
objs, res.Contents)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListObjectsV2_with_checksum(s *S3Conf) error {
|
|
testName := "ListObjectsV2_with_checksum"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
contents := []types.Object{}
|
|
|
|
for i, el := range types.ChecksumAlgorithmCrc32.Values() {
|
|
key := fmt.Sprintf("obj-%v", i)
|
|
size := int64(i * 100)
|
|
out, err := putObjectWithData(size, &s3.PutObjectInput{
|
|
Bucket: &bucket,
|
|
Key: &key,
|
|
ChecksumAlgorithm: el,
|
|
}, s3client, withPutObjectChecksumAlgo(el))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
contents = append(contents, types.Object{
|
|
Key: &key,
|
|
ETag: out.res.ETag,
|
|
Size: &size,
|
|
StorageClass: types.ObjectStorageClassStandard,
|
|
ChecksumAlgorithm: []types.ChecksumAlgorithm{
|
|
el,
|
|
},
|
|
ChecksumType: out.res.ChecksumType,
|
|
})
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !compareObjects(res.Contents, contents) {
|
|
return fmt.Errorf("expected the objects list to be %v, instead got %v",
|
|
contents, res.Contents)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func ListObjectsV2_invalid_parent_prefix(s *S3Conf) error {
|
|
testName := "ListObjectsV2_invalid_parent_prefix"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
_, err := putObjects(s3client, []string{"file"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
delim, maxKeys := "/", int32(100)
|
|
prefix := "file/file/file"
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
Delimiter: &delim,
|
|
MaxKeys: &maxKeys,
|
|
Prefix: &prefix,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(out.CommonPrefixes) > 0 {
|
|
return fmt.Errorf("expected the common prefixes to be %v, instead got %v",
|
|
[]string{""}, out.CommonPrefixes)
|
|
}
|
|
if out.MaxKeys == nil {
|
|
return fmt.Errorf("expected non nil max-keys")
|
|
}
|
|
if *out.MaxKeys != maxKeys {
|
|
return fmt.Errorf("expected the max-keys to be %v, instead got %v",
|
|
maxKeys, *out.MaxKeys)
|
|
}
|
|
if getString(out.Delimiter) != delim {
|
|
return fmt.Errorf("expected the delimiter to be %v, instead got %v",
|
|
delim, getString(out.Delimiter))
|
|
}
|
|
if len(out.Contents) > 0 {
|
|
return fmt.Errorf("expected the objects to be %v, instead got %v",
|
|
[]types.Object{}, out.Contents)
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// ListObjects should not list any pending multipart uploads
|
|
// and no pending mp should block the bucket from deletion
|
|
func ListObjectsV2_should_not_list_pending_mps(s *S3Conf) error {
|
|
testName := "ListObjectsV2_should_not_list_pending_mps"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
for i := range 5 {
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
|
Bucket: &bucket,
|
|
Key: getPtr(fmt.Sprintf("obj-%d", i)),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(res.Contents) != 0 {
|
|
return fmt.Errorf("expected empty object list result, instead got %v", res.Contents)
|
|
}
|
|
if len(res.CommonPrefixes) != 0 {
|
|
return fmt.Errorf("expected empty object common prefixes result, instead got %v", res.CommonPrefixes)
|
|
}
|
|
|
|
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
|
_, err = s3client.DeleteBucket(ctx, &s3.DeleteBucketInput{
|
|
Bucket: &bucket,
|
|
})
|
|
cancel()
|
|
return err
|
|
}, withSkipTearDown())
|
|
}
|
|
|
|
// ListObjectsV2 with startAfter should not surface pending multipart uploads
|
|
// even when real objects are interleaved with the startAfter boundary.
|
|
func ListObjectsV2_mp_masking_start_after(s *S3Conf) error {
|
|
testName := "ListObjectsV2_mp_masking_start_after"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
// Create pending multipart uploads with keys that sort after all real objects
|
|
for i := range 2 {
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
|
Bucket: &bucket,
|
|
Key: getPtr(fmt.Sprintf("zzz-mp-%d", i+1)),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
contents, err := putObjects(s3client, []string{"alpha", "beta", "gamma"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
StartAfter: getPtr("alpha"),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Expect only beta and gamma (after startAfter "alpha"), no multipart upload objects
|
|
if !compareObjects(contents[1:], out.Contents) {
|
|
return fmt.Errorf("expected objects %v, instead got %v",
|
|
contents[1:], out.Contents)
|
|
}
|
|
if out.IsTruncated == nil || *out.IsTruncated {
|
|
return fmt.Errorf("expected non-truncated result")
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// ListObjectsV2 truncation should count only real objects, not pending multipart uploads,
|
|
// and the continuation token should allow correct pagination.
|
|
func ListObjectsV2_mp_masking_truncation(s *S3Conf) error {
|
|
testName := "ListObjectsV2_mp_masking_truncation"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
// Create pending multipart uploads with keys that sort after real objects
|
|
for i := range 2 {
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
|
Bucket: &bucket,
|
|
Key: getPtr(fmt.Sprintf("zzz-mp-%d", i+1)),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
contents, err := putObjects(s3client, []string{"obj-a", "obj-b", "obj-c", "obj-d"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
maxKeys := int32(2)
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out1, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
MaxKeys: &maxKeys,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if out1.IsTruncated == nil || !*out1.IsTruncated {
|
|
return fmt.Errorf("expected first page to be truncated")
|
|
}
|
|
if !compareObjects(contents[:2], out1.Contents) {
|
|
return fmt.Errorf("expected first page objects %v, instead got %v",
|
|
contents[:2], out1.Contents)
|
|
}
|
|
if out1.NextContinuationToken == nil || *out1.NextContinuationToken == "" {
|
|
return fmt.Errorf("expected non-empty NextContinuationToken")
|
|
}
|
|
|
|
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
|
out2, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
ContinuationToken: out1.NextContinuationToken,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if out2.IsTruncated == nil || *out2.IsTruncated {
|
|
return fmt.Errorf("expected second page to not be truncated")
|
|
}
|
|
if !compareObjects(contents[2:], out2.Contents) {
|
|
return fmt.Errorf("expected second page objects %v, instead got %v",
|
|
contents[2:], out2.Contents)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// ListObjectsV2 with a delimiter should not include the .sgwtmp/ multipart prefix
|
|
// in common prefixes, even when pending multipart uploads exist.
|
|
func ListObjectsV2_mp_masking_delimiter(s *S3Conf) error {
|
|
testName := "ListObjectsV2_mp_masking_delimiter"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
// Create pending multipart uploads
|
|
for i := range 2 {
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
_, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
|
Bucket: &bucket,
|
|
Key: getPtr(fmt.Sprintf("zzz-mp-%d", i+1)),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
_, err := putObjects(s3client, []string{"dir1/file1", "dir2/file2"}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
Delimiter: getPtr("/"),
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(out.Contents) != 0 {
|
|
return fmt.Errorf("expected empty Contents, instead got %v", out.Contents)
|
|
}
|
|
if !comparePrefixes([]string{"dir1/", "dir2/"}, out.CommonPrefixes) {
|
|
return fmt.Errorf("expected common prefixes [dir1/ dir2/], instead got %v",
|
|
sprintPrefixes(out.CommonPrefixes))
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// ListObjectsV2_full_pagination walks a listing that spans many pages and
|
|
// checks that every object is returned exactly once, in order, with the
|
|
// continuation token driving the walk. This exercises backends whose
|
|
// continuation token is opaque (e.g. azure) rather than the last key.
|
|
func ListObjectsV2_full_pagination(s *S3Conf) error {
|
|
testName := "ListObjectsV2_full_pagination"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
const objectCount = 25
|
|
keys := make([]string, 0, objectCount)
|
|
for i := 0; i < objectCount; i++ {
|
|
keys = append(keys, fmt.Sprintf("obj-%03d", i))
|
|
}
|
|
contents, err := putObjects(s3client, keys, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
maxKeys := int32(4)
|
|
var continuationToken *string
|
|
var allObjects []types.Object
|
|
seen := map[string]bool{}
|
|
|
|
for pages := 0; ; pages++ {
|
|
if pages > objectCount+1 {
|
|
return fmt.Errorf("pagination did not terminate after %v pages", pages)
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
MaxKeys: &maxKeys,
|
|
ContinuationToken: continuationToken,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if int32(len(out.Contents)) > maxKeys {
|
|
return fmt.Errorf("page returned %v objects, exceeding max-keys %v",
|
|
len(out.Contents), maxKeys)
|
|
}
|
|
|
|
for _, obj := range out.Contents {
|
|
key := getString(obj.Key)
|
|
if seen[key] {
|
|
return fmt.Errorf("object %q returned on more than one page", key)
|
|
}
|
|
seen[key] = true
|
|
}
|
|
allObjects = append(allObjects, out.Contents...)
|
|
|
|
if out.IsTruncated != nil && *out.IsTruncated {
|
|
if getString(out.NextContinuationToken) == "" {
|
|
return fmt.Errorf("truncated page returned an empty NextContinuationToken")
|
|
}
|
|
continuationToken = out.NextContinuationToken
|
|
continue
|
|
}
|
|
|
|
// the final page must not advertise a continuation token
|
|
if getString(out.NextContinuationToken) != "" {
|
|
return fmt.Errorf("non-truncated page returned a NextContinuationToken %q",
|
|
getString(out.NextContinuationToken))
|
|
}
|
|
break
|
|
}
|
|
|
|
if !compareObjects(contents, allObjects) {
|
|
return fmt.Errorf("expected the paginated contents to be %v, instead got %v",
|
|
contents, allObjects)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// ListObjectsV2_pagination_with_delimiter walks a delimited listing across
|
|
// several pages and checks that common prefixes and objects are each returned
|
|
// once and in order, with no prefix repeated across page boundaries.
|
|
func ListObjectsV2_pagination_with_delimiter(s *S3Conf) error {
|
|
testName := "ListObjectsV2_pagination_with_delimiter"
|
|
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
|
_, err := putObjects(s3client, []string{
|
|
"a/1", "a/2", "b/1", "c/1", "d/1", "e/1", "root1", "root2",
|
|
}, bucket)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
delim, maxKeys := "/", int32(2)
|
|
var continuationToken *string
|
|
var gotObjects []string
|
|
var gotPrefixes []types.CommonPrefix
|
|
seenPrefix := map[string]bool{}
|
|
|
|
for pages := 0; ; pages++ {
|
|
if pages > 10 {
|
|
return fmt.Errorf("pagination did not terminate after %v pages", pages)
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
|
out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
|
Bucket: &bucket,
|
|
Delimiter: &delim,
|
|
MaxKeys: &maxKeys,
|
|
ContinuationToken: continuationToken,
|
|
})
|
|
cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if kc := int32(len(out.Contents) + len(out.CommonPrefixes)); kc > maxKeys {
|
|
return fmt.Errorf("page returned %v keys, exceeding max-keys %v", kc, maxKeys)
|
|
}
|
|
|
|
for _, obj := range out.Contents {
|
|
gotObjects = append(gotObjects, getString(obj.Key))
|
|
}
|
|
for _, cp := range out.CommonPrefixes {
|
|
prefix := getString(cp.Prefix)
|
|
if seenPrefix[prefix] {
|
|
return fmt.Errorf("common prefix %q returned on more than one page", prefix)
|
|
}
|
|
seenPrefix[prefix] = true
|
|
gotPrefixes = append(gotPrefixes, cp)
|
|
}
|
|
|
|
if out.IsTruncated == nil || !*out.IsTruncated {
|
|
break
|
|
}
|
|
if getString(out.NextContinuationToken) == "" {
|
|
return fmt.Errorf("truncated page returned an empty NextContinuationToken")
|
|
}
|
|
continuationToken = out.NextContinuationToken
|
|
}
|
|
|
|
wantPrefixes := []string{"a/", "b/", "c/", "d/", "e/"}
|
|
if !comparePrefixes(wantPrefixes, gotPrefixes) {
|
|
return fmt.Errorf("expected common prefixes %v, instead got %v",
|
|
wantPrefixes, sprintPrefixes(gotPrefixes))
|
|
}
|
|
|
|
wantObjects := []string{"root1", "root2"}
|
|
if len(gotObjects) != len(wantObjects) {
|
|
return fmt.Errorf("expected objects %v, instead got %v", wantObjects, gotObjects)
|
|
}
|
|
for i, key := range wantObjects {
|
|
if gotObjects[i] != key {
|
|
return fmt.Errorf("expected objects %v, instead got %v", wantObjects, gotObjects)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|