versitygw/tests/integration/ListMultipartUploads.go

// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package integration

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/aws/aws-sdk-go-v2/service/s3/types"
	"github.com/google/uuid"
	"github.com/versity/versitygw/s3err"
)

func ListMultipartUploads_non_existing_bucket(s *S3Conf) error {
	testName := "ListMultipartUploads_non_existing_bucket"
	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
		bucketName := getBucketName()
		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		_, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket: &bucketName,
		})
		cancel()
		if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrNoSuchBucket)); err != nil {
			return err
		}

		return nil
	})
}

func ListMultipartUploads_empty_result(s *S3Conf) error {
	testName := "ListMultipartUploads_empty_result"
	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket: &bucket,
		})
		cancel()
		if err != nil {
			return err
		}
		if len(out.Uploads) != 0 {
			return fmt.Errorf("expected empty uploads, instead got %+v",
				out.Uploads)
		}

		return nil
	})
}

func ListMultipartUploads_invalid_max_uploads(s *S3Conf) error {
	testName := "ListMultipartUploads_invalid_max_uploads"
	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
		maxUploads := int32(-3)
		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		_, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:     &bucket,
			MaxUploads: &maxUploads,
		})
		cancel()
		if err := checkApiErr(err, s3err.GetInvalidArgNegativeMaxLimiter("max-uploads", fmt.Sprint(maxUploads))); err != nil {
			return err
		}

		return nil
	})
}

func ListMultipartUploads_max_uploads(s *S3Conf) error {
	testName := "ListMultipartUploads_max_uploads"
	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
		uploads := []types.MultipartUpload{}
		for i := 1; i < 6; i++ {
			out, err := createMp(s3client, bucket, fmt.Sprintf("obj%v", i))
			if err != nil {
				return err
			}
			uploads = append(uploads, types.MultipartUpload{
				UploadId:     out.UploadId,
				Key:          out.Key,
				StorageClass: types.StorageClassStandard,
			})
		}
		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		maxUploads := int32(2)
		out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:     &bucket,
			MaxUploads: &maxUploads,
		})
		cancel()
		if err != nil {
			return err
		}

		if out.IsTruncated == nil {
			return fmt.Errorf("unexpected nil is-truncated")
		}
		if out.MaxUploads == nil {
			return fmt.Errorf("unexpected nil max-uploads")
		}
		if !*out.IsTruncated {
			return fmt.Errorf("expected the output to be truncated")
		}
		if *out.MaxUploads != 2 {
			return fmt.Errorf("expected max-uploads to be 2, instead got %v",
				out.MaxUploads)
		}
		if ok := compareMultipartUploads(out.Uploads, uploads[:2]); !ok {
			return fmt.Errorf("expected multipart uploads to be %v, instead got %v",
				uploads[:2], out.Uploads)
		}
		if getString(out.NextKeyMarker) != getString(uploads[1].Key) {
			return fmt.Errorf("expected next-key-marker to be %v, instead got %v",
				getString(uploads[1].Key), getString(out.NextKeyMarker))
		}
		if getString(out.NextUploadIdMarker) != getString(uploads[1].UploadId) {
			return fmt.Errorf("expected next-upload-id-marker to be %v, instead got %v",
				getString(uploads[1].UploadId), getString(out.NextUploadIdMarker))
		}

		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
		out, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:    &bucket,
			KeyMarker: out.NextKeyMarker,
		})
		cancel()
		if err != nil {
			return err
		}
		if ok := compareMultipartUploads(out.Uploads, uploads[2:]); !ok {
			return fmt.Errorf("expected multipart uploads to be %v, instead got %v",
				uploads[2:], out.Uploads)
		}

		return nil
	})
}

func ListMultipartUploads_exceeding_max_uploads(s *S3Conf) error {
	testName := "ListMultipartUploads_exceeding_max_uploads"
	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
		maxUploads := int32(1343235)
		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		res, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:     &bucket,
			MaxUploads: &maxUploads,
		})
		cancel()
		if err != nil {
			return err
		}

		if res.MaxUploads == nil {
			return fmt.Errorf("unexpected nil max-uploads")
		}
		if *res.MaxUploads != 1000 {
			return fmt.Errorf("expected max-uploads to be %v, instaed got %v",
				1000, *res.MaxUploads)
		}

		return nil
	})
}

func ListMultipartUploads_ignore_upload_id_marker(s *S3Conf) error {
	testName := "ListMultipartUploads_ignore_upload_id_marker"
	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
		uploads := []types.MultipartUpload{}
		for i := 1; i < 6; i++ {
			out, err := createMp(s3client, bucket, fmt.Sprintf("obj%v", i))
			if err != nil {
				return err
			}
			uploads = append(uploads, types.MultipartUpload{
				UploadId:     out.UploadId,
				Key:          out.Key,
				StorageClass: types.StorageClassStandard,
			})
		}
		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:         &bucket,
			UploadIdMarker: uploads[2].UploadId,
		})
		cancel()
		if err != nil {
			return err
		}
		if !compareMultipartUploads(out.Uploads, uploads) {
			return fmt.Errorf("expected multipart uploads to be %v, instead got %v",
				uploads, out.Uploads)
		}

		// should ignore invalid uploaId marker
		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
		out, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:         &bucket,
			UploadIdMarker: getPtr("invalid_uploadId_marker"),
		})
		cancel()
		if err != nil {
			return err
		}
		if !compareMultipartUploads(out.Uploads, uploads) {
			return fmt.Errorf("expected multipart uploads to be %v, instead got %v",
				uploads, out.Uploads)
		}

		return nil
	})
}

func ListMultipartUploads_invalid_uploadId_marker(s *S3Conf) error {
	testName := "ListMultipartUploads_invalid_uploadId_marker"
	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
		uploads := make([]types.MultipartUpload, 0, 5)
		for i := range 5 {
			out, err := createMp(s3client, bucket, fmt.Sprintf("obj-%v", i))
			if err != nil {
				return err
			}

			uploads = append(uploads, types.MultipartUpload{
				UploadId:     out.UploadId,
				Key:          out.Key,
				StorageClass: types.StorageClassStandard,
			})
		}

		// invalid UUID
		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		_, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:         &bucket,
			KeyMarker:      getPtr("obj-2"),
			UploadIdMarker: getPtr("invalid_uploadId_marker"),
		})
		cancel()
		if err := checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdMarker, "invalid_uploadId_marker")); err != nil {
			return err
		}

		// valid UUID, but not from the list
		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
		uploadIDMarker := uuid.New().String()
		_, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:         &bucket,
			KeyMarker:      getPtr("obj-2"),
			UploadIdMarker: &uploadIDMarker,
		})
		cancel()
		if err := checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdMarker, uploadIDMarker)); err != nil {
			return err
		}

		// uploadId marker and key marker mismatch
		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
		_, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:         &bucket,
			KeyMarker:      getPtr("obj-2"),
			UploadIdMarker: uploads[4].UploadId,
		})
		cancel()
		return checkApiErr(err, s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdMarker, *uploads[4].UploadId))
	})
}

func ListMultipartUploads_keyMarker_not_from_list(s *S3Conf) error {
	testName := "ListMultipartUploads_keyMarker_not_from_list"
	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
		uploads := make([]types.MultipartUpload, 0, 9)
		for _, mp := range []struct {
			key   string
			count int
		}{
			{"bar", 3},
			{"baz", 4},
			{"foo", 2},
		} {
			for range mp.count {
				out, err := createMp(s3client, bucket, mp.key)
				if err != nil {
					return err
				}
				uploads = append(uploads, types.MultipartUpload{
					Key:          out.Key,
					UploadId:     out.UploadId,
					StorageClass: types.StorageClassStandard,
				})
				if s.azureTests {
					// add an artificial delay for azure tests
					// as azure uploads all these mps with the same
					// identical creation time
					time.Sleep(time.Second)
				}
			}
		}

		// without uploadId marker
		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:    &bucket,
			KeyMarker: getPtr("bat"),
		})
		cancel()
		if err != nil {
			return err
		}

		if !compareMultipartUploads(uploads[3:], out.Uploads) {
			return fmt.Errorf("expected the mp list to be %v, instead got %v", uploads[:3], out.Uploads)
		}

		// should start the listing after the specified uploadId marker
		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
		out, err = s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:         &bucket,
			KeyMarker:      getPtr("bat"),
			UploadIdMarker: uploads[4].UploadId,
		})
		cancel()
		if err != nil {
			return err
		}

		if !compareMultipartUploads(uploads[5:], out.Uploads) {
			return fmt.Errorf("expected the mp list to be %v, instead got %v", uploads[5:], out.Uploads)
		}

		return nil
	})
}

func ListMultipartUploads_delimiter_truncated(s *S3Conf) error {
	testName := "ListMultipartUploads_delimiter_truncated"
	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
		uploads := make([]types.MultipartUpload, 0, 6)
		for _, key := range []string{
			"abc/something",
			"foo/bar/baz",
			"foo/quxx",
			"xyz/hello",
			"zzz/bca",
			"some/very/nested/mp/object",
		} {
			out, err := createMp(s3client, bucket, key)
			if err != nil {
				return err
			}
			uploads = append(uploads, types.MultipartUpload{
				Key:          out.Key,
				UploadId:     out.UploadId,
				StorageClass: types.StorageClassStandard,
			})
		}

		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:     &bucket,
			Delimiter:  getPtr("/"),
			MaxUploads: getPtr(int32(2)),
		})
		cancel()
		if err != nil {
			return err
		}

		if len(out.Uploads) != 0 {
			return fmt.Errorf("expected empty uplodas list, instead got %v", out.Uploads)
		}
		expectedCps := []string{"abc/", "foo/"}
		if !comparePrefixes(expectedCps, out.CommonPrefixes) {
			return fmt.Errorf("expected the common prefixes to be %v, instead got %v", expectedCps, out.CommonPrefixes)
		}
		if getString(out.NextKeyMarker) != "foo/" {
			return fmt.Errorf("expected the next key marker to be 'foo/', instead got %s", getString(out.NextKeyMarker))
		}
		if getString(out.NextUploadIdMarker) != getString(uploads[1].UploadId) {
			return fmt.Errorf("expected the next upload id marker to be %s, instead got %s", getString(uploads[1].UploadId), getString(out.NextUploadIdMarker))
		}
		if !*out.IsTruncated {
			return fmt.Errorf("expected a truncated response")
		}

		ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
		out2, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:         &bucket,
			Delimiter:      getPtr("/"),
			UploadIdMarker: out.NextUploadIdMarker,
			KeyMarker:      out.NextKeyMarker,
		})
		cancel()
		if err != nil {
			return err
		}

		if len(out2.Uploads) != 0 {
			return fmt.Errorf("expected empty uplodas list, instead got %v", out2.Uploads)
		}
		expectedCps = []string{"foo/", "some/", "xyz/", "zzz/"}
		if !comparePrefixes(expectedCps, out2.CommonPrefixes) {
			return fmt.Errorf("expected the common prefixes to be %v, instead got %v", expectedCps, out2.CommonPrefixes)
		}
		if getString(out2.KeyMarker) != "foo/" {
			return fmt.Errorf("expected key marker to be 'foo/', instead got %s", getString(out2.KeyMarker))
		}
		if getString(out2.UploadIdMarker) != getString(uploads[1].UploadId) {
			return fmt.Errorf("expected the upload id marker to be %s, instead got %s", getString(uploads[1].UploadId), getString(out2.UploadIdMarker))
		}
		if getString(out2.NextKeyMarker) != "" {
			return fmt.Errorf("expected empty next key marker, instead got %s", getString(out2.NextKeyMarker))
		}
		if getString(out2.NextUploadIdMarker) != "" {
			return fmt.Errorf("expected empty next upload id marker, instead got %s", getString(out2.NextUploadIdMarker))
		}
		if *out2.IsTruncated {
			return fmt.Errorf("expected a non-truncated response")
		}

		return nil
	})
}

func ListMultipartUploads_prefix(s *S3Conf) error {
	testName := "ListMultipartUploads_prefix"
	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
		prefix := "foo"
		uploads := make([]types.MultipartUpload, 0, 8)
		for _, key := range []string{
			"abc/something",
			"foo/bar/baz",
			"foo/quxx",
			"hello/world",
			"xyz/hello",
			"zzz/bca",
			"some/very/nested/mp/object",
			"foo/xyz",
		} {
			out, err := createMp(s3client, bucket, key)
			if err != nil {
				return err
			}

			if strings.HasPrefix(key, prefix) {
				uploads = append(uploads, types.MultipartUpload{
					Key:          out.Key,
					UploadId:     out.UploadId,
					StorageClass: types.StorageClassStandard,
				})
			}
		}

		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket: &bucket,
			Prefix: &prefix,
		})
		cancel()
		if err != nil {
			return err
		}

		if getString(out.Prefix) != prefix {
			return fmt.Errorf("expected the prefix to be %s, instead got %s", prefix, getString(out.Prefix))
		}
		if !compareMultipartUploads(out.Uploads, uploads) {
			return fmt.Errorf("expected the uploads list to be %v, instead got %v", uploads, out.Uploads)
		}

		return nil
	})
}

func ListMultipartUploads_both_delimiter_and_prefix(s *S3Conf) error {
	testName := "ListMultipartUploads_both_delimiter_and_prefix"
	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
		for _, key := range []string{
			"foo/abc/bbb/aaa/c",
			"abc/something",
			"foo/bar/baz",
			"foo/quxx",
			"hello/world",
			"foo/random/object",
			"foo/random/another/object",
			"xyz/hello",
			"zzz/bca",
			"some/very/nested/mp/object",
			"foo/xyz",
		} {
			_, err := createMp(s3client, bucket, key)
			if err != nil {
				return err
			}
		}

		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:    &bucket,
			Delimiter: getPtr("/"),
			Prefix:    getPtr("foo/"),
		})
		cancel()
		if err != nil {
			return err
		}

		expectedCps := []string{"foo/abc/", "foo/bar/", "foo/random/"}
		if !comparePrefixes(expectedCps, out.CommonPrefixes) {
			return fmt.Errorf("expected the common prefixes to be %v, instead got %v", expectedCps, out.CommonPrefixes)
		}

		return nil
	})
}

func ListMultipartUploads_delimiter_no_matches(s *S3Conf) error {
	testName := "ListMultipartUploads_delimiter_no_matches"
	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
		uploads := make([]types.MultipartUpload, 0, 8)
		for _, key := range []string{
			"abc/something",
			"foo/bar/baz",
			"foo/quxx",
			"hello/world",
			"xyz/hello",
			"zzz/bca",
			"some/very/nested/mp/object",
			"foo/xyz",
		} {
			out, err := createMp(s3client, bucket, key)
			if err != nil {
				return err
			}

			uploads = append(uploads, types.MultipartUpload{
				Key:          out.Key,
				UploadId:     out.UploadId,
				StorageClass: types.StorageClassStandard,
			})
		}

		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket:    &bucket,
			Delimiter: getPtr("delim"),
		})
		cancel()
		if err != nil {
			return err
		}

		sort.SliceStable(uploads, func(i, j int) bool {
			return *uploads[i].Key < *uploads[j].Key
		})

		if !compareMultipartUploads(uploads, out.Uploads) {
			return fmt.Errorf("expected the uploads to be %v, instead got %v", uploads, out.Uploads)
		}
		if len(out.CommonPrefixes) != 0 {
			return fmt.Errorf("expected empty common prefixes, instead got %v", out.CommonPrefixes)
		}

		return nil
	})
}

func ListMultipartUploads_with_checksums(s *S3Conf) error {
	testName := "ListMultipartUploads_with_checksums"
	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
		uploads := []types.MultipartUpload{}
		for _, el := range []struct {
			obj  string
			algo types.ChecksumAlgorithm
			t    types.ChecksumType
		}{
			{
				obj:  "obj-1",
				algo: types.ChecksumAlgorithmCrc32,
				t:    types.ChecksumTypeComposite,
			},
			{
				obj:  "obj-2",
				algo: types.ChecksumAlgorithmCrc32c,
				t:    types.ChecksumTypeFullObject,
			},
			{
				obj:  "obj-3",
				algo: types.ChecksumAlgorithmSha1,
				t:    types.ChecksumTypeComposite,
			},
			{
				obj:  "obj-4",
				algo: types.ChecksumAlgorithmSha256,
				t:    types.ChecksumTypeComposite,
			},
			{
				obj:  "obj-5",
				algo: types.ChecksumAlgorithmCrc64nvme,
				t:    types.ChecksumTypeFullObject,
			},
			{
				obj:  "obj-6",
				algo: types.ChecksumAlgorithmSha512,
				t:    types.ChecksumTypeComposite,
			},
			{
				obj:  "obj-7",
				algo: types.ChecksumAlgorithmMd5,
				t:    types.ChecksumTypeComposite,
			},
			{
				obj:  "obj-8",
				algo: types.ChecksumAlgorithmXxhash64,
				t:    types.ChecksumTypeComposite,
			},
			{
				obj:  "obj-9",
				algo: types.ChecksumAlgorithmXxhash3,
				t:    types.ChecksumTypeComposite,
			},
			// obj-91 to preserve the lexicographical order
			{
				obj:  "obj-91",
				algo: types.ChecksumAlgorithmXxhash128,
				t:    types.ChecksumTypeComposite,
			},
		} {
			key := el.obj
			mp, err := createMp(s3client, bucket, key, withChecksum(el.algo), withChecksumType(el.t))
			if err != nil {
				return err
			}

			uploads = append(uploads, types.MultipartUpload{
				Key:               &key,
				UploadId:          mp.UploadId,
				StorageClass:      types.StorageClassStandard,
				ChecksumAlgorithm: el.algo,
				ChecksumType:      el.t,
			})
		}

		ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
		res, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
			Bucket: &bucket,
		})
		cancel()
		if err != nil {
			return err
		}

		if !compareMultipartUploads(res.Uploads, uploads) {
			return fmt.Errorf("expected the final multipart uploads to be %v, instead got %v",
				uploads, res.Uploads)
		}

		return nil
	})
}