Merge pull request #2476 from dualfroz/fix/2144-list-mp-uploads-next-markers

fix: report the last upload in ListMultipartUploads next markers on a single page

backend/mp-lister.go

@@ -127,8 +127,9 @@ func (l *MultipartUploadLister) Run() (*ListMultipartUploadsPage, error) {
 						if out.IsTruncated {
 							out.NextKeyMarker = lastKey
 							out.NextUploadIDMarker = up.UploadID
+							return out, nil
 						}
-						return out, nil
+						break
 					}
 				}
 				continue
@@ -140,11 +141,20 @@ func (l *MultipartUploadLister) Run() (*ListMultipartUploadsPage, error) {
 			if out.IsTruncated {
 				out.NextKeyMarker = lastKey
 				out.NextUploadIDMarker = up.UploadID
+				return out, nil
 			}
-			return out, nil
+			break
 		}
 	}
 
+	// a listing that is not truncated still reports its last upload
+	// in the next markers
+	if len(out.Uploads) != 0 {
+		last := out.Uploads[len(out.Uploads)-1]
+		out.NextKeyMarker = last.Key
+		out.NextUploadIDMarker = last.UploadID
+	}
+
 	return out, nil
 }
 

tests/integration/ListMultipartUploads.go

@@ -153,6 +153,57 @@ func ListMultipartUploads_max_uploads(s *S3Conf) error {
 	})
 }
 
+func ListMultipartUploads_next_markers_not_truncated(s *S3Conf) error {
+	testName := "ListMultipartUploads_next_markers_not_truncated"
+	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
+		uploads := []types.MultipartUpload{}
+		for _, key := range []string{"bar", "baz", "foo"} {
+			out, err := createMp(s3client, bucket, key)
+			if err != nil {
+				return err
+			}
+			uploads = append(uploads, types.MultipartUpload{
+				UploadId:     out.UploadId,
+				Key:          out.Key,
+				StorageClass: types.StorageClassStandard,
+			})
+		}
+		last := uploads[len(uploads)-1]
+
+		// a single page still reports the last upload in the next markers,
+		// with the default max-uploads and with one equal to the upload count
+		for _, maxUploads := range []*int32{nil, getPtr(int32(len(uploads)))} {
+			ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
+			out, err := s3client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
+				Bucket:     &bucket,
+				MaxUploads: maxUploads,
+			})
+			cancel()
+			if err != nil {
+				return err
+			}
+
+			if out.IsTruncated == nil || *out.IsTruncated {
+				return fmt.Errorf("expected a non-truncated response")
+			}
+			if ok := compareMultipartUploads(out.Uploads, uploads); !ok {
+				return fmt.Errorf("expected multipart uploads to be %v, instead got %v",
+					uploads, out.Uploads)
+			}
+			if getString(out.NextKeyMarker) != getString(last.Key) {
+				return fmt.Errorf("expected next-key-marker to be %v, instead got %v",
+					getString(last.Key), getString(out.NextKeyMarker))
+			}
+			if getString(out.NextUploadIdMarker) != getString(last.UploadId) {
+				return fmt.Errorf("expected next-upload-id-marker to be %v, instead got %v",
+					getString(last.UploadId), getString(out.NextUploadIdMarker))
+			}
+		}
+
+		return nil
+	})
+}
+
 func ListMultipartUploads_exceeding_max_uploads(s *S3Conf) error {
 	testName := "ListMultipartUploads_exceeding_max_uploads"
 	return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {

tests/integration/group-tests.go

@@ -546,6 +546,7 @@ func TestListMultipartUploads(ts *TestState) {
 	ts.Run(ListMultipartUploads_empty_result)
 	ts.Run(ListMultipartUploads_invalid_max_uploads)
 	ts.Run(ListMultipartUploads_max_uploads)
+	ts.Run(ListMultipartUploads_next_markers_not_truncated)
 	ts.Run(ListMultipartUploads_exceeding_max_uploads)
 	ts.Run(ListMultipartUploads_ignore_upload_id_marker)
 	ts.Run(ListMultipartUploads_invalid_uploadId_marker)
@@ -3285,6 +3286,7 @@ func GetIntTests() IntTests {
 		"ListMultipartUploads_empty_result":                                                ListMultipartUploads_empty_result,
 		"ListMultipartUploads_invalid_max_uploads":                                         ListMultipartUploads_invalid_max_uploads,
 		"ListMultipartUploads_max_uploads":                                                 ListMultipartUploads_max_uploads,
+		"ListMultipartUploads_next_markers_not_truncated":                                  ListMultipartUploads_next_markers_not_truncated,
 		"ListMultipartUploads_exceeding_max_uploads":                                       ListMultipartUploads_exceeding_max_uploads,
 		"ListMultipartUploads_ignore_upload_id_marker":                                     ListMultipartUploads_ignore_upload_id_marker,
 		"ListMultipartUploads_invalid_uploadId_marker":                                     ListMultipartUploads_invalid_uploadId_marker,

tests/test_rest_list_multipart_uploads.sh

@@ -50,9 +50,6 @@ source ./tests/setup.sh
 
 # tags: curl,ListMultipartUploads,multipart,uploadId,next-key-marker,next-upload-id-marker
 @test "REST - ListMultipartUploads - NextKeyMarker and NextUploadIdMarker set to last item" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/2144"
-  fi
   run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2
   assert_success
   read -r bucket_name test_file_one test_file_two <<< "$output"