Merge pull request #2447 from versity/ben/list-objects-marker

fix: return Marker when the request omits it

s3api/controllers/bucket-get.go

@@ -651,6 +651,14 @@ func (c S3ApiController) ListObjects(ctx fiber.Ctx) (*Response, error) {
 		}, err
 	}
 
+	// S3 echoes the request marker back in the response, and includes the
+	// element even when the request carried no marker. Backends return a nil
+	// pointer for an empty marker, which encoding/xml drops entirely, so
+	// restore it here for every backend.
+	if res.Marker == nil {
+		res.Marker = &marker
+	}
+
 	return &Response{
 		Headers: map[string]*string{
 			"x-amz-bucket-region": &region,

s3api/controllers/bucket-get_test.go

@@ -1169,6 +1169,13 @@ func TestS3ApiController_ListObjects(t *testing.T) {
 			{Key: utils.GetStringPtr("my-key")},
 		},
 	}
+	// The controller echoes the request marker back, so the element is in
+	// the response even when the backend leaves it unset. utils.GetStringPtr
+	// collapses "" to nil, which is the behaviour under test here, so take
+	// the address of an empty string directly.
+	emptyMarker := ""
+	listResultEchoed := listResult
+	listResultEchoed.Marker = &emptyMarker
 	tests := []struct {
 		name   string
 		input  testInput
@@ -1229,7 +1236,7 @@ func TestS3ApiController_ListObjects(t *testing.T) {
 			},
 			output: testOutput{
 				response: &Response{
-					Data: listResult,
+					Data: listResultEchoed,
 					Headers: map[string]*string{
 						"x-amz-bucket-region": utils.GetStringPtr(defaultRegion),
 					},

tests/drivers/list_objects/list_objects_rest.sh

@@ -564,3 +564,36 @@ list_objects_v1_check_nextmarker_empty() {
   fi
   return 0
 }
+
+# S3 returns the request marker in a v1 listing response, including an empty
+# element when the request carries no marker.
+list_objects_v1_check_empty_marker() {
+  if ! check_param_count "list_objects_v1_check_empty_marker" "bucket" 1 $#; then
+    return 1
+  fi
+  if ! response=$(get_file_names 2 2>&1); then
+    log 2 "error getting file names: $response"
+    return 1
+  fi
+  read -r file_name list_bucket_file <<< "$response"
+
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="FALSE" OUTPUT_FILE="$TEST_FILE_FOLDER/$file_name" ./tests/rest_scripts/list_objects.sh); then
+    log 2 "error listing objects: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "expected '200' was '$result' ($(cat "$TEST_FILE_FOLDER/$file_name"))"
+    return 1
+  fi
+  log 5 "objects: $(cat "$TEST_FILE_FOLDER/$file_name")"
+  if ! list_bucket_result=$(xmllint --xpath '//*[local-name()="ListBucketResult"]' "$TEST_FILE_FOLDER/$file_name" 2>&1); then
+    log 2 "error getting list bucket result: $list_bucket_result"
+    return 1
+  fi
+  echo -n "$list_bucket_result" > "$TEST_FILE_FOLDER/$list_bucket_file"
+  if ! check_xml_element "$TEST_FILE_FOLDER/$list_bucket_file" "" "Marker"; then
+    log 2 "Marker should be present and empty"
+    return 1
+  fi
+  return 0
+}

tests/integration/ListObjects.go

@@ -402,8 +402,13 @@ func ListObjects_list_all_objs(s *S3Conf) error {
 			return err
 		}
 
-		if out.Marker != nil {
-			return fmt.Errorf("expected the Marker to be nil, instead got %v",
+		// S3 echoes the request marker back, so the response carries the
+		// element with an empty value rather than omitting it
+		if out.Marker == nil {
+			return fmt.Errorf("expected the Marker to be an empty string, instead got nil")
+		}
+		if *out.Marker != "" {
+			return fmt.Errorf("expected the Marker to be an empty string, instead got %v",
 				*out.Marker)
 		}
 		if out.NextMarker != nil {

tests/test_rest_list_objects.sh

@@ -347,3 +347,23 @@ list_objects_delimiter() {
   run list_objects_with_prefix_and_delimiter_check_results "$bucket_name" "$1" "$prefix" "/" "a-b/" "--" "a-b-1.txt" "a-b-2.txt"
   assert_success
 }
+
+# tags: curl,ListObjects,marker
+@test "REST - ListObjects - Marker returned when request has no marker" {
+  run get_bucket_name "$BUCKET_ONE_NAME"
+  assert_success
+  bucket_name="$output"
+
+  run get_file_name
+  assert_success
+  test_file="$output"
+
+  run setup_bucket_and_file_v2 "$bucket_name" "$test_file"
+  assert_success
+
+  run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$bucket_name" "$test_file"
+  assert_success
+
+  run list_objects_v1_check_empty_marker "$bucket_name"
+  assert_success
+}