From 24f62731f01496855c7a625ad74906851a3b5ebc Mon Sep 17 00:00:00 2001 From: LuisFigueroaG Date: Wed, 7 Oct 2026 11:27:52 -0300 Subject: [PATCH] fix: echo only the request continuation token in ListObjectsV2 The posix ListObjectsV2 reported the listing marker as the response ContinuationToken. The marker is the larger of start-after and the continuation token, so a request with only start-after got its start-after value back as a ContinuationToken, and a start-after past the continuation token replaced it. S3 echoes the continuation token from the request, and leaves ContinuationToken out when the request had none. Return the request continuation token instead of the marker, and enable the REST test that covers it. Fixes #2007 --- backend/posix/list_objects_v2_test.go | 88 +++++++++++++++++++++++++++ backend/posix/posix.go | 2 +- tests/integration/ListObjectsV2.go | 69 +++++++++++++++++++++ tests/integration/group-tests.go | 2 + tests/test_rest_list_objects.sh | 3 - 5 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 backend/posix/list_objects_v2_test.go diff --git a/backend/posix/list_objects_v2_test.go b/backend/posix/list_objects_v2_test.go new file mode 100644 index 00000000..8c21a709 --- /dev/null +++ b/backend/posix/list_objects_v2_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package posix + +import ( + "context" + "reflect" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/versity/versitygw/s3response" +) + +// TestListObjectsV2ContinuationTokenEcho checks that ListObjectsV2 echoes +// the request continuation token, and only that: start-after must not be +// reported back as a continuation token. +func TestListObjectsV2ContinuationTokenEcho(t *testing.T) { + p := newTestPosix(t, metaModes(t)["xattr"]) + bucket := "testbucket" + createTestBucket(t, p, bucket) + + ctx := context.Background() + for _, key := range []string{"a", "b", "c"} { + _, err := p.PutObject(ctx, s3response.PutObjectInput{ + Bucket: &bucket, + Key: aws.String(key), + Body: strings.NewReader("data"), + ContentLength: aws.Int64(4), + }) + if err != nil { + t.Fatalf("put object %q: %v", key, err) + } + } + + tests := []struct { + name string + cToken string + startAfter string + wantToken *string + wantKeys []string + }{ + {"no token or start-after", "", "", nil, []string{"a", "b", "c"}}, + {"start-after only", "", "a", nil, []string{"b", "c"}}, + {"continuation token only", "a", "", aws.String("a"), []string{"b", "c"}}, + {"start-after past the continuation token", "a", "b", aws.String("a"), []string{"c"}}, + {"continuation token past start-after", "b", "a", aws.String("b"), []string{"c"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out, err := p.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + ContinuationToken: &tt.cToken, + StartAfter: &tt.startAfter, + MaxKeys: aws.Int32(1000), + }) + if err != nil { + t.Fatalf("list objects v2: %v", err) + } + + if !reflect.DeepEqual(out.ContinuationToken, tt.wantToken) { + t.Fatalf("continuation token = %q, want %q", + aws.ToString(out.ContinuationToken), aws.ToString(tt.wantToken)) + } + + var keys []string + for _, obj := range out.Contents { + keys = append(keys, aws.ToString(obj.Key)) + } + if strings.Join(keys, ",") != strings.Join(tt.wantKeys, ",") { + t.Fatalf("keys = %v, want %v", keys, tt.wantKeys) + } + }) + } +} diff --git a/backend/posix/posix.go b/backend/posix/posix.go index a2a39b9e..201c9688 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -7122,7 +7122,7 @@ func (p *Posix) ListObjectsV2Parametrized(ctx context.Context, input *s3.ListObj Name: &bucket, KeyCount: &count, Delimiter: backend.GetPtrFromString(delim), - ContinuationToken: backend.GetPtrFromString(marker), + ContinuationToken: backend.GetPtrFromString(backend.GetStringFromPtr(input.ContinuationToken)), NextContinuationToken: backend.GetPtrFromString(results.NextMarker), Prefix: backend.GetPtrFromString(prefix), StartAfter: backend.GetPtrFromString(*input.StartAfter), diff --git a/tests/integration/ListObjectsV2.go b/tests/integration/ListObjectsV2.go index a0365cc6..f3f85e47 100644 --- a/tests/integration/ListObjectsV2.go +++ b/tests/integration/ListObjectsV2.go @@ -175,6 +175,75 @@ func ListObjectsV2_start_after_empty_result(s *S3Conf) error { }) } +func ListObjectsV2_start_after_continuation_token(s *S3Conf) error { + testName := "ListObjectsV2_start_after_continuation_token" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + contents, err := putObjects(s3client, []string{"foo", "bar", "baz", "quxx"}, bucket) + if err != nil { + return err + } + + // start-after alone is not echoed back as a continuation token + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + StartAfter: getPtr("bar"), + }) + cancel() + if err != nil { + return err + } + + if out.ContinuationToken != nil { + return fmt.Errorf("expected nil ContinuationToken, instead got %v", + *out.ContinuationToken) + } + if !compareObjects(contents[1:], out.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", + contents[1:], out.Contents) + } + + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + MaxKeys: getPtr(int32(1)), + }) + cancel() + if err != nil { + return err + } + + token := getString(out.NextContinuationToken) + if token == "" { + return fmt.Errorf("expected non-empty NextContinuationToken") + } + + // with both, the request continuation token is echoed back even + // when start-after is past it + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) + out, err = s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + ContinuationToken: &token, + StartAfter: getPtr("baz"), + }) + cancel() + if err != nil { + return err + } + + if getString(out.ContinuationToken) != token { + return fmt.Errorf("expected ContinuationToken to be %v, instead got %v", + token, getString(out.ContinuationToken)) + } + if !compareObjects(contents[2:], out.Contents) { + return fmt.Errorf("expected the output to be %v, instead got %v", + contents[2:], out.Contents) + } + + return nil + }) +} + func ListObjectsV2_both_delimiter_and_prefix(s *S3Conf) error { testName := "ListObjectsV2_both_delimiter_and_prefix" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 328683b1..c7c55c57 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -334,6 +334,7 @@ func TestListObjectsV2(ts *TestState) { ts.Run(ListObjectsV2_both_start_after_and_continuation_token) ts.Run(ListObjectsV2_start_after_not_in_list) ts.Run(ListObjectsV2_start_after_empty_result) + ts.Run(ListObjectsV2_start_after_continuation_token) ts.Run(ListObjectsV2_both_delimiter_and_prefix) ts.Run(ListObjectsV2_single_dir_object_with_delim_and_prefix) ts.Run(ListObjectsV2_truncated_common_prefixes) @@ -3136,6 +3137,7 @@ func GetIntTests() IntTests { "ListObjectsV2_both_start_after_and_continuation_token": ListObjectsV2_both_start_after_and_continuation_token, "ListObjectsV2_start_after_not_in_list": ListObjectsV2_start_after_not_in_list, "ListObjectsV2_start_after_empty_result": ListObjectsV2_start_after_empty_result, + "ListObjectsV2_start_after_continuation_token": ListObjectsV2_start_after_continuation_token, "ListObjectsV2_both_delimiter_and_prefix": ListObjectsV2_both_delimiter_and_prefix, "ListObjectsV2_single_dir_object_with_delim_and_prefix": ListObjectsV2_single_dir_object_with_delim_and_prefix, "ListObjectsV2_truncated_common_prefixes": ListObjectsV2_truncated_common_prefixes, diff --git a/tests/test_rest_list_objects.sh b/tests/test_rest_list_objects.sh index f215e700..efc46a7d 100755 --- a/tests/test_rest_list_objects.sh +++ b/tests/test_rest_list_objects.sh @@ -279,9 +279,6 @@ source ./tests/drivers/objects_and_versions.sh # tags: curl,ListObjectsV2,start-after,continuation-token @test "ListObjectsV1 - start-after - doesn't include continuation token" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/2007" - fi run setup_bucket_and_files_v3 "$BUCKET_ONE_NAME" 2 assert_success read -r bucket_name file_one file_two <<< "$output"