From 70865795900d471022c9620f9503aa7b7610b998 Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Fri, 13 Sep 2024 15:00:20 -0700 Subject: [PATCH] fix: list objects trim common prefixes that match marker prefix This checks to see if the common prefix is before the marker and thus would have been returned in earlier list objects request. The error case was aws cli listing multiple entries for the same common prefix when the listing required multiple pagination requests. Fixes #778 --- backend/walk.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/walk.go b/backend/walk.go index 2c8d579..0cf36f6 100644 --- a/backend/walk.go +++ b/backend/walk.go @@ -191,12 +191,19 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin // Common prefixes are a set, so should not have duplicates. // These are abstractly a "directory", so need to include the - // delimiter at the end. + // delimiter at the end when we add to the map. + cprefNoDelim := prefix + before cpref := prefix + before + delimiter if cpref == marker { pastMarker = true return nil } + + if marker != "" && strings.HasPrefix(marker, cprefNoDelim) { + // skip common prefixes that are before the marker + return nil + } + cpmap[cpref] = struct{}{} if (len(objects) + len(cpmap)) == int(max) { newMarker = cpref