diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go
index d77eb3d32..9fe39618b 100644
--- a/weed/s3api/s3api_object_handlers_list.go
+++ b/weed/s3api/s3api_object_handlers_list.go
@@ -109,7 +109,15 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
// Adjust marker if it ends with delimiter to skip all entries with that prefix
marker = adjustMarkerForDelimiter(marker, delimiter)
- response, err := s3a.listFilerEntries(r.Context(), bucket, originalPrefix, maxKeys, marker, delimiter, encodingTypeUrl, fetchOwner)
+ response, err := s3a.listFilerEntries(r.Context(), listObjectsRequest{
+ bucket: bucket,
+ prefix: originalPrefix,
+ marker: marker,
+ delimiter: delimiter,
+ maxKeys: maxKeys,
+ encodingTypeUrl: encodingTypeUrl,
+ fetchOwner: fetchOwner,
+ })
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
@@ -173,7 +181,15 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
// Adjust marker if it ends with delimiter to skip all entries with that prefix
marker = adjustMarkerForDelimiter(marker, delimiter)
- response, err := s3a.listFilerEntries(r.Context(), bucket, originalPrefix, uint16(maxKeys), marker, delimiter, encodingTypeUrl, true)
+ response, err := s3a.listFilerEntries(r.Context(), listObjectsRequest{
+ bucket: bucket,
+ prefix: originalPrefix,
+ marker: marker,
+ delimiter: delimiter,
+ maxKeys: uint16(maxKeys),
+ encodingTypeUrl: encodingTypeUrl,
+ fetchOwner: true,
+ })
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
@@ -232,7 +248,20 @@ func sanitizeV1MarkerEcho(response *ListBucketResult, marker string, encodingTyp
}
}
-func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket string, originalPrefix string, maxKeys uint16, originalMarker string, delimiter string, encodingTypeUrl bool, fetchOwner bool) (response ListBucketResult, err error) {
+type listObjectsRequest struct {
+ bucket string
+ prefix string
+ marker string
+ delimiter string
+ maxKeys uint16
+ encodingTypeUrl bool
+ fetchOwner bool
+}
+
+func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, req listObjectsRequest) (response ListBucketResult, err error) {
+ bucket, originalPrefix, originalMarker := req.bucket, req.prefix, req.marker
+ maxKeys, delimiter := req.maxKeys, req.delimiter
+ encodingTypeUrl, fetchOwner := req.encodingTypeUrl, req.fetchOwner
// convert full path prefix into directory name and prefix for entry name
requestDir, prefix, marker := normalizePrefixMarker(originalPrefix, originalMarker)
bucketPrefix := s3a.bucketPrefix(bucket)
@@ -298,7 +327,7 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket string, ori
for {
empty := true
- nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, bucket, func(dir string, entry *filer_pb.Entry) {
+ nextMarker, doErr = s3a.doListFilerEntries(ctx, client, listDirectoryRequest{dir: reqDir, prefix: prefix, marker: marker, delimiter: delimiter, bucket: bucket}, cursor, func(dir string, entry *filer_pb.Entry) {
empty = false
dirName, entryName, _ := entryUrlEncode(dir, entry.Name, encodingTypeUrl)
if entry.IsDirectory {
@@ -557,7 +586,18 @@ func buildTruncatedNextMarker(requestDir, prefix, nextMarker string, lastEntryWa
return nextMarker
}
-func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, cursor *ListingCursor, marker, delimiter string, inclusiveStartFrom bool, bucket string, eachEntryFn func(dir string, entry *filer_pb.Entry)) (nextMarker string, err error) {
+type listDirectoryRequest struct {
+ dir string
+ prefix string
+ marker string
+ delimiter string
+ bucket string
+ inclusiveStartFrom bool
+}
+
+func (s3a *S3ApiServer) doListFilerEntries(ctx context.Context, client filer_pb.SeaweedFilerClient, req listDirectoryRequest, cursor *ListingCursor, eachEntryFn func(dir string, entry *filer_pb.Entry)) (nextMarker string, err error) {
+ dir, prefix, bucket := req.dir, req.prefix, req.bucket
+ marker, delimiter, inclusiveStartFrom := req.marker, req.delimiter, req.inclusiveStartFrom
// invariants
// prefix and marker should be under dir, marker may contain "/"
// maxKeys should be updated for each recursion
@@ -571,7 +611,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
if strings.Contains(marker, "/") {
subDir, subMarker := toParentAndDescendants(marker)
// println("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker)
- subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", cursor, subMarker, delimiter, false, bucket, eachEntryFn)
+ subNextMarker, subErr := s3a.doListFilerEntries(ctx, client, listDirectoryRequest{dir: dir + "/" + subDir, marker: subMarker, delimiter: delimiter, bucket: bucket}, cursor, eachEntryFn)
if subErr != nil {
err = subErr
return
@@ -585,156 +625,170 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
}
// now marker is also a direct child of dir
- request := &filer_pb.ListEntriesRequest{
- Directory: dir,
- Prefix: prefix,
- Limit: uint32(cursor.maxKeys) + 2, // bucket root directory needs to skip additional s3_constants.MultipartUploadsFolder folder
- StartFromFileName: marker,
- InclusiveStartFrom: inclusiveStartFrom,
- }
-
- ctx, cancel := context.WithCancel(context.Background())
+ ctx, cancel := context.WithCancel(ctx)
defer cancel()
- stream, listErr := client.ListEntries(ctx, request)
- if listErr != nil {
- if errors.Is(listErr, filer_pb.ErrNotFound) {
- return
- }
- err = fmt.Errorf("list entries %+v: %w", request, listErr)
- return
- }
+ // Entries that emit nothing (empty directories, the .uploads folder, the marker
+ // echo) consume the request window without consuming maxKeys, so one window may
+ // end before maxKeys is satisfied. Keep requesting from the last received entry
+ // until the quota is filled or a short window shows the directory is exhausted.
for {
- resp, recvErr := stream.Recv()
- if recvErr != nil {
- if recvErr == io.EOF {
- break
- } else {
- err = fmt.Errorf("iterating entries %+v: %v", request, recvErr)
+ request := &filer_pb.ListEntriesRequest{
+ Directory: dir,
+ Prefix: prefix,
+ Limit: uint32(cursor.maxKeys) + 2, // bucket root directory needs to skip additional s3_constants.MultipartUploadsFolder folder
+ StartFromFileName: marker,
+ InclusiveStartFrom: inclusiveStartFrom,
+ }
+
+ stream, listErr := client.ListEntries(ctx, request)
+ if listErr != nil {
+ if errors.Is(listErr, filer_pb.ErrNotFound) {
return
}
- }
- entry := resp.Entry
- if entry == nil {
- continue
- }
- // listFilerEntries always calls doListFilerEntries with inclusiveStartFrom=false
- // (S3 marker semantics are exclusive), but keep the guard explicit to preserve
- // behavior if inclusive callers are introduced in the future.
- if !inclusiveStartFrom && marker != "" && entry.Name == marker {
- continue
+ err = fmt.Errorf("list entries %+v: %w", request, listErr)
+ return
}
- if cursor.maxKeys <= 0 {
- cursor.isTruncated = true
- break
- }
-
- // Set nextMarker only when we have quota to process this entry
- nextMarker = entry.Name
- // Track whether this entry is the exact directory targeted by a trailing-slash prefix
- // (e.g., prefix "foo" from original prefix "foo/"). After recursing into this directory,
- // we must stop processing siblings to avoid matching unrelated entries like "foo1000".
- matchedPrefixDir := cursor.prefixEndsOnDelimiter && entry.Name == prefix && entry.IsDirectory
- if cursor.prefixEndsOnDelimiter {
- if entry.Name == prefix && entry.IsDirectory {
- if delimiter != "/" {
- cursor.prefixEndsOnDelimiter = false
- }
- } else {
- continue
- }
- }
- if entry.IsDirectory {
- // glog.V(4).Infof("List Dir Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys)
- if entry.Name == s3_constants.MultipartUploadsFolder { // FIXME no need to apply to all directories. this extra also affects maxKeys
- continue
- }
-
- // Process .versions directories immediately to create logical versioned object entries
- // These directories are never traversed (we continue here), so each is only encountered once
- if strings.HasSuffix(entry.Name, s3_constants.VersionsFolder) {
- // Extract object name from .versions directory name
- baseObjectName := strings.TrimSuffix(entry.Name, s3_constants.VersionsFolder)
- // Construct full object path relative to bucket
- bucketFullPath := s3a.bucketDir(bucket)
- bucketRelativePath := strings.TrimPrefix(dir, bucketFullPath)
- bucketRelativePath = strings.TrimPrefix(bucketRelativePath, "/")
- var fullObjectPath string
- if bucketRelativePath == "" {
- fullObjectPath = baseObjectName
+ var entriesReceived uint32
+ var lastEntryName string
+ for {
+ resp, recvErr := stream.Recv()
+ if recvErr != nil {
+ if recvErr == io.EOF {
+ break
} else {
- fullObjectPath = bucketRelativePath + "/" + baseObjectName
- }
- // Use metadata from the already-fetched .versions directory entry
- if latestVersionEntry, err := s3a.getLatestVersionEntryFromDirectoryEntry(bucket, fullObjectPath, entry); err == nil {
- eachEntryFn(dir, latestVersionEntry)
- } else if !errors.Is(err, ErrDeleteMarker) {
- // Log unexpected errors (delete markers are expected)
- glog.V(2).Infof("Skipping versioned object %s due to error: %v", fullObjectPath, err)
+ err = fmt.Errorf("iterating entries %+v: %v", request, recvErr)
+ return
}
+ }
+ entry := resp.Entry
+ if entry == nil {
+ continue
+ }
+ entriesReceived++
+ lastEntryName = entry.Name
+ // listFilerEntries always calls doListFilerEntries with inclusiveStartFrom=false
+ // (S3 marker semantics are exclusive), but keep the guard explicit to preserve
+ // behavior if inclusive callers are introduced in the future.
+ if !inclusiveStartFrom && marker != "" && entry.Name == marker {
continue
}
- if delimiter != "/" || cursor.prefixEndsOnDelimiter {
- // A trailing-slash prefix (e.g. "logs/") names one directory and asks
- // whether it exists, so a real but empty directory must be reported for
- // that probe.
- explicitDirProbe := cursor.prefixEndsOnDelimiter
- if cursor.prefixEndsOnDelimiter {
- cursor.prefixEndsOnDelimiter = false
+ if cursor.maxKeys <= 0 {
+ cursor.isTruncated = true
+ break
+ }
+
+ // Set nextMarker only when we have quota to process this entry
+ nextMarker = entry.Name
+ // Track whether this entry is the exact directory targeted by a trailing-slash prefix
+ // (e.g., prefix "foo" from original prefix "foo/"). After recursing into this directory,
+ // we must stop processing siblings to avoid matching unrelated entries like "foo1000".
+ matchedPrefixDir := cursor.prefixEndsOnDelimiter && entry.Name == prefix && entry.IsDirectory
+ if cursor.prefixEndsOnDelimiter {
+ if entry.Name == prefix && entry.IsDirectory {
+ if delimiter != "/" {
+ cursor.prefixEndsOnDelimiter = false
+ }
+ } else {
+ continue
}
- isKeyObject := entry.IsDirectoryKeyObject()
- if isKeyObject {
- // Directory key objects (created via PutObject with trailing "/")
- // must appear as regular keys in recursive listing mode.
+ }
+ if entry.IsDirectory {
+ // glog.V(4).Infof("List Dir Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys)
+ if entry.Name == s3_constants.MultipartUploadsFolder { // FIXME no need to apply to all directories. this extra also affects maxKeys
+ continue
+ }
+
+ // Process .versions directories immediately to create logical versioned object entries
+ // These directories are never traversed (we continue here), so each is only encountered once
+ if strings.HasSuffix(entry.Name, s3_constants.VersionsFolder) {
+ // Extract object name from .versions directory name
+ baseObjectName := strings.TrimSuffix(entry.Name, s3_constants.VersionsFolder)
+ // Construct full object path relative to bucket
+ bucketFullPath := s3a.bucketDir(bucket)
+ bucketRelativePath := strings.TrimPrefix(dir, bucketFullPath)
+ bucketRelativePath = strings.TrimPrefix(bucketRelativePath, "/")
+ var fullObjectPath string
+ if bucketRelativePath == "" {
+ fullObjectPath = baseObjectName
+ } else {
+ fullObjectPath = bucketRelativePath + "/" + baseObjectName
+ }
+ // Use metadata from the already-fetched .versions directory entry
+ if latestVersionEntry, err := s3a.getLatestVersionEntryFromDirectoryEntry(bucket, fullObjectPath, entry); err == nil {
+ eachEntryFn(dir, latestVersionEntry)
+ } else if !errors.Is(err, ErrDeleteMarker) {
+ // Log unexpected errors (delete markers are expected)
+ glog.V(2).Infof("Skipping versioned object %s due to error: %v", fullObjectPath, err)
+ }
+ continue
+ }
+
+ if delimiter != "/" || cursor.prefixEndsOnDelimiter {
+ // A trailing-slash prefix (e.g. "logs/") names one directory and asks
+ // whether it exists, so a real but empty directory must be reported for
+ // that probe.
+ explicitDirProbe := cursor.prefixEndsOnDelimiter
+ if cursor.prefixEndsOnDelimiter {
+ cursor.prefixEndsOnDelimiter = false
+ }
+ isKeyObject := entry.IsDirectoryKeyObject()
+ if isKeyObject {
+ // Directory key objects (created via PutObject with trailing "/")
+ // must appear as regular keys in recursive listing mode.
+ eachEntryFn(dir, entry)
+ }
+ // Recurse into subdirectory to list any children, noting whether the
+ // subtree produced any entries.
+ childEmitted := false
+ subNextMarker, subErr := s3a.doListFilerEntries(ctx, client, listDirectoryRequest{dir: dir + "/" + entry.Name, delimiter: delimiter, bucket: bucket}, cursor, func(d string, e *filer_pb.Entry) {
+ childEmitted = true
+ eachEntryFn(d, e)
+ })
+ if subErr != nil {
+ err = fmt.Errorf("doListFilerEntries2: %w", subErr)
+ return
+ }
+ // A real but empty directory (created out of band via mount, mkdir or
+ // the filer API, so it carries no MIME) is otherwise invisible to S3
+ // clients that detect directories by listing it under its own "
/"
+ // prefix. Surface it as a directory marker for that explicit probe,
+ // identical to a directory created via PutObject with a trailing "/", so
+ // tools like hadoop-aws can find it. Plain listings are left untouched, so
+ // empty directories left behind by deleted objects are not shown as keys.
+ if explicitDirProbe && !isKeyObject && !childEmitted && !cursor.isTruncated && entry.Attributes != nil {
+ entry.Attributes.Mime = s3_constants.FolderMimeType
+ eachEntryFn(dir, entry)
+ }
+ // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "subNextMarker", subNextMarker)
+ nextMarker = entry.Name + "/" + subNextMarker
+ if cursor.isTruncated {
+ return
+ }
+ if matchedPrefixDir {
+ return
+ }
+ // println("doListFilerEntries2 nextMarker", nextMarker)
+ } else {
eachEntryFn(dir, entry)
}
- // Recurse into subdirectory to list any children, noting whether the
- // subtree produced any entries.
- childEmitted := false
- subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", cursor, "", delimiter, false, bucket, func(d string, e *filer_pb.Entry) {
- childEmitted = true
- eachEntryFn(d, e)
- })
- if subErr != nil {
- err = fmt.Errorf("doListFilerEntries2: %w", subErr)
- return
- }
- // A real but empty directory (created out of band via mount, mkdir or
- // the filer API, so it carries no MIME) is otherwise invisible to S3
- // clients that detect directories by listing it under its own "/"
- // prefix. Surface it as a directory marker for that explicit probe,
- // identical to a directory created via PutObject with a trailing "/", so
- // tools like hadoop-aws can find it. Plain listings are left untouched, so
- // empty directories left behind by deleted objects are not shown as keys.
- if explicitDirProbe && !isKeyObject && !childEmitted && !cursor.isTruncated && entry.Attributes != nil {
- entry.Attributes.Mime = s3_constants.FolderMimeType
- eachEntryFn(dir, entry)
- }
- // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "subNextMarker", subNextMarker)
- nextMarker = entry.Name + "/" + subNextMarker
- if cursor.isTruncated {
- return
- }
- if matchedPrefixDir {
- return
- }
- // println("doListFilerEntries2 nextMarker", nextMarker)
} else {
eachEntryFn(dir, entry)
+ // glog.V(4).Infof("List File Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys)
+ }
+ if cursor.prefixEndsOnDelimiter {
+ cursor.prefixEndsOnDelimiter = false
}
- } else {
- eachEntryFn(dir, entry)
- // glog.V(4).Infof("List File Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys)
}
- if cursor.prefixEndsOnDelimiter {
- cursor.prefixEndsOnDelimiter = false
- }
- }
- // Versioned directories are processed above (lines 524-546)
- return
+ if cursor.isTruncated || entriesReceived < request.Limit {
+ return
+ }
+ marker = lastEntryName
+ inclusiveStartFrom = false
+ }
}
func getListObjectsV2Args(values url.Values) (prefix, startAfter, delimiter string, token OptionalString, encodingTypeUrl bool, fetchOwner bool, maxkeys uint16, allowUnordered bool, errCode s3err.ErrorCode) {
diff --git a/weed/s3api/s3api_object_handlers_list_directory_test.go b/weed/s3api/s3api_object_handlers_list_directory_test.go
index ee7623a2a..e4e65500a 100644
--- a/weed/s3api/s3api_object_handlers_list_directory_test.go
+++ b/weed/s3api/s3api_object_handlers_list_directory_test.go
@@ -1,6 +1,7 @@
package s3api
import (
+ "context"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -38,14 +39,14 @@ func TestDirectoryListedAsCommonPrefix(t *testing.T) {
var seenFiles []string
_, err := s3a.doListFilerEntries(
+ context.Background(),
client,
- "/buckets/xoa-bucket/xo-vm-backups/data",
- "", // prefix
+ listDirectoryRequest{
+ dir: "/buckets/xoa-bucket/xo-vm-backups/data",
+ delimiter: "/", // should yield directory for CommonPrefix processing
+ bucket: "xoa-bucket",
+ },
cursor,
- "", // marker
- "/", // delimiter="/" - should yield directory for CommonPrefix processing
- false,
- "xoa-bucket",
func(dir string, entry *filer_pb.Entry) {
if entry.IsDirectory {
seenDirs = append(seenDirs, entry.Name)
@@ -88,7 +89,7 @@ func TestEmptyDirectorySurfacedAsMarker(t *testing.T) {
// Mirrors the getFileStatus probe: prefix "logs/" with delimiter "/".
cursor := &ListingCursor{maxKeys: 1000, prefixEndsOnDelimiter: true}
var seen []*filer_pb.Entry
- _, err := s3a.doListFilerEntries(client, "/buckets/test", "logs", cursor, "", "/", false, "test",
+ _, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/test", prefix: "logs", delimiter: "/", bucket: "test"}, cursor,
func(dir string, entry *filer_pb.Entry) {
seen = append(seen, entry)
})
@@ -129,7 +130,7 @@ func TestNonEmptyDirectoryGetsNoPhantomMarker(t *testing.T) {
// Recursive listing under the "data/" prefix.
cursor := &ListingCursor{maxKeys: 1000, prefixEndsOnDelimiter: true}
var seen []string
- _, err := s3a.doListFilerEntries(client, "/buckets/test", "data", cursor, "", "", false, "test",
+ _, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/test", prefix: "data", bucket: "test"}, cursor,
func(dir string, entry *filer_pb.Entry) {
seen = append(seen, entry.Name)
})
@@ -173,7 +174,7 @@ func TestEmptyDirectoryHiddenInFlatListing(t *testing.T) {
// Plain flat listing: no prefix, no delimiter.
cursor := &ListingCursor{maxKeys: 1000}
var seen []string
- _, err := s3a.doListFilerEntries(client, "/buckets/test", "", cursor, "", "", false, "test",
+ _, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/test", bucket: "test"}, cursor,
func(dir string, entry *filer_pb.Entry) {
seen = append(seen, entry.Name)
})
diff --git a/weed/s3api/s3api_object_handlers_list_test.go b/weed/s3api/s3api_object_handlers_list_test.go
index ffbe81c80..8d5d990cd 100644
--- a/weed/s3api/s3api_object_handlers_list_test.go
+++ b/weed/s3api/s3api_object_handlers_list_test.go
@@ -44,8 +44,7 @@ type testFilerClient struct {
func (c *testFilerClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
entries := c.entriesByDir[in.Directory]
- // Simplified mock: implements basic prefix filtering but ignores Limit, StartFromFileName, and InclusiveStartFrom
- // to keep test logic focused. Prefix "/" is treated as no filter for bucket root compatibility.
+ // Simplified mock: prefix "/" is treated as no filter for bucket root compatibility.
if in.Prefix != "" && in.Prefix != "/" {
filtered := make([]*filer_pb.Entry, 0)
for _, e := range entries {
@@ -56,7 +55,16 @@ func (c *testFilerClient) ListEntries(ctx context.Context, in *filer_pb.ListEntr
entries = filtered
}
- // Respect Limit
+ if in.StartFromFileName != "" {
+ filtered := make([]*filer_pb.Entry, 0)
+ for _, e := range entries {
+ if e.Name > in.StartFromFileName || (in.InclusiveStartFrom && e.Name == in.StartFromFileName) {
+ filtered = append(filtered, e)
+ }
+ }
+ entries = filtered
+ }
+
if in.Limit > 0 && int(in.Limit) < len(entries) {
entries = entries[:in.Limit]
}
@@ -300,7 +308,7 @@ func TestDoListFilerEntries_BucketRootPrefixSlashDelimiterSlash_ListsDirectories
cursor := &ListingCursor{maxKeys: 1000}
seen := make([]string, 0)
- _, err := s3a.doListFilerEntries(client, "/buckets/test-bucket", "/", cursor, "", "/", false, "test-bucket", func(dir string, entry *filer_pb.Entry) {
+ _, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/test-bucket", prefix: "/", delimiter: "/", bucket: "test-bucket"}, cursor, func(dir string, entry *filer_pb.Entry) {
if entry.IsDirectory {
seen = append(seen, entry.Name)
}
@@ -322,7 +330,7 @@ func TestDoListFilerEntries_ExclusiveStartSkipsMarkerEcho(t *testing.T) {
cursor := &ListingCursor{maxKeys: 1000}
var seen []string
- nextMarker, err := s3a.doListFilerEntries(client, "/buckets/test-bucket", "", cursor, "test.txt", "", false, "test-bucket", func(dir string, entry *filer_pb.Entry) {
+ nextMarker, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/test-bucket", marker: "test.txt", bucket: "test-bucket"}, cursor, func(dir string, entry *filer_pb.Entry) {
seen = append(seen, entry.Name)
})
@@ -346,7 +354,7 @@ func TestDoListFilerEntries_ExclusiveStartSkipsMarkerEchoWithSubsequentEntries(t
cursor := &ListingCursor{maxKeys: 1000}
var seen []string
- nextMarker, err := s3a.doListFilerEntries(client, "/buckets/test-bucket", "", cursor, "test.txt", "", false, "test-bucket", func(dir string, entry *filer_pb.Entry) {
+ nextMarker, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/test-bucket", marker: "test.txt", bucket: "test-bucket"}, cursor, func(dir string, entry *filer_pb.Entry) {
seen = append(seen, entry.Name)
})
@@ -355,6 +363,116 @@ func TestDoListFilerEntries_ExclusiveStartSkipsMarkerEchoWithSubsequentEntries(t
assert.Equal(t, "zebra.txt", nextMarker)
}
+func TestDoListFilerEntries_EmptyDirectoriesDoNotHideLaterEntries(t *testing.T) {
+ // With maxKeys=1 the listing window is 3 entries. The first window holds only
+ // empty directories, so the real object further down must still be found.
+ s3a := &S3ApiServer{}
+ client := &testFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/buckets/test-bucket": {
+ {Name: "00-empty-1", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "00-empty-2", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "00-empty-3", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "00-empty-4", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "00-empty-5", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "zz-real", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ },
+ "/buckets/test-bucket/zz-real": {
+ {Name: "file.log", Attributes: &filer_pb.FuseAttributes{}},
+ },
+ },
+ }
+
+ cursor := &ListingCursor{maxKeys: 1}
+ var seen []string
+ _, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/test-bucket", bucket: "test-bucket"}, cursor, func(dir string, entry *filer_pb.Entry) {
+ seen = append(seen, dir+"/"+entry.Name)
+ cursor.maxKeys--
+ })
+
+ assert.NoError(t, err)
+ assert.Equal(t, []string{"/buckets/test-bucket/zz-real/file.log"}, seen)
+ assert.False(t, cursor.isTruncated)
+}
+
+func TestDoListFilerEntries_EmptyDirectoriesInSubdirectoryDoNotHideSiblings(t *testing.T) {
+ // The recursion into "logs" gets its own listing window; empty subdirectories
+ // filling that window must not skip the object behind them.
+ s3a := &S3ApiServer{}
+ client := &testFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/buckets/test-bucket": {
+ {Name: "logs", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ },
+ "/buckets/test-bucket/logs": {
+ {Name: "00-empty-1", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "00-empty-2", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "00-empty-3", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "00-empty-4", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "00-empty-5", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "zz.log", Attributes: &filer_pb.FuseAttributes{}},
+ },
+ },
+ }
+
+ cursor := &ListingCursor{maxKeys: 1}
+ var seen []string
+ _, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/test-bucket", bucket: "test-bucket"}, cursor, func(dir string, entry *filer_pb.Entry) {
+ seen = append(seen, dir+"/"+entry.Name)
+ cursor.maxKeys--
+ })
+
+ assert.NoError(t, err)
+ assert.Equal(t, []string{"/buckets/test-bucket/logs/zz.log"}, seen)
+ assert.False(t, cursor.isTruncated)
+}
+
+func TestDoListFilerEntries_TruncationAcrossEmptyDirectories(t *testing.T) {
+ // Two real objects separated by empty directories: the first fills the quota
+ // and the listing must report truncation with a resumable marker.
+ s3a := &S3ApiServer{}
+ client := &testFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/buckets/test-bucket": {
+ {Name: "00-empty-1", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "00-empty-2", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "00-empty-3", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "mm-real", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "pp-empty", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ {Name: "zz-real", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
+ },
+ "/buckets/test-bucket/mm-real": {
+ {Name: "file.log", Attributes: &filer_pb.FuseAttributes{}},
+ },
+ "/buckets/test-bucket/zz-real": {
+ {Name: "file.log", Attributes: &filer_pb.FuseAttributes{}},
+ },
+ },
+ }
+
+ cursor := &ListingCursor{maxKeys: 1}
+ var seen []string
+ nextMarker, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/test-bucket", bucket: "test-bucket"}, cursor, func(dir string, entry *filer_pb.Entry) {
+ seen = append(seen, dir+"/"+entry.Name)
+ cursor.maxKeys--
+ })
+
+ assert.NoError(t, err)
+ assert.Equal(t, []string{"/buckets/test-bucket/mm-real/file.log"}, seen)
+ assert.True(t, cursor.isTruncated)
+
+ cursor = &ListingCursor{maxKeys: 1}
+ seen = nil
+ _, err = s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/test-bucket", marker: nextMarker, bucket: "test-bucket"}, cursor, func(dir string, entry *filer_pb.Entry) {
+ seen = append(seen, dir+"/"+entry.Name)
+ cursor.maxKeys--
+ })
+
+ assert.NoError(t, err)
+ assert.Equal(t, []string{"/buckets/test-bucket/zz-real/file.log"}, seen)
+ assert.False(t, cursor.isTruncated)
+}
+
func TestAllowUnorderedWithDelimiterValidation(t *testing.T) {
t.Run("should return error when allow-unordered=true and delimiter are both present", func(t *testing.T) {
// Create a request with both allow-unordered=true and delimiter
@@ -765,7 +883,7 @@ func TestListObjectsV2_Regression(t *testing.T) {
// Call doListFilerEntries directly to unit test listing logic in isolation,
// simulating parameters passed from listFilerEntries for prefix "reports/".
- _, err := s3a.doListFilerEntries(client, "/buckets/reports", "reports", cursor, "", "", false, "reports", func(dir string, entry *filer_pb.Entry) {
+ _, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/reports", prefix: "reports", bucket: "reports"}, cursor, func(dir string, entry *filer_pb.Entry) {
if !entry.IsDirectory {
results = append(results, entry.Name)
}
@@ -803,7 +921,7 @@ func TestListObjectsV2_Regression_Sorting(t *testing.T) {
// Without the fix, Limit=1 would cause the lister to stop after "reports-archive",
// missing the intended "reports" directory.
- _, err := s3a.doListFilerEntries(client, "/buckets/reports", "reports", cursor, "", "", false, "reports", func(dir string, entry *filer_pb.Entry) {
+ _, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/reports", prefix: "reports", bucket: "reports"}, cursor, func(dir string, entry *filer_pb.Entry) {
if !entry.IsDirectory {
results = append(results, entry.Name)
}
@@ -846,7 +964,7 @@ func TestListObjectsV2_PrefixEndingWithSlash_DoesNotMatchSiblings(t *testing.T)
cursor := &ListingCursor{maxKeys: 1000, prefixEndsOnDelimiter: true}
var results []string
- _, err := s3a.doListFilerEntries(client, "/buckets/bucket/path/to/list", "1", cursor, "", "", false, "bucket", func(dir string, entry *filer_pb.Entry) {
+ _, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/bucket/path/to/list", prefix: "1", bucket: "bucket"}, cursor, func(dir string, entry *filer_pb.Entry) {
if !entry.IsDirectory {
results = append(results, entry.Name)
}
@@ -878,7 +996,7 @@ func TestListObjectsV2_PrefixEndingWithSlash_WithDelimiter(t *testing.T) {
cursor := &ListingCursor{maxKeys: 1000, prefixEndsOnDelimiter: true}
var results []string
- _, err := s3a.doListFilerEntries(client, "/buckets/bucket/path/to/list", "1", cursor, "", "/", false, "bucket", func(dir string, entry *filer_pb.Entry) {
+ _, err := s3a.doListFilerEntries(context.Background(), client, listDirectoryRequest{dir: "/buckets/bucket/path/to/list", prefix: "1", delimiter: "/", bucket: "bucket"}, cursor, func(dir string, entry *filer_pb.Entry) {
results = append(results, entry.Name)
})
diff --git a/weed/s3api/s3api_object_handlers_list_versioned_test.go b/weed/s3api/s3api_object_handlers_list_versioned_test.go
index fd2695e03..2bcbfa269 100644
--- a/weed/s3api/s3api_object_handlers_list_versioned_test.go
+++ b/weed/s3api/s3api_object_handlers_list_versioned_test.go
@@ -136,7 +136,7 @@ func TestListObjectsWithVersionedObjects(t *testing.T) {
commonPrefixes := []PrefixEntry{}
bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, tt.bucket)
- _, err := s3a.doListFilerEntries(filerClient, bucketPrefix[:len(bucketPrefix)-1], tt.prefix, cursor, "", tt.delimiter, false, tt.bucket, func(dir string, entry *filer_pb.Entry) {
+ _, err := s3a.doListFilerEntries(context.Background(), filerClient, listDirectoryRequest{dir: bucketPrefix[:len(bucketPrefix)-1], prefix: tt.prefix, delimiter: tt.delimiter, bucket: tt.bucket}, cursor, func(dir string, entry *filer_pb.Entry) {
if cursor.maxKeys <= 0 {
return
}
@@ -237,7 +237,7 @@ func TestVersionedObjectsNoDuplication(t *testing.T) {
cursor := &ListingCursor{maxKeys: uint16(1000)}
contents := []ListEntry{}
- _, err := s3a.doListFilerEntries(filerClient, "/buckets/test-bucket", "", cursor, "", "", false, "test-bucket", func(dir string, entry *filer_pb.Entry) {
+ _, err := s3a.doListFilerEntries(context.Background(), filerClient, listDirectoryRequest{dir: "/buckets/test-bucket", bucket: "test-bucket"}, cursor, func(dir string, entry *filer_pb.Entry) {
if cursor.maxKeys <= 0 {
return
}
@@ -295,7 +295,7 @@ func TestVersionedObjectsWithDeleteMarker(t *testing.T) {
cursor := &ListingCursor{maxKeys: uint16(1000)}
contents := []ListEntry{}
- _, err := s3a.doListFilerEntries(filerClient, "/buckets/test-bucket", "", cursor, "", "", false, "test-bucket", func(dir string, entry *filer_pb.Entry) {
+ _, err := s3a.doListFilerEntries(context.Background(), filerClient, listDirectoryRequest{dir: "/buckets/test-bucket", bucket: "test-bucket"}, cursor, func(dir string, entry *filer_pb.Entry) {
if cursor.maxKeys <= 0 {
return
}
@@ -344,7 +344,7 @@ func TestVersionedObjectsMaxKeys(t *testing.T) {
cursor := &ListingCursor{maxKeys: uint16(3)}
contents := []ListEntry{}
- _, err := s3a.doListFilerEntries(filerClient, "/buckets/test-bucket", "", cursor, "", "", false, "test-bucket", func(dir string, entry *filer_pb.Entry) {
+ _, err := s3a.doListFilerEntries(context.Background(), filerClient, listDirectoryRequest{dir: "/buckets/test-bucket", bucket: "test-bucket"}, cursor, func(dir string, entry *filer_pb.Entry) {
if cursor.maxKeys <= 0 {
return
}
@@ -406,7 +406,7 @@ func TestVersionsDirectoryNotTraversed(t *testing.T) {
cursor := &ListingCursor{maxKeys: uint16(1000)}
contents := []ListEntry{}
- _, err := s3a.doListFilerEntries(customClient, "/buckets/test-bucket", "", cursor, "", "", false, "test-bucket", func(dir string, entry *filer_pb.Entry) {
+ _, err := s3a.doListFilerEntries(context.Background(), customClient, listDirectoryRequest{dir: "/buckets/test-bucket", bucket: "test-bucket"}, cursor, func(dir string, entry *filer_pb.Entry) {
if cursor.maxKeys <= 0 {
return
}