mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-16 20:26:45 +00:00
Chris LuandGitHub
12627d376d
* mount: size the direct listing slice from the batch, not the offset The limit passed here is skipCount+batchSize when a client resumes a fresh handle partway through a directory, so preallocating for it turns the client's cookie into an allocation: a readdir at offset 3,000,000 reserves 24MB before the first entry arrives, and an offset near the uint32 ceiling asks makeslice for ~4.29e9 elements. * mount: stop replaying a directory that shrank past the resume offset A client that opens a fresh handle and resumes at a cookie from an earlier, larger listing gets a preload that cannot reach the entry before that offset. The resume name was then left empty and the follow-up batch listed from the directory's first child again, so the client was handed every name a second time. The stream always runs from the first child, so failing to reach that entry means the directory is simply shorter than the offset. That is the end of it. * mount: page a directory from where the store reached The batch loader treated a short batch as the end of the directory, but the meta cache drops an expired child after the store has already spent it against the limit, so a batch that filled up could still deliver fewer entries than asked for. A directory with a handful of expired children would stop listing early and hide every child behind them; a whole batch of expired ones truncated the listing to nothing. ListDirectoryEntries now reports the name the store itself reached. That is both the sound end-of-directory signal -- the store returning nothing -- and the right cursor, since resuming from the last visible name would re-read the dropped children every round and never get past a batch that was entirely expired. * mount: drop the entries a directory walk has already passed entryStreamOffset was only ever written by reset, so the stream a handle holds grew for the life of the walk and a directory was retained whole even though nothing could read the entries behind the client's position again. A 10M-entry walk parked millions of entries per handle, and NFS-Ganesha opens several on the same directory. Offsets index into the stream from entryStreamOffset, so advancing the two together keeps them lined up. One entry is held back because the next batch resumes from the name immediately before the offset. Seeking back behind what is still held now restarts the directory, which is what the offset scheme can honestly support -- it previously returned nothing.
56 lines
1.7 KiB
Go
56 lines
1.7 KiB
Go
package mount
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
// TestListDirectoryEntriesOmitsChunks covers the wiring the readdir speedup
|
|
// rests on: that the context marker actually reaches the store's decode. Every
|
|
// other test exercises the decoder directly, so a refactor that stopped
|
|
// threading the context would revert the optimisation silently.
|
|
func TestListDirectoryEntriesOmitsChunks(t *testing.T) {
|
|
dir := util.FullPath("/images")
|
|
wfs := newBenchWFS(t, dir, 4)
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
ctx context.Context
|
|
wantChunks bool
|
|
}{
|
|
{"plain listing keeps chunks", context.Background(), true},
|
|
{"marked listing drops chunks", filer_pb.WithChunksOmitted(context.Background()), false},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
var seen int
|
|
_, err := wfs.metaCache.ListDirectoryEntries(tc.ctx, dir, "", false, 100, func(entry *filer.Entry) (bool, error) {
|
|
seen++
|
|
if got := len(entry.Chunks) > 0; got != tc.wantChunks {
|
|
t.Errorf("%s: has chunks = %v, want %v", entry.Name(), got, tc.wantChunks)
|
|
}
|
|
// The size has to survive either way, since that is what the
|
|
// readdir reports.
|
|
if entry.FileSize != 4<<20 {
|
|
t.Errorf("%s: FileSize = %d, want %d", entry.Name(), entry.FileSize, 4<<20)
|
|
}
|
|
return true, nil
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list: %v", err)
|
|
}
|
|
if seen != 4 {
|
|
t.Fatalf("listed %d entries, want 4", seen)
|
|
}
|
|
})
|
|
}
|
|
|
|
// readdirContext is what weedfs_dir_read.go actually passes.
|
|
if !filer_pb.ChunksOmitted(readdirContext) {
|
|
t.Error("readdirContext does not carry the chunks-omitted marker")
|
|
}
|
|
}
|