Files
seaweedfs/weed/mount/meta_cache/meta_cache_build_test.go
T
5456f9d695 mount: confirm an empty directory rebuild before caching it (#10092)
A directory rebuild wiped the cached children, listed the filer once, and
published the directory authoritatively cached over whatever came back. A
transient empty listing -- a momentary list-stream glitch that ends as a
clean EOF with no entries -- then stranded a populated directory cached
over an empty store, hiding every file in it until some unrelated event
happened to rebuild it: stat returns ENOENT and readdir returns nothing
though the files are safe on the filer, and nothing re-triggers a build.

Re-read the directory when the listing comes back empty before trusting
it. The first re-read is immediate, since the likely transient clears on a
fresh stream; later attempts space out. A genuinely empty directory still
lists empty every time and caches as before, so only empty listings pay
the extra read.

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-24 14:25:23 -07:00

636 lines
21 KiB
Go

package meta_cache
import (
"context"
"fmt"
"io"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
type buildListStream struct {
responses []*filer_pb.ListEntriesResponse
onFirstRecv func()
once sync.Once
index int
}
func (s *buildListStream) Recv() (*filer_pb.ListEntriesResponse, error) {
s.once.Do(func() {
if s.onFirstRecv != nil {
s.onFirstRecv()
}
})
if s.index >= len(s.responses) {
return nil, io.EOF
}
resp := s.responses[s.index]
s.index++
return resp, nil
}
func (s *buildListStream) Header() (metadata.MD, error) { return metadata.MD{}, nil }
func (s *buildListStream) Trailer() metadata.MD { return metadata.MD{} }
func (s *buildListStream) CloseSend() error { return nil }
func (s *buildListStream) Context() context.Context { return context.Background() }
func (s *buildListStream) SendMsg(any) error { return nil }
func (s *buildListStream) RecvMsg(any) error { return nil }
type buildListClient struct {
filer_pb.SeaweedFilerClient
responses []*filer_pb.ListEntriesResponse
onFirstRecv func()
}
func (c *buildListClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
return &buildListStream{
responses: c.responses,
onFirstRecv: c.onFirstRecv,
}, nil
}
type buildFilerAccessor struct {
client filer_pb.SeaweedFilerClient
}
func (a *buildFilerAccessor) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
return fn(a.client)
}
func (a *buildFilerAccessor) AdjustedUrl(*filer_pb.Location) string { return "" }
func (a *buildFilerAccessor) GetDataCenter() string { return "" }
func TestEnsureVisitedReplaysBufferedEventsAfterSnapshot(t *testing.T) {
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
"/": true,
})
defer mc.Shutdown()
var applyErr error
accessor := &buildFilerAccessor{
client: &buildListClient{
responses: []*filer_pb.ListEntriesResponse{
{
Entry: &filer_pb.Entry{
Name: "base.txt",
Attributes: &filer_pb.FuseAttributes{
Crtime: 1,
Mtime: 1,
FileMode: 0100644,
FileSize: 3,
},
},
SnapshotTsNs: 100,
},
},
onFirstRecv: func() {
applyErr = mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
Directory: "/dir",
EventNotification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: "after.txt",
Attributes: &filer_pb.FuseAttributes{
Crtime: 2,
Mtime: 2,
FileMode: 0100644,
FileSize: 9,
},
},
},
TsNs: 101,
}, SubscriberMetadataResponseApplyOptions)
},
},
}
if err := EnsureVisited(mc, accessor, util.FullPath("/dir")); err != nil {
t.Fatalf("ensure visited: %v", err)
}
if applyErr != nil {
t.Fatalf("apply buffered event: %v", applyErr)
}
if !mc.IsDirectoryCached(util.FullPath("/dir")) {
t.Fatal("directory /dir should be cached after build completes")
}
baseEntry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/base.txt"))
if err != nil {
t.Fatalf("find base entry: %v", err)
}
if baseEntry.FileSize != 3 {
t.Fatalf("base entry size = %d, want 3", baseEntry.FileSize)
}
afterEntry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/after.txt"))
if err != nil {
t.Fatalf("find replayed entry: %v", err)
}
if afterEntry.FileSize != 9 {
t.Fatalf("replayed entry size = %d, want 9", afterEntry.FileSize)
}
}
// TestDirectoryNotificationsSuppressedDuringBuild verifies that metadata events
// targeting a directory under active build do NOT fire onDirectoryUpdate for
// that directory. In production, onDirectoryUpdate can trigger
// markDirectoryReadThrough → DeleteFolderChildren, which would wipe entries
// that EnsureVisited already inserted mid-build.
func TestDirectoryNotificationsSuppressedDuringBuild(t *testing.T) {
mc, _, notifications, _ := newTestMetaCache(t, map[util.FullPath]bool{
"/": true,
})
defer mc.Shutdown()
// Start building /dir (simulates the beginning of EnsureVisited)
if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
t.Fatalf("begin build: %v", err)
}
// Insert an entry as EnsureVisited would during the filer listing
if err := mc.InsertEntry(context.Background(), &filer.Entry{
FullPath: "/dir/existing.txt",
Attr: filer.Attr{
Crtime: time.Unix(1, 0),
Mtime: time.Unix(1, 0),
Mode: 0100644,
FileSize: 100,
},
}); err != nil {
t.Fatalf("insert entry during build: %v", err)
}
// Simulate multiple metadata events arriving for /dir while the build
// is in progress. Each event would normally call noteDirectoryUpdate,
// which in production can trigger markDirectoryReadThrough and wipe entries.
for i := 0; i < 5; i++ {
resp := &filer_pb.SubscribeMetadataResponse{
Directory: "/dir",
EventNotification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: fmt.Sprintf("new-%d.txt", i),
Attributes: &filer_pb.FuseAttributes{
Crtime: int64(10 + i),
Mtime: int64(10 + i),
FileMode: 0100644,
FileSize: uint64(i + 1),
},
},
},
TsNs: int64(200 + i),
}
if err := mc.ApplyMetadataResponse(context.Background(), resp, SubscriberMetadataResponseApplyOptions); err != nil {
t.Fatalf("apply event %d: %v", i, err)
}
}
// The building directory /dir must NOT have received any notifications.
// If it did, markDirectoryReadThrough would wipe the cache mid-build.
for _, p := range notifications.paths() {
if p == util.FullPath("/dir") {
t.Fatal("onDirectoryUpdate was called for /dir during build; this would cause markDirectoryReadThrough to wipe entries mid-build")
}
}
// The entry inserted during the build must still be present
entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/existing.txt"))
if err != nil {
t.Fatalf("entry wiped during build: %v", err)
}
if entry.FileSize != 100 {
t.Fatalf("entry size = %d, want 100", entry.FileSize)
}
// Complete the build — buffered events should be replayed
if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 150); err != nil {
t.Fatalf("complete build: %v", err)
}
// After build completes, the entry from the listing should still exist
entry, err = mc.FindEntry(context.Background(), util.FullPath("/dir/existing.txt"))
if err != nil {
t.Fatalf("entry lost after build completion: %v", err)
}
if entry.FileSize != 100 {
t.Fatalf("entry size after build = %d, want 100", entry.FileSize)
}
// Buffered events with TsNs > snapshotTsNs (150) should have been replayed
for i := 0; i < 5; i++ {
name := fmt.Sprintf("new-%d.txt", i)
e, err := mc.FindEntry(context.Background(), util.FullPath("/dir/"+name))
if err != nil {
t.Fatalf("replayed entry %s not found: %v", name, err)
}
if e.FileSize != uint64(i+1) {
t.Fatalf("replayed entry %s size = %d, want %d", name, e.FileSize, i+1)
}
}
}
// TestEmptyDirectoryBuildReplaysAllBufferedEvents verifies that when a
// directory build completes with snapshotTsNs=0 (empty directory — server
// returned no entries and no snapshot), ALL buffered events are replayed
// without any TsNs filtering. This prevents clock-skew between client and
// filer from dropping legitimate mutations.
func TestEmptyDirectoryBuildReplaysAllBufferedEvents(t *testing.T) {
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
"/": true,
})
defer mc.Shutdown()
if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/empty")); err != nil {
t.Fatalf("begin build: %v", err)
}
// Buffer events with a range of TsNs values — some very old, some recent.
// With a client-synthesized snapshot, old events could be incorrectly filtered.
tsValues := []int64{1, 50, 500, 5000, 50000}
for i, ts := range tsValues {
resp := &filer_pb.SubscribeMetadataResponse{
Directory: "/empty",
EventNotification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: fmt.Sprintf("file-%d.txt", i),
Attributes: &filer_pb.FuseAttributes{
Crtime: ts,
Mtime: ts,
FileMode: 0100644,
FileSize: uint64(i + 10),
},
},
},
TsNs: ts,
}
if err := mc.ApplyMetadataResponse(context.Background(), resp, SubscriberMetadataResponseApplyOptions); err != nil {
t.Fatalf("apply event %d: %v", i, err)
}
}
// Complete with snapshotTsNs=0 — simulates empty directory listing
if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/empty"), 0); err != nil {
t.Fatalf("complete build: %v", err)
}
// Every buffered event must have been replayed, regardless of TsNs
for i := range tsValues {
name := fmt.Sprintf("file-%d.txt", i)
e, err := mc.FindEntry(context.Background(), util.FullPath("/empty/"+name))
if err != nil {
t.Fatalf("replayed entry %s not found: %v", name, err)
}
if e.FileSize != uint64(i+10) {
t.Fatalf("replayed entry %s size = %d, want %d", name, e.FileSize, i+10)
}
}
if !mc.IsDirectoryCached(util.FullPath("/empty")) {
t.Fatal("/empty should be marked cached after build completes")
}
}
// TestBuildCompletionSurvivesCallerCancellation verifies that once
// CompleteDirectoryBuild is enqueued, a cancelled caller context does not
// prevent the build from completing. The apply loop uses context.Background()
// internally, so the operation finishes even if the caller gives up waiting.
func TestBuildCompletionSurvivesCallerCancellation(t *testing.T) {
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
"/": true,
})
defer mc.Shutdown()
if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
t.Fatalf("begin build: %v", err)
}
// Insert an entry during the build (as EnsureVisited would)
if err := mc.InsertEntry(context.Background(), &filer.Entry{
FullPath: "/dir/kept.txt",
Attr: filer.Attr{
Crtime: time.Unix(1, 0),
Mtime: time.Unix(1, 0),
Mode: 0100644,
FileSize: 42,
},
}); err != nil {
t.Fatalf("insert entry: %v", err)
}
// Buffer an event that should be replayed
if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
Directory: "/dir",
EventNotification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: "buffered.txt",
Attributes: &filer_pb.FuseAttributes{
Crtime: 5,
Mtime: 5,
FileMode: 0100644,
FileSize: 77,
},
},
},
TsNs: 200,
}, SubscriberMetadataResponseApplyOptions); err != nil {
t.Fatalf("apply event: %v", err)
}
// Complete with an already-cancelled context. The operation should still
// succeed because enqueueAndWait sets req.ctx = context.Background().
cancelledCtx, cancel := context.WithCancel(context.Background())
cancel() // cancel immediately
// CompleteDirectoryBuild may return ctx.Err() if the select picks
// ctx.Done() first, but the operation itself still completes in the
// apply loop. Poll for the observable side effect instead of using
// a fixed sleep.
_ = mc.CompleteDirectoryBuild(cancelledCtx, util.FullPath("/dir"), 100)
// Poll until the build completes or a deadline elapses.
deadline := time.After(2 * time.Second)
for !mc.IsDirectoryCached(util.FullPath("/dir")) {
select {
case <-deadline:
t.Fatal("/dir should be cached — CompleteDirectoryBuild must have executed despite cancelled context")
default:
time.Sleep(5 * time.Millisecond)
}
}
// The pre-existing entry must survive
entry, findErr := mc.FindEntry(context.Background(), util.FullPath("/dir/kept.txt"))
if findErr != nil {
t.Fatalf("find kept entry: %v", findErr)
}
if entry.FileSize != 42 {
t.Fatalf("kept entry size = %d, want 42", entry.FileSize)
}
// The buffered event (TsNs 200 > snapshot 100) must have been replayed
buffered, findErr := mc.FindEntry(context.Background(), util.FullPath("/dir/buffered.txt"))
if findErr != nil {
t.Fatalf("find buffered entry: %v", findErr)
}
if buffered.FileSize != 77 {
t.Fatalf("buffered entry size = %d, want 77", buffered.FileSize)
}
}
func TestBufferedRenameUpdatesOtherDirectoryBeforeBuildCompletes(t *testing.T) {
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
"/": true,
"/src": true,
})
defer mc.Shutdown()
if err := mc.InsertEntry(context.Background(), &filer.Entry{
FullPath: "/src/from.txt",
Attr: filer.Attr{
Crtime: time.Unix(1, 0),
Mtime: time.Unix(1, 0),
Mode: 0100644,
FileSize: 7,
},
}); err != nil {
t.Fatalf("insert source entry: %v", err)
}
if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dst")); err != nil {
t.Fatalf("begin build: %v", err)
}
renameResp := &filer_pb.SubscribeMetadataResponse{
Directory: "/src",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "from.txt",
},
NewEntry: &filer_pb.Entry{
Name: "to.txt",
Attributes: &filer_pb.FuseAttributes{
Crtime: 2,
Mtime: 2,
FileMode: 0100644,
FileSize: 12,
},
},
NewParentPath: "/dst",
},
TsNs: 101,
}
if err := mc.ApplyMetadataResponse(context.Background(), renameResp, SubscriberMetadataResponseApplyOptions); err != nil {
t.Fatalf("apply rename: %v", err)
}
oldEntry, err := mc.FindEntry(context.Background(), util.FullPath("/src/from.txt"))
if err != filer_pb.ErrNotFound {
t.Fatalf("find old path error = %v, want %v", err, filer_pb.ErrNotFound)
}
if oldEntry != nil {
t.Fatalf("old path should be removed before build completes: %+v", oldEntry)
}
newEntry, err := mc.FindEntry(context.Background(), util.FullPath("/dst/to.txt"))
if err != filer_pb.ErrNotFound {
t.Fatalf("find buffered new path error = %v, want %v", err, filer_pb.ErrNotFound)
}
if newEntry != nil {
t.Fatalf("new path should stay hidden until build completes: %+v", newEntry)
}
if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dst"), 100); err != nil {
t.Fatalf("complete build: %v", err)
}
newEntry, err = mc.FindEntry(context.Background(), util.FullPath("/dst/to.txt"))
if err != nil {
t.Fatalf("find replayed new path: %v", err)
}
if newEntry.FileSize != 12 {
t.Fatalf("replayed new path size = %d, want 12", newEntry.FileSize)
}
}
// TestEnsureVisitedPreservesLocalOnlyEntry reproduces the residual coherence
// gap behind the FUSE ConcurrentReadWrite ENOENT flake.
//
// A FUSE create on the writeback/deferFilerCreate path inserts the entry into
// the local store directly (weedfs_file_mkrm.go createFile), off the metaCache
// apply loop, before the filer holds it. A concurrent rebuild of the parent —
// triggered when the directory falls out of cache (idle evict, hot-dir
// read-through) — wipes the store and refills it from a filer listing that does
// not yet include the un-flushed local create, then publishes the directory
// authoritatively cached (markCachedFn). The local entry vanishes although the
// client created it: lookupEntry then returns an authoritative ENOENT for it.
func TestEnsureVisitedPreservesLocalOnlyEntry(t *testing.T) {
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
defer mc.Shutdown()
// The mount pins the un-flushed create (open dirty handle / pending flush),
// keyed off the inode the entry carries so a kernel Forget that dropped the
// path→inode mapping cannot unpin an in-flight create.
mc.SetPinnedChildFn(func(e *filer.Entry) bool { return e.Attr.Inode == 42 })
// A deferred local create lands before the rebuild; /dir is not yet cached.
// It carries its allocated inode, as createFile's placeholder does.
if err := mc.InsertEntry(context.Background(), &filer.Entry{
FullPath: "/dir/pending.txt",
Attr: filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1, Inode: 42},
}); err != nil {
t.Fatalf("insert pending entry: %v", err)
}
// A concurrent rebuild lists the filer, whose snapshot pre-dates the
// un-flushed create, so it returns only the already-persisted sibling.
accessor := &buildFilerAccessor{client: &buildListClient{
responses: []*filer_pb.ListEntriesResponse{{
Entry: &filer_pb.Entry{
Name: "base.txt",
Attributes: &filer_pb.FuseAttributes{
Crtime: 1,
Mtime: 1,
FileMode: 0100644,
FileSize: 3,
},
},
SnapshotTsNs: 100,
}},
}}
if err := EnsureVisited(mc, accessor, util.FullPath("/dir")); err != nil {
t.Fatalf("ensure visited: %v", err)
}
if !mc.IsDirectoryCached(util.FullPath("/dir")) {
t.Fatal("/dir should be cached after build completes")
}
// base.txt from the listing is present.
if _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/base.txt")); err != nil {
t.Fatalf("listed entry missing after build: %v", err)
}
// The un-flushed local create must survive the rebuild. With /dir now
// authoritatively cached, losing it is the file-vanishes flake.
if _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/pending.txt")); err != nil {
t.Fatalf("local-only entry lost across concurrent rebuild: %v", err)
}
}
// TestEnsureVisitedDropsUnpinnedStaleEntry guards the preservation's selectivity:
// a cached child the filer listing no longer returns and that is NOT pinned must
// still be wiped, so the rebuild can't resurrect a deleted/renamed entry.
func TestEnsureVisitedDropsUnpinnedStaleEntry(t *testing.T) {
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
defer mc.Shutdown()
mc.SetPinnedChildFn(func(*filer.Entry) bool { return false })
// A stale child sits in the cache; the filer no longer has it.
insertCacheEntry(t, mc, "/dir/stale.txt")
accessor := &buildFilerAccessor{client: &buildListClient{
responses: []*filer_pb.ListEntriesResponse{{
Entry: &filer_pb.Entry{
Name: "base.txt",
Attributes: &filer_pb.FuseAttributes{
Crtime: 1,
Mtime: 1,
FileMode: 0100644,
FileSize: 3,
},
},
SnapshotTsNs: 100,
}},
}}
if err := EnsureVisited(mc, accessor, util.FullPath("/dir")); err != nil {
t.Fatalf("ensure visited: %v", err)
}
if entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/stale.txt")); err != filer_pb.ErrNotFound || entry != nil {
t.Fatalf("unpinned stale entry survived rebuild = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
}
}
// sequencedListClient returns a different ListEntries result per call (repeating
// the last), modelling a filer that lists empty transiently then the real entries.
type sequencedListClient struct {
filer_pb.SeaweedFilerClient
mu sync.Mutex
perCall [][]*filer_pb.ListEntriesResponse
calls int
}
func (c *sequencedListClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
c.mu.Lock()
idx := c.calls
if idx >= len(c.perCall) {
idx = len(c.perCall) - 1
}
c.calls++
resp := c.perCall[idx]
c.mu.Unlock()
return &buildListStream{responses: resp}, nil
}
// TestEnsureVisitedConfirmsTransientEmptyListing: a rebuild whose first filer
// listing comes back empty must re-read and cache the real entries, not strand
// the directory cached over an empty store (the ConcurrentReadWrite ENOENT flake).
func TestEnsureVisitedConfirmsTransientEmptyListing(t *testing.T) {
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
defer mc.Shutdown()
entry := &filer_pb.Entry{
Name: "keep.txt",
Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 7},
}
accessor := &buildFilerAccessor{client: &sequencedListClient{
perCall: [][]*filer_pb.ListEntriesResponse{
{}, // first read: transient empty
{{Entry: entry, SnapshotTsNs: 100}}, // confirm read: the real entry
},
}}
if err := EnsureVisited(mc, accessor, util.FullPath("/dir")); err != nil {
t.Fatalf("ensure visited: %v", err)
}
if !mc.IsDirectoryCached(util.FullPath("/dir")) {
t.Fatal("/dir should be cached after build completes")
}
if _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/keep.txt")); err != nil {
t.Fatalf("/dir/keep.txt stranded after transient empty listing: %v", err)
}
}
// TestEnsureVisitedCachesGenuinelyEmptyDirectory: a really-empty directory lists
// empty on every confirm and must still end up cached.
func TestEnsureVisitedCachesGenuinelyEmptyDirectory(t *testing.T) {
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
defer mc.Shutdown()
client := &sequencedListClient{
perCall: [][]*filer_pb.ListEntriesResponse{{}}, // always empty
}
accessor := &buildFilerAccessor{client: client}
if err := EnsureVisited(mc, accessor, util.FullPath("/empty")); err != nil {
t.Fatalf("ensure visited: %v", err)
}
if !mc.IsDirectoryCached(util.FullPath("/empty")) {
t.Fatal("/empty should be cached even though it has no entries")
}
// The empty result must have been confirmed, not trusted on the first read.
client.mu.Lock()
calls := client.calls
client.mu.Unlock()
if calls != emptyRebuildConfirmations+1 {
t.Fatalf("list calls = %d, want %d (initial + confirmations)", calls, emptyRebuildConfirmations+1)
}
}