mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-07-20 23:12:36 +00:00
Jaehoon Kim
18868e5204
* fix(mount): run entry invalidations off the meta-cache apply loop The apply loop ran invalidateFunc inline, which acquires the open file handle's lock in fhLockTable. Meanwhile flushMetadataToFiler holds that same fh lock and then waits on the apply loop (applyLocalMetadataEvent). When both target the same open file concurrently, the loop blocks on the fh lock while the lock holder blocks on the loop: an ABBA deadlock that backs up every later readdir/flush and hangs the mount. Fix: dispatch entry invalidations to a dedicated FIFO worker goroutine so the apply loop never blocks on locks held by goroutines waiting on it. Adds a regression test reproducing the interleaving. Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> * perf(mount): update invalidate counter once per batch Run the batch's invalidateFunc calls without re-taking invalidateMu per item, then bump invalidateProcessed and broadcast once after the loop. WaitForEntryInvalidations only needs the count to reach its target and a batch always completes together, so the per-item lock + broadcast was wasted work. Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> * mount: extract the invalidate worker into util.AsyncBatchWorker The apply loop's off-thread entry-invalidation queue was a one-off mutex + cond + slice + counters living inside MetaCache. Pull it out as a generic unbounded FIFO worker so the deadlock-avoidance contract (never block the producer, drain on shutdown, wait-for-quiesce) lives in one place. --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Chris Lu <chris.lu@gmail.com>
476 lines
13 KiB
Go
476 lines
13 KiB
Go
package meta_cache
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
func TestApplyMetadataResponseAppliesEventsInOrder(t *testing.T) {
|
|
mc, _, notifications, invalidations := newTestMetaCache(t, map[util.FullPath]bool{
|
|
"/": true,
|
|
"/dir": true,
|
|
})
|
|
defer mc.Shutdown()
|
|
|
|
createResp := &filer_pb.SubscribeMetadataResponse{
|
|
Directory: "/dir",
|
|
EventNotification: &filer_pb.EventNotification{
|
|
NewEntry: &filer_pb.Entry{
|
|
Name: "file.txt",
|
|
Attributes: &filer_pb.FuseAttributes{
|
|
Crtime: 1,
|
|
Mtime: 1,
|
|
FileMode: 0100644,
|
|
FileSize: 11,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
updateResp := &filer_pb.SubscribeMetadataResponse{
|
|
Directory: "/dir",
|
|
EventNotification: &filer_pb.EventNotification{
|
|
OldEntry: &filer_pb.Entry{
|
|
Name: "file.txt",
|
|
},
|
|
NewEntry: &filer_pb.Entry{
|
|
Name: "file.txt",
|
|
Attributes: &filer_pb.FuseAttributes{
|
|
Crtime: 1,
|
|
Mtime: 2,
|
|
FileMode: 0100644,
|
|
FileSize: 29,
|
|
},
|
|
},
|
|
NewParentPath: "/dir",
|
|
},
|
|
}
|
|
deleteResp := &filer_pb.SubscribeMetadataResponse{
|
|
Directory: "/dir",
|
|
EventNotification: &filer_pb.EventNotification{
|
|
OldEntry: &filer_pb.Entry{
|
|
Name: "file.txt",
|
|
},
|
|
},
|
|
}
|
|
|
|
if err := mc.ApplyMetadataResponse(context.Background(), createResp, SubscriberMetadataResponseApplyOptions); err != nil {
|
|
t.Fatalf("apply create: %v", err)
|
|
}
|
|
|
|
entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
|
|
if err != nil {
|
|
t.Fatalf("find created entry: %v", err)
|
|
}
|
|
if entry.FileSize != 11 {
|
|
t.Fatalf("created file size = %d, want 11", entry.FileSize)
|
|
}
|
|
|
|
if err := mc.ApplyMetadataResponse(context.Background(), updateResp, SubscriberMetadataResponseApplyOptions); err != nil {
|
|
t.Fatalf("apply update: %v", err)
|
|
}
|
|
|
|
entry, err = mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
|
|
if err != nil {
|
|
t.Fatalf("find updated entry: %v", err)
|
|
}
|
|
if entry.FileSize != 29 {
|
|
t.Fatalf("updated file size = %d, want 29", entry.FileSize)
|
|
}
|
|
|
|
if err := mc.ApplyMetadataResponse(context.Background(), deleteResp, SubscriberMetadataResponseApplyOptions); err != nil {
|
|
t.Fatalf("apply delete: %v", err)
|
|
}
|
|
|
|
entry, err = mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
|
|
if err != filer_pb.ErrNotFound {
|
|
t.Fatalf("find deleted entry error = %v, want %v", err, filer_pb.ErrNotFound)
|
|
}
|
|
if entry != nil {
|
|
t.Fatalf("deleted entry still cached: %+v", entry)
|
|
}
|
|
|
|
if got := countPath(notifications.paths(), util.FullPath("/dir")); got != 3 {
|
|
t.Fatalf("directory notifications for /dir = %d, want 3", got)
|
|
}
|
|
mc.WaitForEntryInvalidations()
|
|
if got := countPath(invalidations.paths(), util.FullPath("/dir/file.txt")); got != 3 {
|
|
t.Fatalf("invalidations for /dir/file.txt = %d, want 3 (create + update + delete)", got)
|
|
}
|
|
}
|
|
|
|
func TestApplyMetadataResponseRenamesAcrossCachedDirectories(t *testing.T) {
|
|
mc, _, notifications, invalidations := newTestMetaCache(t, map[util.FullPath]bool{
|
|
"/": true,
|
|
"/src": true,
|
|
"/dst": true,
|
|
})
|
|
defer mc.Shutdown()
|
|
|
|
if err := mc.InsertEntry(context.Background(), &filer.Entry{
|
|
FullPath: "/src/file.tmp",
|
|
Attr: filer.Attr{
|
|
Crtime: time.Unix(1, 0),
|
|
Mtime: time.Unix(1, 0),
|
|
Mode: 0100644,
|
|
FileSize: 7,
|
|
},
|
|
}); err != nil {
|
|
t.Fatalf("insert source entry: %v", err)
|
|
}
|
|
|
|
renameResp := &filer_pb.SubscribeMetadataResponse{
|
|
Directory: "/src",
|
|
EventNotification: &filer_pb.EventNotification{
|
|
OldEntry: &filer_pb.Entry{
|
|
Name: "file.tmp",
|
|
},
|
|
NewEntry: &filer_pb.Entry{
|
|
Name: "file.txt",
|
|
Attributes: &filer_pb.FuseAttributes{
|
|
Crtime: 1,
|
|
Mtime: 2,
|
|
FileMode: 0100644,
|
|
FileSize: 41,
|
|
},
|
|
},
|
|
NewParentPath: "/dst",
|
|
},
|
|
}
|
|
|
|
if err := mc.ApplyMetadataResponse(context.Background(), renameResp, SubscriberMetadataResponseApplyOptions); err != nil {
|
|
t.Fatalf("apply rename: %v", err)
|
|
}
|
|
|
|
oldEntry, err := mc.FindEntry(context.Background(), util.FullPath("/src/file.tmp"))
|
|
if err != filer_pb.ErrNotFound {
|
|
t.Fatalf("find old path error = %v, want %v", err, filer_pb.ErrNotFound)
|
|
}
|
|
if oldEntry != nil {
|
|
t.Fatalf("old path still cached: %+v", oldEntry)
|
|
}
|
|
|
|
newEntry, err := mc.FindEntry(context.Background(), util.FullPath("/dst/file.txt"))
|
|
if err != nil {
|
|
t.Fatalf("find new path: %v", err)
|
|
}
|
|
if newEntry.FileSize != 41 {
|
|
t.Fatalf("renamed file size = %d, want 41", newEntry.FileSize)
|
|
}
|
|
|
|
if got := countPath(notifications.paths(), util.FullPath("/src")); got != 1 {
|
|
t.Fatalf("directory notifications for /src = %d, want 1", got)
|
|
}
|
|
if got := countPath(notifications.paths(), util.FullPath("/dst")); got != 1 {
|
|
t.Fatalf("directory notifications for /dst = %d, want 1", got)
|
|
}
|
|
mc.WaitForEntryInvalidations()
|
|
if got := countPath(invalidations.paths(), util.FullPath("/src/file.tmp")); got != 1 {
|
|
t.Fatalf("invalidations for /src/file.tmp = %d, want 1", got)
|
|
}
|
|
if got := countPath(invalidations.paths(), util.FullPath("/dst/file.txt")); got != 1 {
|
|
t.Fatalf("invalidations for /dst/file.txt = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestApplyMetadataResponseLocalOptionsSkipInvalidations(t *testing.T) {
|
|
mc, _, notifications, invalidations := newTestMetaCache(t, map[util.FullPath]bool{
|
|
"/": true,
|
|
"/dir": true,
|
|
})
|
|
defer mc.Shutdown()
|
|
|
|
if err := mc.InsertEntry(context.Background(), &filer.Entry{
|
|
FullPath: "/dir/file.txt",
|
|
Attr: filer.Attr{
|
|
Crtime: time.Unix(1, 0),
|
|
Mtime: time.Unix(1, 0),
|
|
Mode: 0100644,
|
|
FileSize: 7,
|
|
},
|
|
}); err != nil {
|
|
t.Fatalf("insert source entry: %v", err)
|
|
}
|
|
|
|
updateResp := &filer_pb.SubscribeMetadataResponse{
|
|
Directory: "/dir",
|
|
EventNotification: &filer_pb.EventNotification{
|
|
OldEntry: &filer_pb.Entry{
|
|
Name: "file.txt",
|
|
},
|
|
NewEntry: &filer_pb.Entry{
|
|
Name: "file.txt",
|
|
Attributes: &filer_pb.FuseAttributes{
|
|
Crtime: 1,
|
|
Mtime: 2,
|
|
FileMode: 0100644,
|
|
FileSize: 17,
|
|
},
|
|
},
|
|
NewParentPath: "/dir",
|
|
},
|
|
}
|
|
|
|
if err := mc.ApplyMetadataResponse(context.Background(), updateResp, LocalMetadataResponseApplyOptions); err != nil {
|
|
t.Fatalf("apply local update: %v", err)
|
|
}
|
|
|
|
entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
|
|
if err != nil {
|
|
t.Fatalf("find updated entry: %v", err)
|
|
}
|
|
if entry.FileSize != 17 {
|
|
t.Fatalf("updated file size = %d, want 17", entry.FileSize)
|
|
}
|
|
if got := countPath(notifications.paths(), util.FullPath("/dir")); got != 1 {
|
|
t.Fatalf("directory notifications for /dir = %d, want 1", got)
|
|
}
|
|
mc.WaitForEntryInvalidations()
|
|
if got := len(invalidations.paths()); got != 0 {
|
|
t.Fatalf("invalidations = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
func TestApplyMetadataResponseDeduplicatesRepeatedFilerEvent(t *testing.T) {
|
|
mc, _, notifications, invalidations := newTestMetaCache(t, map[util.FullPath]bool{
|
|
"/": true,
|
|
"/dir": true,
|
|
})
|
|
defer mc.Shutdown()
|
|
|
|
if err := mc.InsertEntry(context.Background(), &filer.Entry{
|
|
FullPath: "/dir/file.txt",
|
|
Attr: filer.Attr{
|
|
Crtime: time.Unix(1, 0),
|
|
Mtime: time.Unix(1, 0),
|
|
Mode: 0100644,
|
|
FileSize: 5,
|
|
},
|
|
}); err != nil {
|
|
t.Fatalf("insert source entry: %v", err)
|
|
}
|
|
|
|
updateResp := &filer_pb.SubscribeMetadataResponse{
|
|
Directory: "/dir",
|
|
EventNotification: &filer_pb.EventNotification{
|
|
OldEntry: &filer_pb.Entry{
|
|
Name: "file.txt",
|
|
},
|
|
NewEntry: &filer_pb.Entry{
|
|
Name: "file.txt",
|
|
Attributes: &filer_pb.FuseAttributes{
|
|
Crtime: 1,
|
|
Mtime: 2,
|
|
FileMode: 0100644,
|
|
FileSize: 15,
|
|
},
|
|
},
|
|
NewParentPath: "/dir",
|
|
Signatures: []int32{7},
|
|
},
|
|
TsNs: 99,
|
|
}
|
|
|
|
if err := mc.ApplyMetadataResponse(context.Background(), updateResp, SubscriberMetadataResponseApplyOptions); err != nil {
|
|
t.Fatalf("first apply: %v", err)
|
|
}
|
|
if err := mc.ApplyMetadataResponse(context.Background(), updateResp, SubscriberMetadataResponseApplyOptions); err != nil {
|
|
t.Fatalf("second apply: %v", err)
|
|
}
|
|
|
|
entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt"))
|
|
if err != nil {
|
|
t.Fatalf("find updated entry: %v", err)
|
|
}
|
|
if entry.FileSize != 15 {
|
|
t.Fatalf("updated file size = %d, want 15", entry.FileSize)
|
|
}
|
|
if got := countPath(notifications.paths(), util.FullPath("/dir")); got != 1 {
|
|
t.Fatalf("directory notifications for /dir = %d, want 1", got)
|
|
}
|
|
mc.WaitForEntryInvalidations()
|
|
if got := countPath(invalidations.paths(), util.FullPath("/dir/file.txt")); got != 1 {
|
|
t.Fatalf("invalidations for /dir/file.txt = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestApplyMetadataResponseSkipsHiddenSystemEntryWhenDisabled(t *testing.T) {
|
|
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
|
|
"/": true,
|
|
})
|
|
defer mc.Shutdown()
|
|
|
|
createResp := &filer_pb.SubscribeMetadataResponse{
|
|
Directory: "/",
|
|
EventNotification: &filer_pb.EventNotification{
|
|
NewEntry: &filer_pb.Entry{
|
|
Name: "topics",
|
|
Attributes: &filer_pb.FuseAttributes{
|
|
Crtime: 1,
|
|
Mtime: 1,
|
|
FileMode: uint32(os.ModeDir | 0o755),
|
|
},
|
|
IsDirectory: true,
|
|
},
|
|
},
|
|
}
|
|
|
|
if err := mc.ApplyMetadataResponse(context.Background(), createResp, SubscriberMetadataResponseApplyOptions); err != nil {
|
|
t.Fatalf("apply create: %v", err)
|
|
}
|
|
|
|
entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics"))
|
|
if err != filer_pb.ErrNotFound {
|
|
t.Fatalf("find hidden entry error = %v, want %v", err, filer_pb.ErrNotFound)
|
|
}
|
|
if entry != nil {
|
|
t.Fatalf("hidden entry still cached: %+v", entry)
|
|
}
|
|
}
|
|
|
|
func TestApplyMetadataResponsePurgesHiddenDestinationPath(t *testing.T) {
|
|
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
|
|
"/": true,
|
|
"/src": true,
|
|
})
|
|
defer mc.Shutdown()
|
|
|
|
if err := mc.InsertEntry(context.Background(), &filer.Entry{
|
|
FullPath: "/topics",
|
|
Attr: filer.Attr{
|
|
Crtime: time.Unix(1, 0),
|
|
Mtime: time.Unix(1, 0),
|
|
Mode: os.ModeDir | 0o755,
|
|
},
|
|
}); err != nil {
|
|
t.Fatalf("insert stale hidden dir: %v", err)
|
|
}
|
|
if err := mc.InsertEntry(context.Background(), &filer.Entry{
|
|
FullPath: "/topics/leaked.txt",
|
|
Attr: filer.Attr{
|
|
Crtime: time.Unix(1, 0),
|
|
Mtime: time.Unix(1, 0),
|
|
Mode: 0o644,
|
|
FileSize: 7,
|
|
},
|
|
}); err != nil {
|
|
t.Fatalf("insert leaked hidden child: %v", err)
|
|
}
|
|
if err := mc.InsertEntry(context.Background(), &filer.Entry{
|
|
FullPath: "/src/visible",
|
|
Attr: filer.Attr{
|
|
Crtime: time.Unix(1, 0),
|
|
Mtime: time.Unix(1, 0),
|
|
Mode: os.ModeDir | 0o755,
|
|
},
|
|
}); err != nil {
|
|
t.Fatalf("insert source dir: %v", err)
|
|
}
|
|
|
|
renameResp := &filer_pb.SubscribeMetadataResponse{
|
|
Directory: "/src",
|
|
EventNotification: &filer_pb.EventNotification{
|
|
OldEntry: &filer_pb.Entry{
|
|
Name: "visible",
|
|
IsDirectory: true,
|
|
},
|
|
NewEntry: &filer_pb.Entry{
|
|
Name: "topics",
|
|
Attributes: &filer_pb.FuseAttributes{
|
|
Crtime: 2,
|
|
Mtime: 2,
|
|
FileMode: uint32(os.ModeDir | 0o755),
|
|
},
|
|
IsDirectory: true,
|
|
},
|
|
NewParentPath: "/",
|
|
},
|
|
}
|
|
|
|
if err := mc.ApplyMetadataResponse(context.Background(), renameResp, SubscriberMetadataResponseApplyOptions); err != nil {
|
|
t.Fatalf("apply rename: %v", err)
|
|
}
|
|
|
|
if entry, err := mc.FindEntry(context.Background(), util.FullPath("/src/visible")); err != filer_pb.ErrNotFound || entry != nil {
|
|
t.Fatalf("source dir after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
|
|
}
|
|
if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics")); err != filer_pb.ErrNotFound || entry != nil {
|
|
t.Fatalf("hidden destination after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
|
|
}
|
|
if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics/leaked.txt")); err != filer_pb.ErrNotFound || entry != nil {
|
|
t.Fatalf("hidden child after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
|
|
}
|
|
}
|
|
|
|
func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache, map[util.FullPath]bool, *recordedPaths, *recordedPaths) {
|
|
t.Helper()
|
|
|
|
mapper, err := NewUidGidMapper("", "")
|
|
if err != nil {
|
|
t.Fatalf("uid/gid mapper: %v", err)
|
|
}
|
|
|
|
var cachedMu sync.Mutex
|
|
notifications := &recordedPaths{}
|
|
invalidations := &recordedPaths{}
|
|
|
|
mc := NewMetaCache(
|
|
filepath.Join(t.TempDir(), "meta"),
|
|
mapper,
|
|
util.FullPath("/"),
|
|
false,
|
|
func(path util.FullPath) {
|
|
cachedMu.Lock()
|
|
defer cachedMu.Unlock()
|
|
cached[path] = true
|
|
},
|
|
func(path util.FullPath) bool {
|
|
cachedMu.Lock()
|
|
defer cachedMu.Unlock()
|
|
return cached[path]
|
|
},
|
|
func(path util.FullPath, entry *filer_pb.Entry) {
|
|
invalidations.record(path)
|
|
},
|
|
func(dir util.FullPath) {
|
|
notifications.record(dir)
|
|
},
|
|
)
|
|
|
|
return mc, cached, notifications, invalidations
|
|
}
|
|
|
|
type recordedPaths struct {
|
|
mu sync.Mutex
|
|
items []util.FullPath
|
|
}
|
|
|
|
func (r *recordedPaths) record(path util.FullPath) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.items = append(r.items, path)
|
|
}
|
|
|
|
func (r *recordedPaths) paths() []util.FullPath {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return append([]util.FullPath(nil), r.items...)
|
|
}
|
|
|
|
func countPath(paths []util.FullPath, target util.FullPath) int {
|
|
count := 0
|
|
for _, path := range paths {
|
|
if path == target {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|