diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index d350b117a..359f9f819 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -121,6 +121,12 @@ message LookupDirectoryEntryRequest { message LookupDirectoryEntryResponse { Entry entry = 1; + // filer log position stamped before the entry read: every event at or + // below it is reflected in the returned entry + int64 log_ts_ns = 2; + // signature of the filer whose clock stamped log_ts_ns; positions are + // only comparable with events that filer logged + int32 log_signature = 3; } message ListEntriesRequest { @@ -446,6 +452,10 @@ message CreateEntryResponse { string error = 1; // kept for human readability + backward compat SubscribeMetadataResponse metadata_event = 2; FilerError error_code = 3; // machine-readable error code + // filer log position stamped under the path lock before the write: + // every event at or below it is reflected in the acknowledged state + int64 log_ts_ns = 4; + int32 log_signature = 5; // filer whose clock stamped log_ts_ns } message UpdateEntryRequest { @@ -461,6 +471,10 @@ message UpdateEntryRequest { } message UpdateEntryResponse { SubscribeMetadataResponse metadata_event = 1; + // filer log position stamped under the path lock before the write: + // every event at or below it is reflected in the acknowledged state + int64 log_ts_ns = 2; + int32 log_signature = 3; // filer whose clock stamped log_ts_ns } message TouchAccessTimeRequest { @@ -763,6 +777,10 @@ message CacheRemoteObjectToLocalClusterRequest { message CacheRemoteObjectToLocalClusterResponse { Entry entry = 1; SubscribeMetadataResponse metadata_event = 2; + // filer log position stamped before the entry read: every event at or + // below it is reflected in the returned entry + int64 log_ts_ns = 3; + int32 log_signature = 4; // filer whose clock stamped log_ts_ns } ///////////////////////// diff --git a/weed/command/filer_remote_sync_dir.go b/weed/command/filer_remote_sync_dir.go index cc03389df..4051f396e 100644 --- a/weed/command/filer_remote_sync_dir.go +++ b/weed/command/filer_remote_sync_dir.go @@ -272,7 +272,7 @@ func collectLastSyncOffset(filerClient filer_pb.FilerClient, grpcDialOption grpc // 3. directory creation time var lastOffsetTs time.Time if timeAgo == 0 { - mountedDirEntry, err := filer_pb.GetEntry(context.Background(), filerClient, util.FullPath(mountedDir)) + mountedDirEntry, _, _, err := filer_pb.GetEntry(context.Background(), filerClient, util.FullPath(mountedDir)) if err != nil { glog.V(0).Infof("get mounted directory %s: %v", mountedDir, err) return time.Now() diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index eabd561b4..7a3fb21c2 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -68,7 +68,7 @@ func ensureBucketAllowEmptyFolders(ctx context.Context, filerClient filer_pb.Fil return nil } - entry, err := filer_pb.GetEntry(ctx, filerClient, util.FullPath(bucketPath)) + entry, _, _, err := filer_pb.GetEntry(ctx, filerClient, util.FullPath(bucketPath)) if err != nil { return err } diff --git a/weed/filer/leveldb/leveldb_store_kv.go b/weed/filer/leveldb/leveldb_store_kv.go index de189607d..ee2c17075 100644 --- a/weed/filer/leveldb/leveldb_store_kv.go +++ b/weed/filer/leveldb/leveldb_store_kv.go @@ -6,6 +6,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/syndtr/goleveldb/leveldb" + leveldb_util "github.com/syndtr/goleveldb/leveldb/util" ) func (store *LevelDBStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { @@ -34,6 +35,21 @@ func (store *LevelDBStore) KvGet(ctx context.Context, key []byte) (value []byte, return } +// VisitKvPrefix calls visit for every key-value pair whose key starts with +// prefix, in key order. Returning an error from visit stops the scan. +func (store *LevelDBStore) VisitKvPrefix(ctx context.Context, prefix []byte, visit func(key []byte, value []byte) error) error { + iter := store.db.NewIterator(leveldb_util.BytesPrefix(prefix), nil) + defer iter.Release() + for iter.Next() { + key := append([]byte(nil), iter.Key()...) + value := append([]byte(nil), iter.Value()...) + if err := visit(key, value); err != nil { + return err + } + } + return iter.Error() +} + func (store *LevelDBStore) KvDelete(ctx context.Context, key []byte) (err error) { err = store.db.Delete(key, nil) diff --git a/weed/mount/filehandle.go b/weed/mount/filehandle.go index ba4a2cdb0..5b956aeb7 100644 --- a/weed/mount/filehandle.go +++ b/weed/mount/filehandle.go @@ -3,6 +3,9 @@ package mount import ( "os" "sync" + "sync/atomic" + + "google.golang.org/protobuf/proto" "github.com/seaweedfs/go-fuse/v2/fuse" "github.com/seaweedfs/seaweedfs/weed/cluster" @@ -39,11 +42,31 @@ type FileHandle struct { isDeleted bool isRenamed bool // set by Rename before waiting for async flush; skips old-path metadata flush + // entryVersionTsNs is the filer log position the handle's entry reflects. + // State at or below it must not replace the entry — that rolls it back. + entryVersionTsNs atomic.Int64 + + // entryVersionSignature identifies the filer whose clock stamped + // entryVersionTsNs, when it came from an RPC fence. Positions from a + // different filer are not comparable, so an event that filer did not log + // is applied rather than fenced out. Zero when the version came from an + // event, whose ordering the subscription already provides. + entryVersionSignature atomic.Int32 + + // baseEntry snapshots the filer state last installed or acknowledged. + // Local writes move the live entry away from it, so "is this event new" + // must be judged here, not against the live entry. Always store a clone. + baseEntry atomic.Pointer[filer_pb.Entry] + // dlmLock holds the distributed lock for cross-mount write coordination. // Non-nil only when -dlm is enabled and the file was opened for writing. // Acquired in AcquireHandle, released in ReleaseHandle. dlmLock *cluster.LiveLock + // remoteInstallMu serializes downloadRemoteEntry's install, which holds + // only the handle's shared lock and so races a second concurrent read. + remoteInstallMu sync.Mutex + // RDMA chunk offset cache for performance optimization chunkOffsetCache []int64 chunkCacheValid bool @@ -67,6 +90,7 @@ func newFileHandle(wfs *WFS, handleId FileHandleId, inode uint64, entry *filer_p } if entry != nil { fh.SetEntry(entry) + fh.baseEntry.Store(proto.Clone(entry).(*filer_pb.Entry)) } if IsDebugFileReadWrite { @@ -119,6 +143,57 @@ func (fh *FileHandle) SetEntry(entry *filer_pb.Entry) { fh.invalidateChunkCache() } +// installAckedEntry installs filer-acknowledged state under the handle lock +// when it outranks the handle. A version never advances without its value: +// stamping alone would fence out the events carrying what the handle lacks. +// Dirty handles are skipped — local writes supersede the ack. +func (fh *FileHandle) installAckedEntry(entry *filer_pb.Entry, versionTsNs int64, signature int32) { + fhActiveLock := fh.wfs.fhLockTable.AcquireLock("installAckedEntry", fh.fh, util.ExclusiveLock) + defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock) + if versionTsNs == 0 || fh.dirtyMetadata || entry == fh.GetEntry().GetEntry() { + return + } + // Refuse only what is provably older. Two known, differing filer + // signatures mean the positions come from unrelated clocks and say + // nothing about each other; dropping the acknowledgment there would leave + // the handle holding the very state this mutation replaced. Unknown + // signatures still compare, as they did before. + handleSignature := fh.entryVersionSignature.Load() + provablyOtherClock := signature != 0 && handleSignature != 0 && signature != handleSignature + if !provablyOtherClock && versionTsNs <= fh.entryVersionTsNs.Load() { + return + } + fh.SetEntry(entry) + fh.setAuthoritativeBase(proto.Clone(entry).(*filer_pb.Entry)) + fh.advanceEntryVersion(versionTsNs, signature) +} + +// setAuthoritativeBase installs the base snapshot a local ack acknowledged. +func (fh *FileHandle) setAuthoritativeBase(base *filer_pb.Entry) { + fh.baseEntry.Store(base) +} + +// advanceEntryVersion raises the entry version, never regresses it, and +// records the clock domain the new position belongs to: a filer signature for +// an RPC fence, zero for an event. The signature travels with the timestamp so +// the two never disagree. A zero position (an unversioned old filer) is a +// no-op, leaving the handle open to refreshes. +func (fh *FileHandle) advanceEntryVersion(tsNs int64, signature int32) { + if tsNs == 0 { + return + } + for { + current := fh.entryVersionTsNs.Load() + if tsNs <= current { + return + } + if fh.entryVersionTsNs.CompareAndSwap(current, tsNs) { + fh.entryVersionSignature.Store(signature) + return + } + } +} + func (fh *FileHandle) ResetDirtyPages() { fh.dirtyPages.Destroy() fh.dirtyPages = newPageWriter(fh, fh.wfs.option.ChunkSizeLimit) diff --git a/weed/mount/filehandle_map.go b/weed/mount/filehandle_map.go index 8a81eb1fa..9bc585bf6 100644 --- a/weed/mount/filehandle_map.go +++ b/weed/mount/filehandle_map.go @@ -49,21 +49,25 @@ func (i *FileHandleToInode) MarkInodeRenamed(inode uint64) { } } -func (i *FileHandleToInode) AcquireFileHandle(wfs *WFS, inode uint64, entry *filer_pb.Entry) *FileHandle { +// AcquireFileHandle fully initializes a handle — entry and the log position it +// reflects — before exposing it in the map, and reports whether one already +// existed. An existing handle only gets its counter bumped: its entry belongs +// to whoever holds the handle lock, so callers install there (AcquireHandle) +// rather than under this one. +func (i *FileHandleToInode) AcquireFileHandle(wfs *WFS, inode uint64, entry *filer_pb.Entry, versionTsNs int64, signature int32) (*FileHandle, bool) { i.Lock() defer i.Unlock() fh, found := i.inode2fh[inode] if !found { fh = newFileHandle(wfs, FileHandleId(util.RandomUint64()), inode, entry) + fh.entryVersionTsNs.Store(versionTsNs) + fh.entryVersionSignature.Store(signature) i.inode2fh[inode] = fh i.fh2inode[fh.fh] = inode - } else { - fh.counter++ + return fh, false } - if fh.GetEntry().GetEntry() != entry { - fh.SetEntry(entry) - } - return fh + fh.counter++ + return fh, true } func (i *FileHandleToInode) ReleaseByHandle(fh FileHandleId) *FileHandle { diff --git a/weed/mount/filehandle_read.go b/weed/mount/filehandle_read.go index fac9cc23d..b804dab22 100644 --- a/weed/mount/filehandle_read.go +++ b/weed/mount/filehandle_read.go @@ -6,6 +6,8 @@ import ( "io" "sort" + "google.golang.org/protobuf/proto" + "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -193,14 +195,52 @@ func (fh *FileHandle) downloadRemoteEntry(entry *LockedEntry) error { return fmt.Errorf("CacheRemoteObjectToLocalCluster file %s: %v", fileFullPath, err) } - fh.SetEntry(resp.Entry) + // Entry and base are kept in local uid/gid form like every other + // install path; the response carries filer ids. Without mapping, an + // unchanged re-delivery compares unequal and destroys dirty pages. + localEntry := proto.Clone(resp.Entry).(*filer_pb.Entry) + if localEntry.Attributes != nil && fh.wfs.option.UidGidMapper != nil { + localEntry.Attributes.Uid, localEntry.Attributes.Gid = fh.wfs.option.UidGidMapper.FilerToLocal(localEntry.Attributes.Uid, localEntry.Attributes.Gid) + } + versionTsNs := ackVersionTsNs(resp) + + // Two concurrent reads can both be here (the handle lock is shared; + // invalidation is excluded by the exclusive one). Install as one unit, + // and only if at least as new — an older response landing last would + // keep the newer version over an older entry, fencing out corrections. + installed := false + fh.remoteInstallMu.Lock() + switch { + case versionTsNs >= fh.entryVersionTsNs.Load(): + fh.SetEntry(localEntry) + fh.setAuthoritativeBase(proto.Clone(localEntry).(*filer_pb.Entry)) + fh.advanceEntryVersion(versionTsNs, resp.GetLogSignature()) + installed = true + case versionTsNs == 0 && fh.GetEntry().GetEntry().IsInRemoteOnly(): + // Unversioned response (pre-upgrade filer) and the handle still has + // no local chunks to read: take the content, but claim no position. + // A response that is merely *older* is refused instead — its content + // predates what the handle already reflects. + fh.SetEntry(localEntry) + fh.setAuthoritativeBase(proto.Clone(localEntry).(*filer_pb.Entry)) + } + fh.remoteInstallMu.Unlock() + + // Only publish state we accepted, and only when it carries a position + // to order it by: an unversioned event would clear the cache entry's + // version and let an older subscriber event roll the cache back. // Async: a sync apply deadlocks against the apply loop's invalidate, which needs this read's file-handle lock. - event := resp.GetMetadataEvent() - if event == nil { - event = metadataUpdateEvent(request.Directory, resp.Entry) + if installed && versionTsNs != 0 { + event := resp.GetMetadataEvent() + if event == nil { + event = metadataUpdateEvent(request.Directory, resp.Entry) + if event != nil { + event.TsNs = versionTsNs + } + } + fh.wfs.applyLocalMetadataEventAsync(event) } - fh.wfs.applyLocalMetadataEventAsync(event) return nil }) diff --git a/weed/mount/filehandle_read_remote_test.go b/weed/mount/filehandle_read_remote_test.go index a6cb0a5cc..70d353b2e 100644 --- a/weed/mount/filehandle_read_remote_test.go +++ b/weed/mount/filehandle_read_remote_test.go @@ -93,7 +93,8 @@ func TestReadUncachedRemoteEntryDoesNotDeadlock(t *testing.T) { func(path util.FullPath) { wfs.inodeToPath.MarkChildrenCached(path) }, func(path util.FullPath) bool { return wfs.inodeToPath.IsChildrenCached(path) }, // Mirror weedfs.go's invalidateFunc: take the file handle exclusive lock. - func(path util.FullPath, _ *filer_pb.Entry) { + func(inv meta_cache.EntryInvalidation) { + path := inv.Path inode, ok := wfs.inodeToPath.GetInode(path) if !ok { return @@ -117,11 +118,11 @@ func TestReadUncachedRemoteEntryDoesNotDeadlock(t *testing.T) { t.Cleanup(func() { wfs.metaCache.Shutdown() }) inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) - fh := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: uint64(len(testServer.content))}, RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: int64(len(testServer.content))}, - }) + }, 0, 0) buff := make([]byte, len(testServer.content)) done := make(chan fuse.Status, 1) diff --git a/weed/mount/meta_cache/meta_cache.go b/weed/mount/meta_cache/meta_cache.go index be586343b..e4d41fb16 100644 --- a/weed/mount/meta_cache/meta_cache.go +++ b/weed/mount/meta_cache/meta_cache.go @@ -31,7 +31,7 @@ type MetaCache struct { uidGidMapper *UidGidMapper markCachedFn func(fullpath util.FullPath) isCachedFn func(fullpath util.FullPath) bool - invalidateFunc func(fullpath util.FullPath, entry *filer_pb.Entry) + invalidateFunc func(EntryInvalidation) onDirectoryUpdate func(dir util.FullPath) pinnedChildFn func(*filer.Entry) bool // a child a rebuild must not drop (local-only, not yet on the filer); nil disables visitGroup singleflight.Group // deduplicates concurrent EnsureVisited calls for the same path @@ -43,11 +43,17 @@ type MetaCache struct { dedupRing dedupRingBuffer includeSystemEntries bool + // dirVersionFloors is each cached directory's listing snapshot: the + // version of every child the listing covered, present or absent, unless + // a later event gave that child its own record. One map write per build + // instead of a record per child. + dirVersionFloors map[util.FullPath]int64 + // Entry invalidations run on a worker, not inline on the apply loop: // invalidateFunc takes the fh lock, which a flush can hold while waiting on // the apply loop (flushMetadataToFiler -> applyLocalMetadataEvent), so inline // invalidation deadlocks the mount. - invalidateWorker *util.AsyncBatchWorker[metadataInvalidation] + invalidateWorker *util.AsyncBatchWorker[EntryInvalidation] } var errMetaCacheClosed = errors.New("metadata cache is shut down") @@ -96,7 +102,7 @@ type metadataApplyRequest struct { } func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPath, includeSystemEntries bool, - markCachedFn func(path util.FullPath), isCachedFn func(path util.FullPath) bool, invalidateFunc func(util.FullPath, *filer_pb.Entry), onDirectoryUpdate func(dir util.FullPath)) *MetaCache { + markCachedFn func(path util.FullPath), isCachedFn func(path util.FullPath) bool, invalidateFunc func(EntryInvalidation), onDirectoryUpdate func(dir util.FullPath)) *MetaCache { leveldbStore, virtualStore := openMetaStore(dbFolder) mc := &MetaCache{ root: root, @@ -107,17 +113,16 @@ func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPat uidGidMapper: uidGidMapper, onDirectoryUpdate: onDirectoryUpdate, includeSystemEntries: includeSystemEntries, - invalidateFunc: func(fullpath util.FullPath, entry *filer_pb.Entry) { - invalidateFunc(fullpath, entry) - }, - applyCh: make(chan metadataApplyRequest, 128), - applyDone: make(chan struct{}), - buildingDirs: make(map[util.FullPath]*directoryBuildState), - dedupRing: newDedupRingBuffer(), + invalidateFunc: invalidateFunc, + applyCh: make(chan metadataApplyRequest, 128), + applyDone: make(chan struct{}), + buildingDirs: make(map[util.FullPath]*directoryBuildState), + dedupRing: newDedupRingBuffer(), + dirVersionFloors: make(map[util.FullPath]int64), } - mc.invalidateWorker = util.NewAsyncBatchWorker(func(batch []metadataInvalidation) { + mc.invalidateWorker = util.NewAsyncBatchWorker(func(batch []EntryInvalidation) { for _, invalidation := range batch { - mc.invalidateFunc(invalidation.path, invalidation.entry) + mc.invalidateFunc(invalidation) } }) go mc.runApplyLoop() @@ -142,18 +147,26 @@ func openMetaStore(dbFolder string) (*leveldb.LevelDBStore, filer.VirtualFilerSt } -func (mc *MetaCache) InsertEntry(ctx context.Context, entry *filer.Entry) error { +// InsertEntry stores an entry at the filer log position it reflects. Zero means +// local content no log position describes, which records that explicitly so the +// entry does not inherit its directory's listing floor. +func (mc *MetaCache) InsertEntry(ctx context.Context, entry *filer.Entry, versionTsNs int64) error { mc.Lock() defer mc.Unlock() - return mc.doInsertEntry(ctx, entry) + return mc.doInsertEntry(ctx, entry, versionTsNs) } -func (mc *MetaCache) doInsertEntry(ctx context.Context, entry *filer.Entry) error { - return mc.localStore.InsertEntry(ctx, entry) +func (mc *MetaCache) doInsertEntry(ctx context.Context, entry *filer.Entry, versionTsNs int64) error { + if err := mc.localStore.InsertEntry(ctx, entry); err != nil { + return err + } + mc.setEntryVersionLocked(ctx, entry.FullPath, versionTsNs) + return nil } // doBatchInsertEntries inserts multiple entries using LevelDB's batch write. // This is more efficient than inserting entries one by one. + func (mc *MetaCache) doBatchInsertEntries(ctx context.Context, entries []*filer.Entry) error { return mc.leveldbStore.BatchInsertEntries(ctx, entries) } @@ -161,30 +174,35 @@ func (mc *MetaCache) doBatchInsertEntries(ctx context.Context, entries []*filer. func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry) error { mc.Lock() defer mc.Unlock() - return mc.atomicUpdateEntryFromFilerLocked(ctx, oldPath, newEntry, false) + return mc.atomicUpdateEntryFromFilerLocked(ctx, oldPath, newEntry, false, 0) } -func (mc *MetaCache) atomicUpdateEntryFromFilerLocked(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry, allowUncachedInsert bool) error { +func (mc *MetaCache) atomicUpdateEntryFromFilerLocked(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry, allowUncachedInsert bool, versionTsNs int64) error { entry, err := mc.localStore.FindEntry(ctx, oldPath) if err != nil && err != filer_pb.ErrNotFound { glog.Errorf("Metacache: find entry error: %v", err) return err } - if entry != nil { - if oldPath != "" { - if newEntry != nil && oldPath == newEntry.FullPath { - // skip the unnecessary deletion - // leave the update to the following InsertEntry operation - } else { - ctx = context.WithValue(ctx, "OP", "MV") - glog.V(3).Infof("DeleteEntry %s", oldPath) - if err := mc.localStore.DeleteEntry(ctx, oldPath); err != nil { - return err - } - } + vacatingOldPath := oldPath != "" && !(newEntry != nil && oldPath == newEntry.FullPath) + if entry != nil && vacatingOldPath { + ctx = context.WithValue(ctx, "OP", "MV") + glog.V(3).Infof("DeleteEntry %s", oldPath) + if err := mc.localStore.DeleteEntry(ctx, oldPath); err != nil { + return err + } + } + if vacatingOldPath { + // A deletion is a fact about the path with no entry left to carry it: + // without a tombstone a delayed older event resurrects it, and the + // deletion's own redelivery is dedup-suppressed. Only for cached + // parents — an uncached one gates its own inserts, so a tombstone + // there would just accumulate. + oldDir, _ := oldPath.DirAndName() + if versionTsNs != 0 && (allowUncachedInsert || mc.isCachedFn(util.FullPath(oldDir))) { + mc.setEntryTombstoneLocked(ctx, oldPath, versionTsNs) + } else { + mc.clearEntryVersionLocked(ctx, oldPath) } - } else { - // println("unknown old directory:", oldDir) } if newEntry != nil { @@ -194,6 +212,7 @@ func (mc *MetaCache) atomicUpdateEntryFromFilerLocked(ctx context.Context, oldPa if err := mc.localStore.InsertEntry(ctx, newEntry); err != nil { return err } + mc.setEntryVersionLocked(ctx, newEntry.FullPath, versionTsNs) } } return nil @@ -323,7 +342,11 @@ func (mc *MetaCache) PurgeDirectoryChildren(dirPath util.FullPath, resetFn func( func (mc *MetaCache) UpdateEntry(ctx context.Context, entry *filer.Entry) error { mc.Lock() defer mc.Unlock() - return mc.localStore.UpdateEntry(ctx, entry) + if err := mc.localStore.UpdateEntry(ctx, entry); err != nil { + return err + } + mc.markEntryUnversionedLocked(ctx, entry.FullPath) + return nil } // TouchDirMtimeCtime updates the mtime and ctime of a directory entry @@ -342,31 +365,228 @@ func (mc *MetaCache) TouchDirMtimeCtime(ctx context.Context, dirPath util.FullPa } entry.Attr.Mtime = now entry.Attr.Ctime = now - return mc.localStore.UpdateEntry(ctx, entry) + if err := mc.localStore.UpdateEntry(ctx, entry); err != nil { + return err + } + mc.markEntryUnversionedLocked(ctx, dirPath) + return nil } -func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *filer.Entry, err error) { +// FindEntry returns the entry together with the filer log position it +// reflects: the position of the write that produced it, its directory's +// listing snapshot when the listing covered it, or zero for local content no +// log position describes. +func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *filer.Entry, versionTsNs int64, err error) { mc.RLock() defer mc.RUnlock() entry, err = mc.localStore.FindEntry(ctx, fp) if err != nil { - return nil, err + return nil, 0, err } - if entry.TtlSec > 0 && entry.Crtime.Add(time.Duration(entry.TtlSec)*time.Second).Before(time.Now()) { - return nil, filer_pb.ErrNotFound + if isTtlExpired(entry) { + return nil, 0, filer_pb.ErrNotFound } mc.mapIdFromFilerToLocal(entry) + recordTsNs, _, unversioned := mc.entryVersionRecordLocked(ctx, fp) + if unversioned { + return entry, 0, nil + } + return entry, mc.entryVersionFloorLocked(fp, recordTsNs), nil +} + +// entryVersionKeyPrefix namespaces per-entry version records apart from entry +// keys. Keys encode parent-NUL-name, so a directory's direct children form one +// contiguous range: pruning scans exactly them, never a subtree. +const ( + entryVersionTombstone = 1 // the path was deleted at the recorded position + entryVersionUnversioned = 2 // local content no log position describes +) + +const entryVersionKeyPrefix = "\x00mount.entry.ver\x00" + +func entryVersionKey(fp util.FullPath) []byte { + dir, name := fp.DirAndName() + return []byte(entryVersionKeyPrefix + dir + "\x00" + name) +} + +func entryVersionChildPrefix(dirPath util.FullPath) []byte { + return []byte(entryVersionKeyPrefix + string(dirPath) + "\x00") +} + +// setEntryVersionLocked records the log position an entry write reflects. +// Zero (an unversioned local write) clears the claim: the new content is not +// proven at the old position. +func (mc *MetaCache) setEntryVersionLocked(ctx context.Context, fp util.FullPath, tsNs int64) { + if tsNs == 0 { + mc.markEntryUnversionedLocked(ctx, fp) + return + } + value := make([]byte, 8) + util.Uint64toBytes(value, uint64(tsNs)) + if err := mc.localStore.KvPut(ctx, entryVersionKey(fp), value); err != nil { + glog.V(1).Infof("set entry version %s: %v", fp, err) + } +} + +// markEntryUnversionedLocked records that a local write replaced an entry's +// content with state no log position describes. Distinct from having no +// record at all: a path with no record is one the directory listing covered, +// so the listing snapshot is its version, while this content is not covered +// by anything and must not inherit that floor. +func (mc *MetaCache) markEntryUnversionedLocked(ctx context.Context, fp util.FullPath) { + value := make([]byte, 9) + value[8] = entryVersionUnversioned + if err := mc.localStore.KvPut(ctx, entryVersionKey(fp), value); err != nil { + glog.V(1).Infof("mark entry unversioned %s: %v", fp, err) + } +} + +// setEntryTombstoneLocked records a versioned deletion. The ninth byte marks +// a tombstone, which fences even with no entry present. +// +// Lifetime: a recreate at the same name overwrites the key, so repeating names +// are self-limiting; distinct names (unique temp files, rotated logs) each keep +// a record until the directory is rebuilt or evicted, which prunes everything +// at or below the new snapshot. The ceiling is therefore the number of +// distinct names deleted in a cached directory since its last listing. +func (mc *MetaCache) setEntryTombstoneLocked(ctx context.Context, fp util.FullPath, tsNs int64) { + value := make([]byte, 9) + util.Uint64toBytes(value, uint64(tsNs)) + value[8] = entryVersionTombstone + if err := mc.localStore.KvPut(ctx, entryVersionKey(fp), value); err != nil { + glog.V(1).Infof("set entry tombstone %s: %v", fp, err) + } +} + +func (mc *MetaCache) clearEntryVersionLocked(ctx context.Context, fp util.FullPath) { + if err := mc.localStore.KvDelete(ctx, entryVersionKey(fp)); err != nil { + glog.V(4).Infof("clear entry version %s: %v", fp, err) + } +} + +func (mc *MetaCache) getEntryVersionRecordLocked(ctx context.Context, fp util.FullPath) (tsNs int64, tombstone bool) { + tsNs, tombstone, _ = mc.entryVersionRecordLocked(ctx, fp) return } +func (mc *MetaCache) entryVersionRecordLocked(ctx context.Context, fp util.FullPath) (tsNs int64, tombstone, unversioned bool) { + value, err := mc.localStore.KvGet(ctx, entryVersionKey(fp)) + if err != nil || len(value) < 8 { + return 0, false, false + } + if len(value) == 9 { + tombstone = value[8] == entryVersionTombstone + unversioned = value[8] == entryVersionUnversioned + } + return int64(util.BytesToUint64(value[:8])), tombstone, unversioned +} + +// entryVersionBlocksLocked reports whether a write at tsNs is already +// reflected at fp. A tombstone fences with no entry present. Otherwise the +// path's version is its own record or, lacking one, its directory's listing +// floor — which covers children the listing saw present and absent alike. +// A plain record only counts while its entry exists: records linger after a +// bulk folder wipe and must not fence a recreate. +func (mc *MetaCache) entryVersionBlocksLocked(ctx context.Context, fp util.FullPath, tsNs int64) bool { + recordTsNs, tombstone, unversioned := mc.entryVersionRecordLocked(ctx, fp) + if tombstone { + return recordTsNs >= tsNs + } + if unversioned { + // Local content no log position describes: fence nothing, so any + // event can correct it. + return false + } + if !mc.entryExistsLocked(ctx, fp) { + recordTsNs = 0 + } + return mc.entryVersionFloorLocked(fp, recordTsNs) >= tsNs +} + +// entryExistsLocked reports whether fp has a live entry, applying the same TTL +// expiry the read path does so both agree on what "exists" means. +func (mc *MetaCache) entryExistsLocked(ctx context.Context, fp util.FullPath) bool { + entry, err := mc.localStore.FindEntry(ctx, fp) + if err != nil || entry == nil { + return false + } + return !isTtlExpired(entry) +} + +// entryVersionFloorLocked raises a path's own version record to its +// directory's listing floor: the listing covered every child at its snapshot, +// so a child without a later record of its own is versioned at the snapshot. +func (mc *MetaCache) entryVersionFloorLocked(fp util.FullPath, recordTsNs int64) int64 { + dir, _ := fp.DirAndName() + if floor := mc.dirVersionFloors[util.FullPath(dir)]; floor > recordTsNs { + return floor + } + return recordTsNs +} + +func isTtlExpired(entry *filer.Entry) bool { + return entry.TtlSec > 0 && entry.Crtime.Add(time.Duration(entry.TtlSec)*time.Second).Before(time.Now()) +} + +// pruneSupersededTombstonesLocked drops direct-child tombstones at or below +// the listing snapshot: the absence floor now fences what they fenced. +// Tombstones above it (deletions the listing has not seen) survive; deeper +// descendants answer to their own directory's floor. +func (mc *MetaCache) pruneSupersededTombstonesLocked(ctx context.Context, dirPath util.FullPath, snapshotTsNs int64) { + var superseded [][]byte + if err := mc.leveldbStore.VisitKvPrefix(ctx, entryVersionChildPrefix(dirPath), func(key, value []byte) error { + if len(value) != 9 || value[8] != entryVersionTombstone { + return nil + } + if int64(util.BytesToUint64(value[:8])) <= snapshotTsNs { + superseded = append(superseded, key) + } + return nil + }); err != nil { + glog.V(1).Infof("prune tombstones %s: %v", dirPath, err) + return + } + for _, key := range superseded { + if err := mc.localStore.KvDelete(ctx, key); err != nil { + glog.V(1).Infof("prune tombstone %s: %v", string(key), err) + } + } +} + +// deleteChildVersionRecordsLocked drops a directory's direct-child version +// records when it is evicted. An uncached directory reads through to the filer +// and gates its own inserts, so its records fence nothing — keeping them only +// grows the store. A rebuild re-derives the floor and tombstones. +func (mc *MetaCache) deleteChildVersionRecordsLocked(ctx context.Context, dirPath util.FullPath) { + var keys [][]byte + if err := mc.leveldbStore.VisitKvPrefix(ctx, entryVersionChildPrefix(dirPath), func(key, value []byte) error { + keys = append(keys, key) + return nil + }); err != nil { + glog.V(1).Infof("collect version records %s: %v", dirPath, err) + return + } + for _, key := range keys { + if err := mc.localStore.KvDelete(ctx, key); err != nil { + glog.V(1).Infof("delete version record %s: %v", string(key), err) + } + } +} + func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) { mc.Lock() defer mc.Unlock() - return mc.localStore.DeleteEntry(ctx, fp) + if err = mc.localStore.DeleteEntry(ctx, fp); err != nil { + return err + } + mc.clearEntryVersionLocked(ctx, fp) + return nil } func (mc *MetaCache) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) { mc.Lock() defer mc.Unlock() + delete(mc.dirVersionFloors, fp) + mc.deleteChildVersionRecordsLocked(ctx, fp) return mc.localStore.DeleteFolderChildren(ctx, fp) } @@ -562,14 +782,24 @@ func (mc *MetaCache) handleApplyRequest(req metadataApplyRequest) error { } } -type metadataInvalidation struct { - path util.FullPath - entry *filer_pb.Entry +// EntryInvalidation describes one path's metadata change for an open-handle +// refresh. +type EntryInvalidation struct { + Path util.FullPath + Entry *filer_pb.Entry // entry now at path per the event; nil when the path was vacated + TsNs int64 // the event's filer log position; 0 for locally built events + Deleted bool // vacated by a delete, not a rename away — the file lives on elsewhere + // RenamedTo is the destination when a rename vacated this path: the file + // lives on there, so an open handle follows it rather than being orphaned. + RenamedTo util.FullPath + // Signatures from the event. The filer that logged it appends its own, so + // this identifies the clock domain TsNs belongs to. + Signatures []int32 } type metadataResponseSideEffects struct { dirsToNotify []util.FullPath - invalidations []metadataInvalidation + invalidations []EntryInvalidation } func (mc *MetaCache) applyMetadataResponseNow(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions) error { @@ -582,6 +812,11 @@ func (mc *MetaCache) applyMetadataResponseNow(ctx context.Context, resp *filer_p return mc.applyMetadataResponseDirect(ctx, resp, options, false) } + for _, immediateEvent := range immediateEvents { + if err := mc.applyMetadataResponseDirect(ctx, immediateEvent, MetadataResponseApplyOptions{}, false); err != nil { + return err + } + } // Apply side effects but skip directory notifications for dirs that are // currently being built. Notifying a building dir can trigger // markDirectoryReadThrough → DeleteFolderChildren, wiping entries that @@ -594,11 +829,6 @@ func (mc *MetaCache) applyMetadataResponseNow(ctx context.Context, resp *filer_p } state.bufferedEvents = append(state.bufferedEvents, events...) } - for _, immediateEvent := range immediateEvents { - if err := mc.applyMetadataResponseDirect(ctx, immediateEvent, MetadataResponseApplyOptions{}, false); err != nil { - return err - } - } return nil } @@ -650,7 +880,7 @@ func (mc *MetaCache) WaitForEntryInvalidations() { mc.invalidateWorker.Drain() } -func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, _ MetadataResponseApplyOptions, allowUncachedInsert bool) (metadataResponseSideEffects, error) { +func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions, allowUncachedInsert bool) (metadataResponseSideEffects, error) { message := resp.GetEventNotification() if message == nil { return metadataResponseSideEffects{}, nil @@ -677,7 +907,18 @@ func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *file } mc.Lock() - err := mc.atomicUpdateEntryFromFilerLocked(ctx, oldPath, newEntry, allowUncachedInsert) + // Last-writer-wins per entry: an event at or below an entry's version is + // already reflected in it, and applying it would roll the entry back while + // the version keeps the newer claim. Each half is gated independently. + if resp.TsNs != 0 { + if oldPath != "" && mc.entryVersionBlocksLocked(ctx, oldPath, resp.TsNs) { + oldPath = "" + } + if newEntry != nil && mc.entryVersionBlocksLocked(ctx, newEntry.FullPath, resp.TsNs) { + newEntry = nil + } + } + err := mc.atomicUpdateEntryFromFilerLocked(ctx, oldPath, newEntry, allowUncachedInsert, resp.TsNs) if err == nil && hideNewPath { if purgeErr := mc.purgeEntryLocked(ctx, newPath, message.NewEntry.IsDirectory); purgeErr != nil { err = purgeErr @@ -729,6 +970,8 @@ func (mc *MetaCache) purgeDirectoryChildrenNow(ctx context.Context, dirPath util } mc.Lock() defer mc.Unlock() + delete(mc.dirVersionFloors, dirPath) + mc.deleteChildVersionRecordsLocked(ctx, dirPath) return mc.localStore.DeleteFolderChildren(ctx, dirPath) } @@ -740,6 +983,20 @@ func (mc *MetaCache) completeDirectoryBuildNow(ctx context.Context, dirPath util return nil } + // The listing covered every child at its snapshot, so one directory floor + // versions them all — a child only needs its own record once a later event + // touches it. An unversioned listing (pre-upgrade filer) instead clears the + // children's records, or a re-inserted entry would inherit a stale one. + mc.Lock() + if snapshotTsNs != 0 { + mc.dirVersionFloors[dirPath] = snapshotTsNs + mc.pruneSupersededTombstonesLocked(ctx, dirPath, snapshotTsNs) + } else { + delete(mc.dirVersionFloors, dirPath) + mc.deleteChildVersionRecordsLocked(ctx, dirPath) + } + mc.Unlock() + for _, event := range state.bufferedEvents { // When the server provided a snapshot timestamp, skip events that // the listing already included. When snapshotTsNs == 0 (empty @@ -755,6 +1012,16 @@ func (mc *MetaCache) completeDirectoryBuildNow(ctx context.Context, dirPath util } mc.markCachedFn(dirPath) + + // Re-invalidate every buffered event: each ran against a mid-build store, + // so a handle can hold older state than the completed directory. After + // markCachedFn, versioned at the snapshot to outrank the mid-build install. + for _, event := range state.bufferedEvents { + if event.TsNs < snapshotTsNs { + event.TsNs = snapshotTsNs + } + mc.applyMetadataSideEffects(event, MetadataResponseApplyOptions{InvalidateEntries: true}) + } return nil } @@ -966,16 +1233,16 @@ func collectDirectoryNotifications(resp *filer_pb.SubscribeMetadataResponse) []u return dirs[:n] } -func collectEntryInvalidations(resp *filer_pb.SubscribeMetadataResponse) []metadataInvalidation { +func collectEntryInvalidations(resp *filer_pb.SubscribeMetadataResponse) []EntryInvalidation { message := resp.GetEventNotification() if message == nil { return nil } - var invalidations []metadataInvalidation + var invalidations []EntryInvalidation + signatures := message.Signatures if message.OldEntry != nil && message.NewEntry != nil { oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name) - invalidations = append(invalidations, metadataInvalidation{path: oldKey, entry: message.OldEntry}) // Normalize NewParentPath: empty means same directory as resp.Directory newDir := resp.Directory if message.NewParentPath != "" { @@ -983,7 +1250,10 @@ func collectEntryInvalidations(resp *filer_pb.SubscribeMetadataResponse) []metad } if message.OldEntry.Name != message.NewEntry.Name || resp.Directory != newDir { newKey := util.NewFullPath(newDir, message.NewEntry.Name) - invalidations = append(invalidations, metadataInvalidation{path: newKey, entry: message.NewEntry}) + invalidations = append(invalidations, EntryInvalidation{Path: oldKey, TsNs: resp.TsNs, Signatures: signatures, RenamedTo: newKey}) + invalidations = append(invalidations, EntryInvalidation{Path: newKey, Entry: message.NewEntry, TsNs: resp.TsNs, Signatures: signatures}) + } else { + invalidations = append(invalidations, EntryInvalidation{Path: oldKey, Entry: message.NewEntry, TsNs: resp.TsNs, Signatures: signatures}) } return invalidations } @@ -994,12 +1264,12 @@ func collectEntryInvalidations(resp *filer_pb.SubscribeMetadataResponse) []metad newDir = message.NewParentPath } newKey := util.NewFullPath(newDir, message.NewEntry.Name) - invalidations = append(invalidations, metadataInvalidation{path: newKey, entry: message.NewEntry}) + invalidations = append(invalidations, EntryInvalidation{Path: newKey, Entry: message.NewEntry, TsNs: resp.TsNs, Signatures: signatures}) } if filer_pb.IsDelete(resp) && message.OldEntry != nil { oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name) - invalidations = append(invalidations, metadataInvalidation{path: oldKey, entry: message.OldEntry}) + invalidations = append(invalidations, EntryInvalidation{Path: oldKey, TsNs: resp.TsNs, Deleted: true, Signatures: signatures}) } return invalidations diff --git a/weed/mount/meta_cache/meta_cache_apply_test.go b/weed/mount/meta_cache/meta_cache_apply_test.go index aa97675f9..9d830d881 100644 --- a/weed/mount/meta_cache/meta_cache_apply_test.go +++ b/weed/mount/meta_cache/meta_cache_apply_test.go @@ -65,7 +65,7 @@ func TestApplyMetadataResponseAppliesEventsInOrder(t *testing.T) { t.Fatalf("apply create: %v", err) } - entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")) + entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")) if err != nil { t.Fatalf("find created entry: %v", err) } @@ -77,7 +77,7 @@ func TestApplyMetadataResponseAppliesEventsInOrder(t *testing.T) { t.Fatalf("apply update: %v", err) } - entry, err = mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")) + entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")) if err != nil { t.Fatalf("find updated entry: %v", err) } @@ -89,7 +89,7 @@ func TestApplyMetadataResponseAppliesEventsInOrder(t *testing.T) { t.Fatalf("apply delete: %v", err) } - entry, err = mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")) + entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")) if err != filer_pb.ErrNotFound { t.Fatalf("find deleted entry error = %v, want %v", err, filer_pb.ErrNotFound) } @@ -122,7 +122,7 @@ func TestApplyMetadataResponseRenamesAcrossCachedDirectories(t *testing.T) { Mode: 0100644, FileSize: 7, }, - }); err != nil { + }, 0); err != nil { t.Fatalf("insert source entry: %v", err) } @@ -149,7 +149,7 @@ func TestApplyMetadataResponseRenamesAcrossCachedDirectories(t *testing.T) { t.Fatalf("apply rename: %v", err) } - oldEntry, err := mc.FindEntry(context.Background(), util.FullPath("/src/file.tmp")) + oldEntry, _, err := mc.FindEntry(context.Background(), util.FullPath("/src/file.tmp")) if err != filer_pb.ErrNotFound { t.Fatalf("find old path error = %v, want %v", err, filer_pb.ErrNotFound) } @@ -157,7 +157,7 @@ func TestApplyMetadataResponseRenamesAcrossCachedDirectories(t *testing.T) { t.Fatalf("old path still cached: %+v", oldEntry) } - newEntry, err := mc.FindEntry(context.Background(), util.FullPath("/dst/file.txt")) + newEntry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dst/file.txt")) if err != nil { t.Fatalf("find new path: %v", err) } @@ -195,7 +195,7 @@ func TestApplyMetadataResponseLocalOptionsSkipInvalidations(t *testing.T) { Mode: 0100644, FileSize: 7, }, - }); err != nil { + }, 0); err != nil { t.Fatalf("insert source entry: %v", err) } @@ -222,7 +222,7 @@ func TestApplyMetadataResponseLocalOptionsSkipInvalidations(t *testing.T) { t.Fatalf("apply local update: %v", err) } - entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")) + entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")) if err != nil { t.Fatalf("find updated entry: %v", err) } @@ -253,7 +253,7 @@ func TestApplyMetadataResponseDeduplicatesRepeatedFilerEvent(t *testing.T) { Mode: 0100644, FileSize: 5, }, - }); err != nil { + }, 0); err != nil { t.Fatalf("insert source entry: %v", err) } @@ -285,7 +285,7 @@ func TestApplyMetadataResponseDeduplicatesRepeatedFilerEvent(t *testing.T) { t.Fatalf("second apply: %v", err) } - entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")) + entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")) if err != nil { t.Fatalf("find updated entry: %v", err) } @@ -326,7 +326,7 @@ func TestApplyMetadataResponseSkipsHiddenSystemEntryWhenDisabled(t *testing.T) { t.Fatalf("apply create: %v", err) } - entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics")) + entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/topics")) if err != filer_pb.ErrNotFound { t.Fatalf("find hidden entry error = %v, want %v", err, filer_pb.ErrNotFound) } @@ -349,7 +349,7 @@ func TestApplyMetadataResponsePurgesHiddenDestinationPath(t *testing.T) { Mtime: time.Unix(1, 0), Mode: os.ModeDir | 0o755, }, - }); err != nil { + }, 0); err != nil { t.Fatalf("insert stale hidden dir: %v", err) } if err := mc.InsertEntry(context.Background(), &filer.Entry{ @@ -360,7 +360,7 @@ func TestApplyMetadataResponsePurgesHiddenDestinationPath(t *testing.T) { Mode: 0o644, FileSize: 7, }, - }); err != nil { + }, 0); err != nil { t.Fatalf("insert leaked hidden child: %v", err) } if err := mc.InsertEntry(context.Background(), &filer.Entry{ @@ -370,7 +370,7 @@ func TestApplyMetadataResponsePurgesHiddenDestinationPath(t *testing.T) { Mtime: time.Unix(1, 0), Mode: os.ModeDir | 0o755, }, - }); err != nil { + }, 0); err != nil { t.Fatalf("insert source dir: %v", err) } @@ -398,17 +398,81 @@ func TestApplyMetadataResponsePurgesHiddenDestinationPath(t *testing.T) { t.Fatalf("apply rename: %v", err) } - if entry, err := mc.FindEntry(context.Background(), util.FullPath("/src/visible")); err != filer_pb.ErrNotFound || entry != nil { + if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/src/visible")); err != filer_pb.ErrNotFound || entry != nil { t.Fatalf("source dir after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound) } - if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics")); err != filer_pb.ErrNotFound || entry != nil { + if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/topics")); err != filer_pb.ErrNotFound || entry != nil { t.Fatalf("hidden destination after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound) } - if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics/leaked.txt")); err != filer_pb.ErrNotFound || entry != nil { + if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/topics/leaked.txt")); err != filer_pb.ErrNotFound || entry != nil { t.Fatalf("hidden child after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound) } } +// The entry attached to each invalidation is what an open file handle gets +// refreshed with, so it must be the entry now at that path — or nil when the +// path was vacated and the handle should keep its last entry — versioned by +// the event's filer log timestamp. +func TestCollectEntryInvalidationsCarryAuthoritativeEntries(t *testing.T) { + newEntry := &filer_pb.Entry{ + Name: "file.txt", + Attributes: &filer_pb.FuseAttributes{FileSize: 42}, + } + + update := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 77, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file.txt"}, + NewEntry: newEntry, + NewParentPath: "/dir", + }, + } + got := collectEntryInvalidations(update) + if len(got) != 1 || got[0].Path != "/dir/file.txt" || got[0].Entry != newEntry || got[0].TsNs != 77 { + t.Fatalf("in-place update invalidations = %+v, want [{/dir/file.txt NewEntry ts 77}]", got) + } + + rename := &filer_pb.SubscribeMetadataResponse{ + Directory: "/src", + TsNs: 78, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file.tmp"}, + NewEntry: newEntry, + NewParentPath: "/dst", + }, + } + got = collectEntryInvalidations(rename) + if len(got) != 2 || got[0].Path != "/src/file.tmp" || got[0].Entry != nil || got[0].TsNs != 78 || + got[1].Path != "/dst/file.txt" || got[1].Entry != newEntry || got[1].TsNs != 78 { + t.Fatalf("rename invalidations = %+v, want [{/src/file.tmp nil} {/dst/file.txt NewEntry}]", got) + } + + create := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 79, + EventNotification: &filer_pb.EventNotification{ + NewEntry: newEntry, + }, + } + got = collectEntryInvalidations(create) + if len(got) != 1 || got[0].Path != "/dir/file.txt" || got[0].Entry != newEntry || got[0].TsNs != 79 { + t.Fatalf("create invalidations = %+v, want [{/dir/file.txt NewEntry ts 79}]", got) + } + + deleteResp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 80, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file.txt"}, + }, + } + got = collectEntryInvalidations(deleteResp) + if len(got) != 1 || got[0].Path != "/dir/file.txt" || got[0].Entry != nil || got[0].TsNs != 80 { + t.Fatalf("delete invalidations = %+v, want [{/dir/file.txt nil ts 80}]", got) + } +} + func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache, map[util.FullPath]bool, *recordedPaths, *recordedPaths) { t.Helper() @@ -436,8 +500,8 @@ func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache, defer cachedMu.Unlock() return cached[path] }, - func(path util.FullPath, entry *filer_pb.Entry) { - invalidations.record(path) + func(inv EntryInvalidation) { + invalidations.record(inv.Path) }, func(dir util.FullPath) { notifications.record(dir) @@ -473,3 +537,395 @@ func countPath(paths []util.FullPath, target util.FullPath) int { } return count } + +// An unversioned local write replaces the content, so the previous version +// claim no longer describes it and must be cleared — keeping it would fence +// out events correcting the unversioned state. +func TestUnversionedWriteClearsEntryVersion(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{ + "/": true, + "/dir": true, + }) + defer mc.Shutdown() + + versioned := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1000, + EventNotification: &filer_pb.EventNotification{ + NewEntry: &filer_pb.Entry{ + Name: "file.txt", + Attributes: &filer_pb.FuseAttributes{FileSize: 11, FileMode: 0100644}, + }, + }, + } + if err := mc.ApplyMetadataResponse(context.Background(), versioned, SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply versioned event: %v", err) + } + if _, versionTsNs, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")); err != nil || versionTsNs != 1000 { + t.Fatalf("versioned entry = ts %d, %v; want 1000", versionTsNs, err) + } + + if err := mc.UpdateEntry(context.Background(), &filer.Entry{ + FullPath: "/dir/file.txt", + Attr: filer.Attr{ + Crtime: time.Unix(1, 0), + Mtime: time.Unix(2, 0), + Mode: 0100644, + FileSize: 22, + }, + }); err != nil { + t.Fatalf("local update: %v", err) + } + if _, versionTsNs, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")); err != nil || versionTsNs != 0 { + t.Fatalf("after unversioned write = ts %d, %v; want 0", versionTsNs, err) + } + mc.WaitForEntryInvalidations() +} + +// A rebuild against a pre-upgrade filer returns no snapshot; the reinserted +// entries must not inherit the version records their pre-rebuild +// incarnations left behind, or valid events below the stale claim are +// rejected. +func TestUnversionedRebuildClearsStaleVersions(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{ + "/": true, + "/dir": true, + }) + defer mc.Shutdown() + + versioned := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 3000, + EventNotification: &filer_pb.EventNotification{ + NewEntry: &filer_pb.Entry{ + Name: "file.txt", + Attributes: &filer_pb.FuseAttributes{FileSize: 11, FileMode: 0100644}, + }, + }, + } + if err := mc.ApplyMetadataResponse(context.Background(), versioned, SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply versioned event: %v", err) + } + + if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil { + t.Fatalf("begin build: %v", err) + } + if err := mc.doBatchInsertEntries(context.Background(), []*filer.Entry{{ + FullPath: "/dir/file.txt", + Attr: filer.Attr{ + Crtime: time.Unix(1, 0), + Mtime: time.Unix(1, 0), + Mode: 0100644, + FileSize: 22, + }, + }}); err != nil { + t.Fatalf("batch insert: %v", err) + } + if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 0); err != nil { + t.Fatalf("complete unversioned build: %v", err) + } + + if _, versionTsNs, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file.txt")); err != nil || versionTsNs != 0 { + t.Fatalf("rebuilt entry version = %d, %v; want 0 (stale claim must not survive an unversioned rebuild)", versionTsNs, err) + } + mc.WaitForEntryInvalidations() +} + +// Tombstones are scoped to directories whose cached state the fence +// protects; an uncached parent never serves from the store nor applies the +// resurrecting insert, so a tombstone there would only accumulate. +func TestTombstonesScopedToCachedDirectories(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{ + "/": true, + "/cached": true, + }) + defer mc.Shutdown() + + deleteEventFor := func(dir, name string, tsNs int64) *filer_pb.SubscribeMetadataResponse { + return &filer_pb.SubscribeMetadataResponse{ + Directory: dir, + TsNs: tsNs, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: name}, + }, + } + } + + if err := mc.ApplyMetadataResponse(context.Background(), deleteEventFor("/cached", "file", 2000), SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply cached delete: %v", err) + } + if tsNs, tombstone := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/cached/file")); tsNs != 2000 || !tombstone { + t.Fatalf("cached-dir delete record = (%d, %v), want tombstone at 2000", tsNs, tombstone) + } + + if err := mc.ApplyMetadataResponse(context.Background(), deleteEventFor("/uncached", "file", 2000), SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply uncached delete: %v", err) + } + if tsNs, _ := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/uncached/file")); tsNs != 0 { + t.Fatalf("uncached-dir delete record = %d, want none", tsNs) + } + mc.WaitForEntryInvalidations() +} + +// A completed listing's absence floor supersedes the direct-child tombstones +// at or below its snapshot; newer tombstones survive. +func TestBuildCompletionPrunesSupersededTombstones(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{ + "/": true, + "/dir": true, + }) + defer mc.Shutdown() + + apply := func(resp *filer_pb.SubscribeMetadataResponse) { + t.Helper() + if err := mc.ApplyMetadataResponse(context.Background(), resp, SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply: %v", err) + } + } + createEvent := func(name string, tsNs int64) *filer_pb.SubscribeMetadataResponse { + return &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: tsNs, + EventNotification: &filer_pb.EventNotification{ + NewEntry: &filer_pb.Entry{ + Name: name, + Attributes: &filer_pb.FuseAttributes{FileSize: 1, FileMode: 0100644}, + }, + }, + } + } + deleteEvent := func(name string, tsNs int64) *filer_pb.SubscribeMetadataResponse { + return &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: tsNs, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: name}, + }, + } + } + + apply(createEvent("a", 900)) + apply(deleteEvent("a", 1000)) + apply(createEvent("b", 3900)) + apply(deleteEvent("b", 4000)) + + // A deeper descendant's tombstone belongs to its own directory's floor. + mc.Lock() + mc.setEntryTombstoneLocked(context.Background(), util.FullPath("/dir/sub/x"), 1000) + mc.Unlock() + + if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil { + t.Fatalf("begin build: %v", err) + } + if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 3000); err != nil { + t.Fatalf("complete build: %v", err) + } + + if tsNs, _ := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/a")); tsNs != 0 { + t.Fatalf("tombstone at 1000 should be pruned by the floor at 3000, got %d", tsNs) + } + if tsNs, tombstone := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/b")); tsNs != 4000 || !tombstone { + t.Fatalf("tombstone at 4000 must survive the floor at 3000, got (%d, %v)", tsNs, tombstone) + } + if tsNs, tombstone := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/sub/x")); tsNs != 1000 || !tombstone { + t.Fatalf("descendant tombstone must survive the parent's prune, got (%d, %v)", tsNs, tombstone) + } + mc.WaitForEntryInvalidations() +} + +// Evicting a directory clears its children's version records (plain and +// tombstone) so they cannot accumulate for the lifetime of a long-running +// mount. +func TestDirectoryEvictionClearsVersionRecords(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{ + "/": true, + "/dir": true, + }) + defer mc.Shutdown() + + create := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1000, + EventNotification: &filer_pb.EventNotification{ + NewEntry: &filer_pb.Entry{Name: "keep", Attributes: &filer_pb.FuseAttributes{FileSize: 1, FileMode: 0100644}}, + }, + } + if err := mc.ApplyMetadataResponse(context.Background(), create, SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply create: %v", err) + } + del := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1100, + EventNotification: &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: "gone"}}, + } + if err := mc.ApplyMetadataResponse(context.Background(), del, SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply delete: %v", err) + } + + if tsNs, _ := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/keep")); tsNs != 1000 { + t.Fatalf("version record for /dir/keep = %d, want 1000 before eviction", tsNs) + } + if tsNs, tombstone := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/gone")); tsNs != 1100 || !tombstone { + t.Fatalf("tombstone for /dir/gone = (%d,%v), want (1100,true) before eviction", tsNs, tombstone) + } + + if err := mc.DeleteFolderChildren(context.Background(), util.FullPath("/dir")); err != nil { + t.Fatalf("evict directory: %v", err) + } + + if tsNs, _ := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/keep")); tsNs != 0 { + t.Fatalf("version record for /dir/keep survived eviction: %d", tsNs) + } + if tsNs, _ := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/gone")); tsNs != 0 { + t.Fatalf("tombstone for /dir/gone survived eviction: %d", tsNs) + } + mc.WaitForEntryInvalidations() +} + +// A completed build versions its children through one directory floor rather +// than a version record per child, and the floor fences exactly as per-child +// records did. +func TestBuildFloorVersionsChildrenWithoutPerChildRecords(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{ + "/": true, + "/dir": true, + }) + defer mc.Shutdown() + + if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil { + t.Fatalf("begin build: %v", err) + } + if err := mc.doBatchInsertEntries(context.Background(), []*filer.Entry{{ + FullPath: "/dir/file", + Attr: filer.Attr{ + Crtime: time.Unix(1, 0), + Mtime: time.Unix(1, 0), + Mode: 0100644, + FileSize: 300, + }, + }}); err != nil { + t.Fatalf("batch insert: %v", err) + } + if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 2000); err != nil { + t.Fatalf("complete build: %v", err) + } + + // No per-child record was written... + if tsNs, _ := mc.getEntryVersionRecordLocked(context.Background(), util.FullPath("/dir/file")); tsNs != 0 { + t.Fatalf("per-child version record = %d, want none (the floor versions the child)", tsNs) + } + // ...yet the child reads back at the listing snapshot. + if _, versionTsNs, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file")); err != nil || versionTsNs != 2000 { + t.Fatalf("child version = %d, %v; want 2000 from the directory floor", versionTsNs, err) + } + + // And an event the snapshot already covered is still fenced out. + covered := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + NewEntry: &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: 100, FileMode: 0100644}}, + NewParentPath: "/dir", + }, + } + if err := mc.ApplyMetadataResponse(context.Background(), covered, SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply covered event: %v", err) + } + entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file")) + if err != nil || entry.FileSize != 300 { + t.Fatalf("entry = %+v, %v; want size 300 (floor must fence the covered event)", entry, err) + } + mc.WaitForEntryInvalidations() +} + +// The presence probe applies the same TTL expiry the read path does, so a +// logically-expired entry is judged by its directory's floor rather than by +// the stale version record it still carries. +func TestExpiredEntryIsJudgedByDirectoryFloor(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{ + "/": true, + "/dir": true, + }) + defer mc.Shutdown() + + if err := mc.InsertEntry(context.Background(), &filer.Entry{ + FullPath: "/dir/file", + Attr: filer.Attr{ + Crtime: time.Now().Add(-2 * time.Hour), + Mtime: time.Now().Add(-2 * time.Hour), + Mode: 0100644, + FileSize: 7, + TtlSec: 1, + }, + }, 0); err != nil { + t.Fatalf("insert expiring entry: %v", err) + } + + mc.Lock() + // A record newer than the directory floor: it only governs while the entry + // it describes still exists. Once the entry has logically expired the floor + // is the accurate answer, and an event above the floor must apply. + mc.setEntryVersionLocked(context.Background(), util.FullPath("/dir/file"), 5000) + mc.dirVersionFloors[util.FullPath("/dir")] = 3000 + blocks := mc.entryVersionBlocksLocked(context.Background(), util.FullPath("/dir/file"), 4000) + mc.Unlock() + + if blocks { + t.Fatal("event at 4000 fenced by an expired entry's stale record at 5000; the floor at 3000 should govern") + } +} + +// An unversioned local write leaves content no log position describes, so it +// must not inherit the directory's listing floor — that would fence the very +// events needed to correct it. +func TestUnversionedWriteDoesNotInheritDirectoryFloor(t *testing.T) { + mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{ + "/": true, + "/dir": true, + }) + defer mc.Shutdown() + + if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil { + t.Fatalf("begin build: %v", err) + } + if err := mc.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 2000); err != nil { + t.Fatalf("complete build: %v", err) + } + + // A local write with no log position behind it. + if err := mc.InsertEntry(context.Background(), &filer.Entry{ + FullPath: "/dir/file", + Attr: filer.Attr{ + Crtime: time.Unix(1, 0), + Mtime: time.Unix(1, 0), + Mode: 0100644, + FileSize: 7, + }, + }, 0); err != nil { + t.Fatalf("local insert: %v", err) + } + + if _, versionTsNs, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file")); err != nil || versionTsNs != 0 { + t.Fatalf("version = %d, %v; want 0 (unversioned content must not inherit the floor at 2000)", versionTsNs, err) + } + + // And an event below the floor can still correct it. + correcting := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + NewEntry: &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: 42, FileMode: 0100644}}, + NewParentPath: "/dir", + }, + } + if err := mc.ApplyMetadataResponse(context.Background(), correcting, SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply correcting event: %v", err) + } + entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/file")) + if err != nil || entry.FileSize != 42 { + t.Fatalf("entry = %+v, %v; want size 42 (the floor must not fence a correction of unversioned content)", entry, err) + } + mc.WaitForEntryInvalidations() +} diff --git a/weed/mount/meta_cache/meta_cache_build_test.go b/weed/mount/meta_cache/meta_cache_build_test.go index 64ce96aca..2bb8ba21f 100644 --- a/weed/mount/meta_cache/meta_cache_build_test.go +++ b/weed/mount/meta_cache/meta_cache_build_test.go @@ -120,7 +120,7 @@ func TestEnsureVisitedReplaysBufferedEventsAfterSnapshot(t *testing.T) { t.Fatal("directory /dir should be cached after build completes") } - baseEntry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/base.txt")) + baseEntry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/base.txt")) if err != nil { t.Fatalf("find base entry: %v", err) } @@ -128,7 +128,7 @@ func TestEnsureVisitedReplaysBufferedEventsAfterSnapshot(t *testing.T) { t.Fatalf("base entry size = %d, want 3", baseEntry.FileSize) } - afterEntry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/after.txt")) + afterEntry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/after.txt")) if err != nil { t.Fatalf("find replayed entry: %v", err) } @@ -162,7 +162,7 @@ func TestDirectoryNotificationsSuppressedDuringBuild(t *testing.T) { Mode: 0100644, FileSize: 100, }, - }); err != nil { + }, 0); err != nil { t.Fatalf("insert entry during build: %v", err) } @@ -199,7 +199,7 @@ func TestDirectoryNotificationsSuppressedDuringBuild(t *testing.T) { } // The entry inserted during the build must still be present - entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/existing.txt")) + entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/existing.txt")) if err != nil { t.Fatalf("entry wiped during build: %v", err) } @@ -213,7 +213,7 @@ func TestDirectoryNotificationsSuppressedDuringBuild(t *testing.T) { } // After build completes, the entry from the listing should still exist - entry, err = mc.FindEntry(context.Background(), util.FullPath("/dir/existing.txt")) + entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/existing.txt")) if err != nil { t.Fatalf("entry lost after build completion: %v", err) } @@ -224,7 +224,7 @@ func TestDirectoryNotificationsSuppressedDuringBuild(t *testing.T) { // Buffered events with TsNs > snapshotTsNs (150) should have been replayed for i := 0; i < 5; i++ { name := fmt.Sprintf("new-%d.txt", i) - e, err := mc.FindEntry(context.Background(), util.FullPath("/dir/"+name)) + e, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/"+name)) if err != nil { t.Fatalf("replayed entry %s not found: %v", name, err) } @@ -281,7 +281,7 @@ func TestEmptyDirectoryBuildReplaysAllBufferedEvents(t *testing.T) { // Every buffered event must have been replayed, regardless of TsNs for i := range tsValues { name := fmt.Sprintf("file-%d.txt", i) - e, err := mc.FindEntry(context.Background(), util.FullPath("/empty/"+name)) + e, _, err := mc.FindEntry(context.Background(), util.FullPath("/empty/"+name)) if err != nil { t.Fatalf("replayed entry %s not found: %v", name, err) } @@ -318,7 +318,7 @@ func TestBuildCompletionSurvivesCallerCancellation(t *testing.T) { Mode: 0100644, FileSize: 42, }, - }); err != nil { + }, 0); err != nil { t.Fatalf("insert entry: %v", err) } @@ -364,7 +364,7 @@ func TestBuildCompletionSurvivesCallerCancellation(t *testing.T) { } // The pre-existing entry must survive - entry, findErr := mc.FindEntry(context.Background(), util.FullPath("/dir/kept.txt")) + entry, _, findErr := mc.FindEntry(context.Background(), util.FullPath("/dir/kept.txt")) if findErr != nil { t.Fatalf("find kept entry: %v", findErr) } @@ -373,7 +373,7 @@ func TestBuildCompletionSurvivesCallerCancellation(t *testing.T) { } // The buffered event (TsNs 200 > snapshot 100) must have been replayed - buffered, findErr := mc.FindEntry(context.Background(), util.FullPath("/dir/buffered.txt")) + buffered, _, findErr := mc.FindEntry(context.Background(), util.FullPath("/dir/buffered.txt")) if findErr != nil { t.Fatalf("find buffered entry: %v", findErr) } @@ -397,7 +397,7 @@ func TestBufferedRenameUpdatesOtherDirectoryBeforeBuildCompletes(t *testing.T) { Mode: 0100644, FileSize: 7, }, - }); err != nil { + }, 0); err != nil { t.Fatalf("insert source entry: %v", err) } @@ -429,7 +429,7 @@ func TestBufferedRenameUpdatesOtherDirectoryBeforeBuildCompletes(t *testing.T) { t.Fatalf("apply rename: %v", err) } - oldEntry, err := mc.FindEntry(context.Background(), util.FullPath("/src/from.txt")) + oldEntry, _, err := mc.FindEntry(context.Background(), util.FullPath("/src/from.txt")) if err != filer_pb.ErrNotFound { t.Fatalf("find old path error = %v, want %v", err, filer_pb.ErrNotFound) } @@ -437,7 +437,7 @@ func TestBufferedRenameUpdatesOtherDirectoryBeforeBuildCompletes(t *testing.T) { t.Fatalf("old path should be removed before build completes: %+v", oldEntry) } - newEntry, err := mc.FindEntry(context.Background(), util.FullPath("/dst/to.txt")) + newEntry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dst/to.txt")) if err != filer_pb.ErrNotFound { t.Fatalf("find buffered new path error = %v, want %v", err, filer_pb.ErrNotFound) } @@ -449,7 +449,7 @@ func TestBufferedRenameUpdatesOtherDirectoryBeforeBuildCompletes(t *testing.T) { t.Fatalf("complete build: %v", err) } - newEntry, err = mc.FindEntry(context.Background(), util.FullPath("/dst/to.txt")) + newEntry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dst/to.txt")) if err != nil { t.Fatalf("find replayed new path: %v", err) } @@ -483,7 +483,7 @@ func TestEnsureVisitedPreservesLocalOnlyEntry(t *testing.T) { if err := mc.InsertEntry(context.Background(), &filer.Entry{ FullPath: "/dir/pending.txt", Attr: filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1, Inode: 42}, - }); err != nil { + }, 0); err != nil { t.Fatalf("insert pending entry: %v", err) } @@ -512,13 +512,13 @@ func TestEnsureVisitedPreservesLocalOnlyEntry(t *testing.T) { } // base.txt from the listing is present. - if _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/base.txt")); err != nil { + if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/base.txt")); err != nil { t.Fatalf("listed entry missing after build: %v", err) } // The un-flushed local create must survive the rebuild. With /dir now // authoritatively cached, losing it is the file-vanishes flake. - if _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/pending.txt")); err != nil { + if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/pending.txt")); err != nil { t.Fatalf("local-only entry lost across concurrent rebuild: %v", err) } } @@ -553,7 +553,7 @@ func TestEnsureVisitedDropsUnpinnedStaleEntry(t *testing.T) { if err := EnsureVisited(mc, accessor, util.FullPath("/dir")); err != nil { t.Fatalf("ensure visited: %v", err) } - if entry, err := mc.FindEntry(context.Background(), util.FullPath("/dir/stale.txt")); err != filer_pb.ErrNotFound || entry != nil { + if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/stale.txt")); err != filer_pb.ErrNotFound || entry != nil { t.Fatalf("unpinned stale entry survived rebuild = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound) } } @@ -603,7 +603,7 @@ func TestEnsureVisitedConfirmsTransientEmptyListing(t *testing.T) { if !mc.IsDirectoryCached(util.FullPath("/dir")) { t.Fatal("/dir should be cached after build completes") } - if _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/keep.txt")); err != nil { + if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/keep.txt")); err != nil { t.Fatalf("/dir/keep.txt stranded after transient empty listing: %v", err) } } diff --git a/weed/mount/meta_cache/meta_cache_deadlock_test.go b/weed/mount/meta_cache/meta_cache_deadlock_test.go index ce94bee85..8f688e970 100644 --- a/weed/mount/meta_cache/meta_cache_deadlock_test.go +++ b/weed/mount/meta_cache/meta_cache_deadlock_test.go @@ -58,7 +58,7 @@ func TestApplyLoopInvalidateDoesNotDeadlockWithLockHoldingEnqueuer(t *testing.T) defer cachedMu.Unlock() return cached[path] }, - func(path util.FullPath, entry *filer_pb.Entry) { + func(inv EntryInvalidation) { // Mirrors the wfs invalidateFunc: it takes the open file // handle's exclusive lock before refreshing the handle entry. enteredOnce.Do(func() { close(invalidateEntered) }) diff --git a/weed/mount/meta_cache/meta_cache_purge_test.go b/weed/mount/meta_cache/meta_cache_purge_test.go index 922925b2a..ef6b885ee 100644 --- a/weed/mount/meta_cache/meta_cache_purge_test.go +++ b/weed/mount/meta_cache/meta_cache_purge_test.go @@ -20,7 +20,7 @@ func insertCacheEntry(t *testing.T, mc *MetaCache, path util.FullPath) { Mode: 0100644, FileSize: 1, }, - }); err != nil { + }, 0); err != nil { t.Fatalf("insert %s: %v", path, err) } } @@ -69,7 +69,7 @@ func TestPurgeSkippedWhileDirectoryBuilding(t *testing.T) { t.Fatal("/dir should be cached after build completes") } for _, name := range []string{"/dir/a.txt", "/dir/b.txt"} { - if _, err := mc.FindEntry(context.Background(), util.FullPath(name)); err != nil { + if _, _, err := mc.FindEntry(context.Background(), util.FullPath(name)); err != nil { t.Fatalf("%s missing after mid-build purge: %v", name, err) } } @@ -92,7 +92,7 @@ func TestPurgeClearsWhenNotBuilding(t *testing.T) { if got := atomic.LoadInt32(&resetCalls); got != 1 { t.Fatalf("resetFn ran %d times; want 1", got) } - if _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/a.txt")); err == nil { + if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/a.txt")); err == nil { t.Fatal("/dir/a.txt should have been purged") } } diff --git a/weed/mount/metadata_events.go b/weed/mount/metadata_events.go index 0954ff3c3..fe1688d3f 100644 --- a/weed/mount/metadata_events.go +++ b/weed/mount/metadata_events.go @@ -8,6 +8,22 @@ import ( "google.golang.org/protobuf/proto" ) +// filerAckResponse is any mutation response carrying the log position of the +// state it acknowledged. +type filerAckResponse interface { + GetMetadataEvent() *filer_pb.SubscribeMetadataResponse + GetLogTsNs() int64 +} + +// ackVersionTsNs is the log position an ack confirmed: its event's timestamp, +// or the response's fence when the ack carried no event. +func ackVersionTsNs(resp filerAckResponse) int64 { + if ts := resp.GetMetadataEvent().GetTsNs(); ts != 0 { + return ts + } + return resp.GetLogTsNs() +} + func (wfs *WFS) applyLocalMetadataEvent(ctx context.Context, event *filer_pb.SubscribeMetadataResponse) error { if ctx == nil { ctx = context.Background() diff --git a/weed/mount/peer_announcer.go b/weed/mount/peer_announcer.go index e7469e592..f6e98ed4b 100644 --- a/weed/mount/peer_announcer.go +++ b/weed/mount/peer_announcer.go @@ -14,11 +14,11 @@ import ( // HRW-assigned owner mounts for each cached fid this mount holds. // // Shape: -// * EnqueueAnnounce(fid) is a non-blocking push into an in-memory set. -// * A background ticker flushes every announceInterval. Each flush +// - EnqueueAnnounce(fid) is a non-blocking push into an in-memory set. +// - A background ticker flushes every announceInterval. Each flush // drains the set, adds fids due for TTL renewal, groups by owner // mount via HRW, and sends one ChunkAnnounce RPC per distinct owner. -// * A successful send records announced_at[fid]. A rejected (not-owner) +// - A successful send records announced_at[fid]. A rejected (not-owner) // fid is requeued so it will be retried after the seed view refreshes. // // See design-weed-mount-peer-chunk-sharing.md §4.4. @@ -28,7 +28,7 @@ type PeerAnnouncer struct { selfRack string ownerFor func(fid string) string dialPeer MountPeerDialer - localDir *PeerDirectory // populated directly for fids whose HRW owner == self + localDir *PeerDirectory // populated directly for fids whose HRW owner == self isCached func(fid string) bool // residency check; set nil to disable announceInterval time.Duration announceTTL time.Duration diff --git a/weed/mount/peer_announcer_test.go b/weed/mount/peer_announcer_test.go index 063c2b2f1..f53ad8600 100644 --- a/weed/mount/peer_announcer_test.go +++ b/weed/mount/peer_announcer_test.go @@ -88,7 +88,7 @@ func TestPeerAnnouncer_FlushGroupsByOwner(t *testing.T) { return "owner-2:18081" } - a := NewPeerAnnouncer("self:18080", "", "",ownerFor, fakeDialer(fc, rec), nil) + a := NewPeerAnnouncer("self:18080", "", "", ownerFor, fakeDialer(fc, rec), nil) a.EnqueueAnnounce("3,a") a.EnqueueAnnounce("3,b") @@ -96,7 +96,8 @@ func TestPeerAnnouncer_FlushGroupsByOwner(t *testing.T) { a.FlushForTest(context.Background()) - dialed := rec.snapshot(); sort.Strings(dialed) + dialed := rec.snapshot() + sort.Strings(dialed) if len(dialed) != 2 || dialed[0] != "owner-1:18081" || dialed[1] != "owner-2:18081" { t.Errorf("expected both owners dialed once, got %v", dialed) } @@ -111,7 +112,7 @@ func TestPeerAnnouncer_SkipOwnerEqualsSelf(t *testing.T) { fc := &fakeMountPeerClient{announcedBy: map[string][]filerAnnouncement{}} rec := &dialRecorder{} - a := NewPeerAnnouncer("self:18080", "", "",func(fid string) string { + a := NewPeerAnnouncer("self:18080", "", "", func(fid string) string { return "self:18080" }, fakeDialer(fc, rec), nil) @@ -140,7 +141,7 @@ func TestPeerAnnouncer_RequeueOnRejection(t *testing.T) { } rec := &dialRecorder{} - a := NewPeerAnnouncer("self:18080", "", "",func(fid string) string { + a := NewPeerAnnouncer("self:18080", "", "", func(fid string) string { return "owner:18081" }, fakeDialer(fc, rec), nil) a.EnqueueAnnounce("3,x") @@ -170,7 +171,7 @@ func TestPeerAnnouncer_TTLRenewal(t *testing.T) { rec := &dialRecorder{} now := time.Unix(1000, 0) - a := NewPeerAnnouncer("self:18080", "", "",func(fid string) string { + a := NewPeerAnnouncer("self:18080", "", "", func(fid string) string { return "owner:18081" }, fakeDialer(fc, rec), nil) a.clock = func() time.Time { return now } @@ -355,7 +356,7 @@ func TestPeerAnnouncer_DialerErrorRequeues(t *testing.T) { errDialer := func(ctx context.Context, peerAddr string) (mount_peer_pb.MountPeerClient, func(), error) { return nil, func() {}, context.DeadlineExceeded } - a := NewPeerAnnouncer("self:18080", "", "",func(fid string) string { + a := NewPeerAnnouncer("self:18080", "", "", func(fid string) string { return "owner:18081" }, errDialer, nil) a.EnqueueAnnounce("3,a") diff --git a/weed/mount/peer_fetcher.go b/weed/mount/peer_fetcher.go index c04b69410..87d816c1f 100644 --- a/weed/mount/peer_fetcher.go +++ b/weed/mount/peer_fetcher.go @@ -46,13 +46,14 @@ const maxPeerFetchChunkBytes = 64 * 1024 * 1024 // entryChunkGroup.ReadDataAt (the volume-server path). // // Flow: -// 1. Resolve the offset's leaf chunk (flattening manifests). -// 2. Ask the HRW owner mount for current holders via ChunkLookup. -// 3. For each holder (LRU order from PR #5), open a FetchChunk stream, -// assemble frames into a size-bounded buffer, and verify MD5 -// end-to-end against FileChunk.ETag. -// 4. On success, populate chunk_cache and enqueue an announce so -// other mounts can discover us as a new holder. +// 1. Resolve the offset's leaf chunk (flattening manifests). +// 2. Ask the HRW owner mount for current holders via ChunkLookup. +// 3. For each holder (LRU order from PR #5), open a FetchChunk stream, +// assemble frames into a size-bounded buffer, and verify MD5 +// end-to-end against FileChunk.ETag. +// 4. On success, populate chunk_cache and enqueue an announce so +// other mounts can discover us as a new holder. +// // chunks is a snapshot captured under the LockedEntry lock by the caller. func (fh *FileHandle) tryPeerRead(ctx context.Context, fileSize int64, buff []byte, offset int64, chunks []*filer_pb.FileChunk) (int64, int64, error) { if fh.wfs.peerRegistrar == nil || fh.wfs.peerConnPool == nil { diff --git a/weed/mount/peer_fetcher_test.go b/weed/mount/peer_fetcher_test.go index 867093941..4bf765dc9 100644 --- a/weed/mount/peer_fetcher_test.go +++ b/weed/mount/peer_fetcher_test.go @@ -109,13 +109,13 @@ func TestSortHoldersByLocality(t *testing.T) { selfDC, selfRack := "dc1", "r1" // Input order mimics a server-side LRU list (newest first). holders := []peerHolder{ - {addr: "far-newest", dc: "dc2", rack: "r9"}, // bucket 2 (diff DC) - {addr: "mid-newer", dc: "dc1", rack: "r2"}, // bucket 1 (same DC, diff rack) - {addr: "local-newer", dc: "dc1", rack: "r1"}, // bucket 0 (same rack) - {addr: "far-older", dc: "dc2", rack: "r9"}, // bucket 2 - {addr: "mid-older", dc: "dc1", rack: "r2"}, // bucket 1 - {addr: "local-older", dc: "dc1", rack: "r1"}, // bucket 0 - {addr: "unlabeled-older", dc: "", rack: ""}, // bucket 2 (unknown) + {addr: "far-newest", dc: "dc2", rack: "r9"}, // bucket 2 (diff DC) + {addr: "mid-newer", dc: "dc1", rack: "r2"}, // bucket 1 (same DC, diff rack) + {addr: "local-newer", dc: "dc1", rack: "r1"}, // bucket 0 (same rack) + {addr: "far-older", dc: "dc2", rack: "r9"}, // bucket 2 + {addr: "mid-older", dc: "dc1", rack: "r2"}, // bucket 1 + {addr: "local-older", dc: "dc1", rack: "r1"}, // bucket 0 + {addr: "unlabeled-older", dc: "", rack: ""}, // bucket 2 (unknown) } sortHoldersByLocality(holders, selfDC, selfRack) diff --git a/weed/mount/weedfs.go b/weed/mount/weedfs.go index fd2ef7cc9..4b10fabaa 100644 --- a/weed/mount/weedfs.go +++ b/weed/mount/weedfs.go @@ -1,6 +1,7 @@ package mount import ( + "bytes" "context" "math/rand/v2" "os" @@ -12,6 +13,7 @@ import ( "github.com/seaweedfs/go-fuse/v2/fuse" "google.golang.org/grpc" + "google.golang.org/protobuf/proto" "github.com/seaweedfs/seaweedfs/weed/cluster" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -131,46 +133,46 @@ type WFS struct { fuse.RawFileSystem mount_pb.UnimplementedSeaweedMountServer fs.Inode - option *Option - metaCache *meta_cache.MetaCache - stats statsCache - chunkCache *chunk_cache.TieredChunkCache + option *Option + metaCache *meta_cache.MetaCache + stats statsCache + chunkCache *chunk_cache.TieredChunkCache writeBufferAccountant *page_writer.WriteBufferAccountant - signature int32 - concurrentWriters *util.LimitedConcurrentExecutor - copyBufferPool sync.Pool - concurrentCopiersSem chan struct{} - inodeToPath *InodeToPath - fhMap *FileHandleToInode - dhMap *DirectoryHandleToInode - fuseServer *fuse.Server - IsOverQuota bool - fhLockTable *util.LockTable[FileHandleId] - hardLinkLockTable *util.LockTable[string] - posixLocks *PosixLockTable - posixSid uint64 // this mount's session id, for routed-lock owner identity - posixHint *posixLockHint // local fcntl-lock hint for routed mode - posixOwn *posixlock.Manager // mirror of locks this mount holds, re-asserted via keepalive - rdmaClient *RDMAMountClient - peerRegistrar *PeerRegistrar - peerDirectory *PeerDirectory - peerGrpcServer *PeerGrpcServer - peerAnnouncer *PeerAnnouncer - peerConnPool *PeerConnPool - peerDirectoryStop chan struct{} // closed on unmount to stop the sweeper goroutine - FilerConf *filer.FilerConf - filerClient *wdclient.FilerClient // Cached volume location client - refreshMu sync.Mutex - refreshingDirs map[util.FullPath]struct{} - atimeMu sync.Mutex - atimeMap map[uint64]time.Time // inode -> atime, in-memory only, bounded - dirMtimeMu sync.Mutex - dirMtimeMap map[uint64]time.Time // inode -> mtime/ctime, in-memory overlay for dirs - entryValidSec uint64 // kernel FUSE entry cache TTL in seconds - attrValidSec uint64 // kernel FUSE attr cache TTL in seconds - dirHotWindow time.Duration - dirHotThreshold int - dirIdleEvict time.Duration + signature int32 + concurrentWriters *util.LimitedConcurrentExecutor + copyBufferPool sync.Pool + concurrentCopiersSem chan struct{} + inodeToPath *InodeToPath + fhMap *FileHandleToInode + dhMap *DirectoryHandleToInode + fuseServer *fuse.Server + IsOverQuota bool + fhLockTable *util.LockTable[FileHandleId] + hardLinkLockTable *util.LockTable[string] + posixLocks *PosixLockTable + posixSid uint64 // this mount's session id, for routed-lock owner identity + posixHint *posixLockHint // local fcntl-lock hint for routed mode + posixOwn *posixlock.Manager // mirror of locks this mount holds, re-asserted via keepalive + rdmaClient *RDMAMountClient + peerRegistrar *PeerRegistrar + peerDirectory *PeerDirectory + peerGrpcServer *PeerGrpcServer + peerAnnouncer *PeerAnnouncer + peerConnPool *PeerConnPool + peerDirectoryStop chan struct{} // closed on unmount to stop the sweeper goroutine + FilerConf *filer.FilerConf + filerClient *wdclient.FilerClient // Cached volume location client + refreshMu sync.Mutex + refreshingDirs map[util.FullPath]struct{} + atimeMu sync.Mutex + atimeMap map[uint64]time.Time // inode -> atime, in-memory only, bounded + dirMtimeMu sync.Mutex + dirMtimeMap map[uint64]time.Time // inode -> mtime/ctime, in-memory overlay for dirs + entryValidSec uint64 // kernel FUSE entry cache TTL in seconds + attrValidSec uint64 // kernel FUSE attr cache TTL in seconds + dirHotWindow time.Duration + dirHotThreshold int + dirIdleEvict time.Duration // openMtimeCache maps inode -> [mtime_sec, mtime_ns] from the last Open. // Used to decide whether to set FOPEN_KEEP_CACHE on subsequent opens. @@ -262,8 +264,8 @@ func NewSeaweedFileSystem(option *Option) *WFS { atimeMap: make(map[uint64]time.Time, 8192), openMtimeCache: make(map[uint64][2]int64, 8192), dirMtimeMap: make(map[uint64]time.Time, 1024), - entryValidSec: 1, - attrValidSec: 1, + entryValidSec: 1, + attrValidSec: 1, dirHotWindow: dirHotWindow, dirHotThreshold: dirHotThreshold, dirIdleEvict: dirIdleEvict, @@ -301,28 +303,7 @@ func NewSeaweedFileSystem(option *Option) *WFS { wfs.inodeToPath.MarkChildrenCached(path) }, func(path util.FullPath) bool { return wfs.inodeToPath.IsChildrenCached(path) - }, func(filePath util.FullPath, entry *filer_pb.Entry) { - // Find inode if it is not a deleted path - if inode, inodeFound := wfs.inodeToPath.GetInode(filePath); inodeFound { - // Find open file handle - if fh, fhFound := wfs.fhMap.FindFileHandle(inode); fhFound { - fhActiveLock := fh.wfs.fhLockTable.AcquireLock("invalidateFunc", fh.fh, util.ExclusiveLock) - defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock) - - // Recreate dirty pages - fh.dirtyPages.Destroy() - fh.dirtyPages = newPageWriter(fh, wfs.option.ChunkSizeLimit) - - // Update handle entry - newEntry, status := wfs.maybeLoadEntry(filePath) - if status == fuse.OK { - if fh.GetEntry().GetEntry() != newEntry { - fh.SetEntry(newEntry) - } - } - } - } - }, func(dirPath util.FullPath) { + }, wfs.invalidateOpenFileHandle, func(dirPath util.FullPath) { if wfs.inodeToPath.RecordDirectoryUpdate(dirPath, time.Now(), wfs.dirHotWindow, wfs.dirHotThreshold) { wfs.markDirectoryReadThrough(dirPath) } @@ -552,7 +533,7 @@ func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle } }) } else { - entry, status = wfs.maybeLoadEntry(path) + entry, _, status = wfs.maybeLoadEntry(path) } return } @@ -580,7 +561,9 @@ func (wfs *WFS) isLocalOnlyEntry(entry *filer.Entry) bool { return pending } -func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, fuse.Status) { +// maybeLoadEntry returns the entry and the log position it reflects, or a zero +// position when unknown. +func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, entryVersion, fuse.Status) { // glog.V(3).Infof("read entry cache miss %s", fullpath) _, name := fullpath.DirAndName() @@ -596,33 +579,36 @@ func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, fuse.St Gid: wfs.option.MountGid, Crtime: wfs.option.MountCtime.Unix(), }, - }, fuse.OK + }, entryVersion{}, fuse.OK } - entry, status := wfs.lookupEntry(fullpath) + entry, version, status := wfs.lookupEntry(fullpath) if status != fuse.OK { - return nil, status + return nil, entryVersion{}, status } - return entry.ToProtoEntry(), fuse.OK + return entry.ToProtoEntry(), version, fuse.OK } // lookupEntry looks up an entry by path, checking the local cache first. // Cached metadata is only authoritative when the parent directory itself is cached. // For uncached/read-through directories, always consult the filer directly so stale // local entries do not leak back into lookup results. -func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, fuse.Status) { +// It also returns the log position the entry reflects: the entry's stored +// version, the lookup response's, or zero if unknown. +func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion, fuse.Status) { dir, _ := fullpath.DirAndName() dirPath := util.FullPath(dir) if wfs.metaCache.IsDirectoryCached(dirPath) { - cachedEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullpath) + cachedEntry, cachedVersionTsNs, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullpath) if cacheErr != nil && cacheErr != filer_pb.ErrNotFound { glog.Errorf("lookupEntry: cache lookup for %s failed: %v", fullpath, cacheErr) - return nil, fuse.EIO + return nil, entryVersion{}, fuse.EIO } if cachedEntry != nil { glog.V(4).Infof("lookupEntry cache hit %s", fullpath) - return cachedEntry, fuse.OK + // Store versions come from applied events, not an RPC fence. + return cachedEntry, entryVersion{tsNs: cachedVersionTsNs}, fuse.OK } // Re-check: the directory may have been evicted from cache between // our IsDirectoryCached check and FindEntry (e.g. markDirectoryReadThrough). @@ -634,7 +620,7 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, fuse.Status) // filer-ErrNotFound branch below logs the confirmed drift). if _, inodeFound := wfs.inodeToPath.GetInode(fullpath); !inodeFound { glog.V(4).Infof("lookupEntry cache miss (dir cached) %s", fullpath) - return nil, fuse.ENOENT + return nil, entryVersion{}, fuse.ENOENT } glog.V(2).Infof("lookupEntry: %s missing from cache while parent %s is cached; inode tracked, consulting filer", fullpath, dirPath) } @@ -642,7 +628,21 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, fuse.Status) // Directory not cached - fetch directly from filer without caching the entire directory. glog.V(4).Infof("lookupEntry fetching from filer %s", fullpath) - entry, err := filer_pb.GetEntry(context.Background(), wfs, fullpath) + var entry *filer_pb.Entry + var lookupVersion entryVersion + lookupDir, lookupName := fullpath.DirAndName() + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + resp, lookupErr := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{ + Directory: lookupDir, + Name: lookupName, + }) + if lookupErr != nil { + return lookupErr + } + entry = resp.Entry + lookupVersion = entryVersion{tsNs: resp.LogTsNs, signature: resp.LogSignature} + return nil + }) if err != nil { if err == filer_pb.ErrNotFound { // The entry may exist in the local store from a deferred create @@ -662,9 +662,9 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, fuse.Status) wfs.pendingAsyncFlushMu.Unlock() if hasDirtyHandle || hasPendingFlush { - if localEntry, localErr := wfs.metaCache.FindEntry(context.Background(), fullpath); localErr == nil && localEntry != nil { + if localEntry, localVersionTsNs, localErr := wfs.metaCache.FindEntry(context.Background(), fullpath); localErr == nil && localEntry != nil { glog.V(4).Infof("lookupEntry found deferred entry in local cache %s", fullpath) - return localEntry, fuse.OK + return localEntry, entryVersion{tsNs: localVersionTsNs}, fuse.OK } } } @@ -675,7 +675,7 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, fuse.Status) // up without -v=4 — and include layer-by-layer state so the // next failed run pinpoints which layer dropped the entry. localPresent := false - if localEntry, localErr := wfs.metaCache.FindEntry(context.Background(), fullpath); localErr == nil && localEntry != nil { + if localEntry, _, localErr := wfs.metaCache.FindEntry(context.Background(), fullpath); localErr == nil && localEntry != nil { localPresent = true } glog.Warningf("lookupEntry: filer ErrNotFound for tracked path %s (inode=%d dirtyHandle=%v pendingFlush=%v localCache=%v dirCached=%v) — possible coherence bug", @@ -683,15 +683,174 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, fuse.Status) } else { glog.V(4).Infof("lookupEntry not found %s", fullpath) } - return nil, fuse.ENOENT + return nil, entryVersion{}, fuse.ENOENT } glog.Warningf("lookupEntry GetEntry %s: %v", fullpath, err) - return nil, fuse.EIO + return nil, entryVersion{}, fuse.EIO } if entry != nil && entry.Attributes != nil && wfs.option.UidGidMapper != nil { entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid) } - return filer.FromPbEntry(dir, entry), fuse.OK + return filer.FromPbEntry(dir, entry), lookupVersion, fuse.OK +} + +// entryVersion is a filer log position together with the clock domain it +// belongs to: the signature of the filer that stamped it, or zero when the +// position came from an event rather than an RPC fence. +type entryVersion struct { + tsNs int64 + signature int32 +} + +// sameClockDomain reports whether an event's timestamp is comparable with a +// fence stamped by the filer identified by fenceSignature. The filer that logs +// an event appends its own signature, so its presence means one clock produced +// both positions. A zero fence signature means the handle's position came from +// an event, whose ordering the subscription already provides. +func sameClockDomain(fenceSignature int32, eventSignatures []int32) bool { + if fenceSignature == 0 { + return true + } + for _, sig := range eventSignatures { + if sig == fenceSignature { + return true + } + } + return false +} + +// sameEntryContent reports whether two entries carry the same file content: +// size, inline bytes, and chunk list. Attributes are deliberately excluded — +// its caller decides whether the dirty-page overlay is still valid, and a +// metadata change (chmod, chown, touch, xattr) does not invalidate it. +func sameEntryContent(a, b *filer_pb.Entry) bool { + if a == nil || b == nil { + return a == b + } + if filer.FileSize(a) != filer.FileSize(b) || !bytes.Equal(a.Content, b.Content) { + return false + } + if len(a.Chunks) != len(b.Chunks) { + return false + } + for i := range a.Chunks { + if a.Chunks[i].GetFileIdString() != b.Chunks[i].GetFileIdString() || + a.Chunks[i].Offset != b.Chunks[i].Offset || a.Chunks[i].Size != b.Chunks[i].Size { + return false + } + } + return true +} + +// invalidateOpenFileHandle refreshes an open file handle from a metadata +// subscription event. No filer lookup here: it can fail transiently, and with +// the subscription cursor already past the event, nothing would retry. +func (wfs *WFS) invalidateOpenFileHandle(invalidation meta_cache.EntryInvalidation) { + filePath, eventEntry, eventTsNs := invalidation.Path, invalidation.Entry, invalidation.TsNs + inode, inodeFound := wfs.inodeToPath.GetInode(filePath) + if !inodeFound { + return + } + fh, fhFound := wfs.fhMap.FindFileHandle(inode) + if !fhFound { + return + } + fhActiveLock := wfs.fhLockTable.AcquireLock("invalidateFunc", fh.fh, util.ExclusiveLock) + defer wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock) + + // Invalidations apply asynchronously: the handle may already reflect this + // event or newer state, and rolling it back would never be corrected. + // Only skip within one clock domain — the handle's position may have been + // stamped by a different filer, whose clock says nothing about this + // event's. Applying across domains costs a re-apply the base-equality + // check absorbs; skipping across them leaves the handle stale for good. + if eventTsNs != 0 && eventTsNs <= fh.entryVersionTsNs.Load() && + sameClockDomain(fh.entryVersionSignature.Load(), invalidation.Signatures) { + return + } + + // A cached parent's store entry is the ordered merge of this event and + // anything applied since. An uncached parent takes no store writes, so a + // hit there is a stale leftover — use the event entry instead. + var candidate *filer_pb.Entry + candidateTsNs := eventTsNs + dir, _ := filePath.DirAndName() + if wfs.metaCache.IsDirectoryCached(util.FullPath(dir)) { + if storeEntry, storeVersionTsNs, findErr := wfs.metaCache.FindEntry(context.Background(), filePath); findErr == nil && storeEntry != nil && storeVersionTsNs >= eventTsNs { + candidate = storeEntry.ToProtoEntry() + candidateTsNs = storeVersionTsNs + } + } + if candidate == nil && eventEntry != nil { + candidate = proto.Clone(eventEntry).(*filer_pb.Entry) + if candidate.Attributes == nil { + candidate.Attributes = &filer_pb.FuseAttributes{} + } + if wfs.option.UidGidMapper != nil { + candidate.Attributes.Uid, candidate.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(candidate.Attributes.Uid, candidate.Attributes.Gid) + } + } + if candidate == nil { + // Path vacated. A rename left the file alive at its new name, so the + // handle follows it — an open fd tracks the inode, and leaving it on + // the old path would make its next flush recreate that name instead of + // updating the renamed file. An actual delete instead marks the handle + // so no flush recreates the unlinked name. Either way the entry and + // dirty pages stay, so the open fd still reads its buffered writes. + if invalidation.RenamedTo != "" { + _, replacedInode := wfs.inodeToPath.MovePath(filePath, invalidation.RenamedTo) + // A rename over an existing file destroys that file. Mark its + // handle deleted so its flush cannot resurrect it on top of the + // renamed source now occupying the name. + if replacedInode != 0 && replacedInode != inode { + if replacedFh, found := wfs.fhMap.FindFileHandle(replacedInode); found { + replacedFh.isDeleted = true + } + } + fh.RememberPath(invalidation.RenamedTo) + if _, newName := invalidation.RenamedTo.DirAndName(); newName != "" { + fh.UpdateEntry(func(entry *filer_pb.Entry) { + if entry != nil { + entry.Name = newName + } + }) + } + } + if invalidation.Deleted { + fh.isDeleted = true + } + if !fh.dirtyMetadata { + fh.dirtyPages.Destroy() + fh.dirtyPages = newPageWriter(fh, wfs.option.ChunkSizeLimit) + } + fh.advanceEntryVersion(eventTsNs, 0) + return + } + if candidate.Attributes != nil { + candidate.Attributes.FileSize = filer.FileSize(candidate) + } + // Already reflected — an under-fenced re-delivery (a fence is a lower + // bound). Judged against the base, not the live entry: local writes move + // the live entry, and a re-delivered base must not discard them. + base := fh.baseEntry.Load() + if base != nil && proto.Equal(candidate, base) { + fh.advanceEntryVersion(candidateTsNs, 0) + return + } + // Dirty pages overlay content, so only a content change invalidates them: + // a metadata-only event (chmod, touch, or a committed copy's own event + // against an approximate base) leaves them valid. A dirty handle likewise + // keeps its diverged entry unless foreign content supersedes it. + contentChanged := base == nil || !sameEntryContent(candidate, base) + if contentChanged { + fh.dirtyPages.Destroy() + fh.dirtyPages = newPageWriter(fh, wfs.option.ChunkSizeLimit) + } + if contentChanged || !fh.dirtyMetadata { + fh.SetEntry(candidate) + } + fh.baseEntry.Store(proto.Clone(candidate).(*filer_pb.Entry)) + fh.advanceEntryVersion(candidateTsNs, 0) } func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType { diff --git a/weed/mount/weedfs_access.go b/weed/mount/weedfs_access.go index 463827eb4..90890d823 100644 --- a/weed/mount/weedfs_access.go +++ b/weed/mount/weedfs_access.go @@ -17,8 +17,8 @@ type cachedGroupIDs struct { } var ( - supplementaryGroupCache = make(map[uint32]*cachedGroupIDs) - supplementaryGroupCacheMu sync.RWMutex + supplementaryGroupCache = make(map[uint32]*cachedGroupIDs) + supplementaryGroupCacheMu sync.RWMutex supplementaryGroupCacheTTL = 5 * time.Minute lookupSupplementaryGroupIDs = func(callerUid uint32) ([]string, error) { diff --git a/weed/mount/weedfs_attr.go b/weed/mount/weedfs_attr.go index 9882b256c..8d26da4cd 100644 --- a/weed/mount/weedfs_attr.go +++ b/weed/mount/weedfs_attr.go @@ -317,7 +317,7 @@ func (wfs *WFS) touchDirMtimeCtimeBest(dirPath util.FullPath) { // touchDirMtimeCtime updates a directory's mtime and ctime on the filer. // POSIX requires this when entries are created or removed in the directory. func (wfs *WFS) touchDirMtimeCtime(dirPath util.FullPath) { - dirEntry, code := wfs.maybeLoadEntry(dirPath) + dirEntry, _, code := wfs.maybeLoadEntry(dirPath) if code != fuse.OK || dirEntry == nil || dirEntry.Attributes == nil { return } diff --git a/weed/mount/weedfs_dir_lookup.go b/weed/mount/weedfs_dir_lookup.go index 319a476e2..24e39f684 100644 --- a/weed/mount/weedfs_dir_lookup.go +++ b/weed/mount/weedfs_dir_lookup.go @@ -27,7 +27,7 @@ func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name strin fullFilePath := dirPath.Child(name) // Use shared lookup logic that checks cache first, then filer if needed - localEntry, status := wfs.lookupEntry(fullFilePath) + localEntry, _, status := wfs.lookupEntry(fullFilePath) if status != fuse.OK { return status } diff --git a/weed/mount/weedfs_dir_mkrm.go b/weed/mount/weedfs_dir_mkrm.go index ef0f8ab43..1989bb309 100644 --- a/weed/mount/weedfs_dir_mkrm.go +++ b/weed/mount/weedfs_dir_mkrm.go @@ -140,9 +140,9 @@ func (wfs *WFS) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string entryFullPath := dirFullPath.Child(name) // POSIX: enforce sticky bit on the parent directory. - if dirEntry, dirCode := wfs.maybeLoadEntry(dirFullPath); dirCode == fuse.OK && dirEntry != nil && dirEntry.Attributes != nil { + if dirEntry, _, dirCode := wfs.maybeLoadEntry(dirFullPath); dirCode == fuse.OK && dirEntry != nil && dirEntry.Attributes != nil { targetUid := uint32(0) - if targetEntry, targetCode := wfs.maybeLoadEntry(entryFullPath); targetCode == fuse.OK && targetEntry != nil && targetEntry.Attributes != nil { + if targetEntry, _, targetCode := wfs.maybeLoadEntry(entryFullPath); targetCode == fuse.OK && targetEntry != nil && targetEntry.Attributes != nil { targetUid = targetEntry.Attributes.Uid } if code := checkStickyBit(dirEntry.Attributes.FileMode, dirEntry.Attributes.Uid, targetUid, header.Uid); code != fuse.OK { diff --git a/weed/mount/weedfs_file_copy_range.go b/weed/mount/weedfs_file_copy_range.go index eb9ca4282..3ce3c4ad7 100644 --- a/weed/mount/weedfs_file_copy_range.go +++ b/weed/mount/weedfs_file_copy_range.go @@ -46,7 +46,7 @@ type wholeFileServerCopyRequest struct { // performServerSideWholeFileCopy is a package-level seam so tests can override // the filer call without standing up an HTTP endpoint. -var performServerSideWholeFileCopy = func(cancel <-chan struct{}, wfs *WFS, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, serverSideWholeFileCopyOutcome, error) { +var performServerSideWholeFileCopy = func(cancel <-chan struct{}, wfs *WFS, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, entryVersion, serverSideWholeFileCopyOutcome, error) { return wfs.copyEntryViaFiler(cancel, copyRequest) } @@ -216,7 +216,7 @@ func (wfs *WFS) tryServerSideWholeFileCopy(cancel <-chan struct{}, in *fuse.Copy glog.V(1).Infof("CopyFileRange server-side copy %s => %s (%d bytes)", copyRequest.srcPath, copyRequest.dstPath, copyRequest.sourceSize) - entry, outcome, err := performServerSideWholeFileCopy(cancel, wfs, copyRequest) + entry, entryVersionTsNs, outcome, err := performServerSideWholeFileCopy(cancel, wfs, copyRequest) switch outcome { case serverSideWholeFileCopyCommitted: if err != nil { @@ -224,7 +224,7 @@ func (wfs *WFS) tryServerSideWholeFileCopy(cancel <-chan struct{}, in *fuse.Copy } else { glog.V(1).Infof("CopyFileRange server-side copy %s => %s completed (%d bytes)", copyRequest.srcPath, copyRequest.dstPath, copyRequest.sourceSize) } - wfs.applyServerSideWholeFileCopyResult(fhIn, fhOut, copyRequest.dstPath, entry, copyRequest.sourceSize) + wfs.applyServerSideWholeFileCopyResult(fhIn, fhOut, copyRequest.dstPath, entry, entryVersionTsNs, copyRequest.sourceSize) return uint32(copyRequest.sourceSize), true, fuse.OK case serverSideWholeFileCopyAmbiguous: glog.Warningf("CopyFileRange server-side copy %s => %s outcome ambiguous: %v", copyRequest.srcPath, copyRequest.dstPath, err) @@ -235,9 +235,10 @@ func (wfs *WFS) tryServerSideWholeFileCopy(cancel <-chan struct{}, in *fuse.Copy } } -func (wfs *WFS) applyServerSideWholeFileCopyResult(fhIn, fhOut *FileHandle, dstPath util.FullPath, entry *filer_pb.Entry, sourceSize int64) { +func (wfs *WFS) applyServerSideWholeFileCopyResult(fhIn, fhOut *FileHandle, dstPath util.FullPath, entry *filer_pb.Entry, entryVersionTsNs entryVersion, sourceSize int64) { if entry == nil { entry = synthesizeLocalEntryForServerSideWholeFileCopy(fhIn, fhOut, sourceSize) + entryVersionTsNs = entryVersion{} } if entry == nil { glog.Warningf("CopyFileRange server-side copy %s left no local entry to apply", dstPath) @@ -245,6 +246,11 @@ func (wfs *WFS) applyServerSideWholeFileCopyResult(fhIn, fhOut *FileHandle, dstP } fhOut.SetEntry(entry) + // Enroll the install in the versioned-base protocol: the copy's own + // event must read as a no-op against this base, not as a foreign change + // that destroys writes made to the destination after the copy. + fhOut.baseEntry.Store(proto.Clone(entry).(*filer_pb.Entry)) + fhOut.advanceEntryVersion(entryVersionTsNs.tsNs, entryVersionTsNs.signature) fhOut.RememberPath(dstPath) if entry.Attributes != nil { fhOut.contentType = entry.Attributes.Mime @@ -378,7 +384,7 @@ func wholeFileServerCopyCandidate(fhIn, fhOut *FileHandle, in *fuse.CopyFileRang }, true } -func (wfs *WFS) copyEntryViaFiler(cancel <-chan struct{}, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, serverSideWholeFileCopyOutcome, error) { +func (wfs *WFS) copyEntryViaFiler(cancel <-chan struct{}, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, entryVersion, serverSideWholeFileCopyOutcome, error) { baseCtx, baseCancel := context.WithCancel(context.Background()) defer baseCancel() @@ -400,7 +406,7 @@ func (wfs *WFS) copyEntryViaFiler(cancel <-chan struct{}, copyRequest wholeFileS var err error httpClient, err = util_http.NewGlobalHttpClient() if err != nil { - return nil, serverSideWholeFileCopyNotCommitted, fmt.Errorf("create filer copy http client: %w", err) + return nil, entryVersion{}, serverSideWholeFileCopyNotCommitted, fmt.Errorf("create filer copy http client: %w", err) } } @@ -424,7 +430,7 @@ func (wfs *WFS) copyEntryViaFiler(cancel <-chan struct{}, copyRequest wholeFileS req, err := http.NewRequestWithContext(postCtx, http.MethodPost, copyURL.String(), nil) if err != nil { - return nil, serverSideWholeFileCopyNotCommitted, fmt.Errorf("create filer copy request: %w", err) + return nil, entryVersion{}, serverSideWholeFileCopyNotCommitted, fmt.Errorf("create filer copy request: %w", err) } if jwt := wfs.filerCopyJWT(); jwt != "" { req.Header.Set("Authorization", security.BearerPrefix+string(jwt)) @@ -438,45 +444,47 @@ func (wfs *WFS) copyEntryViaFiler(cancel <-chan struct{}, copyRequest wholeFileS if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) - return nil, serverSideWholeFileCopyNotCommitted, fmt.Errorf("filer copy %s => %s failed: status %d: %s", copyRequest.srcPath, copyRequest.dstPath, resp.StatusCode, string(body)) + return nil, entryVersion{}, serverSideWholeFileCopyNotCommitted, fmt.Errorf("filer copy %s => %s failed: status %d: %s", copyRequest.srcPath, copyRequest.dstPath, resp.StatusCode, string(body)) } readbackCtx, readbackCancel := context.WithTimeout(baseCtx, filerCopyReadbackTimeout) defer readbackCancel() - entry, err := filer_pb.GetEntry(readbackCtx, wfs, copyRequest.dstPath) + entry, logTsNs, logSignature, err := filer_pb.GetEntry(readbackCtx, wfs, copyRequest.dstPath) + entryVersionTsNs := entryVersion{tsNs: logTsNs, signature: logSignature} if err != nil { - return nil, serverSideWholeFileCopyCommitted, fmt.Errorf("reload copied entry %s: %w", copyRequest.dstPath, err) + return nil, entryVersion{}, serverSideWholeFileCopyCommitted, fmt.Errorf("reload copied entry %s: %w", copyRequest.dstPath, err) } if entry == nil { - return nil, serverSideWholeFileCopyCommitted, fmt.Errorf("reload copied entry %s: not found", copyRequest.dstPath) + return nil, entryVersion{}, serverSideWholeFileCopyCommitted, fmt.Errorf("reload copied entry %s: not found", copyRequest.dstPath) } if entry.Attributes != nil && wfs.option != nil && wfs.option.UidGidMapper != nil { entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid) } - return entry, serverSideWholeFileCopyCommitted, nil + return entry, entryVersionTsNs, serverSideWholeFileCopyCommitted, nil } -func (wfs *WFS) confirmServerSideWholeFileCopyAfterAmbiguousRequest(baseCtx context.Context, copyRequest wholeFileServerCopyRequest, requestErr error) (*filer_pb.Entry, serverSideWholeFileCopyOutcome, error) { +func (wfs *WFS) confirmServerSideWholeFileCopyAfterAmbiguousRequest(baseCtx context.Context, copyRequest wholeFileServerCopyRequest, requestErr error) (*filer_pb.Entry, entryVersion, serverSideWholeFileCopyOutcome, error) { readbackCtx, readbackCancel := context.WithTimeout(baseCtx, filerCopyReadbackTimeout) defer readbackCancel() - entry, err := filer_pb.GetEntry(readbackCtx, wfs, copyRequest.dstPath) + entry, logTsNs, logSignature, err := filer_pb.GetEntry(readbackCtx, wfs, copyRequest.dstPath) + entryVersionTsNs := entryVersion{tsNs: logTsNs, signature: logSignature} if err == nil && entry != nil && entryMatchesServerSideWholeFileCopy(copyRequest, entry) { if entry.Attributes != nil && wfs.option != nil && wfs.option.UidGidMapper != nil { entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid) } - return entry, serverSideWholeFileCopyCommitted, nil + return entry, entryVersionTsNs, serverSideWholeFileCopyCommitted, nil } if err != nil { - return nil, serverSideWholeFileCopyAmbiguous, fmt.Errorf("%w; post-copy readback failed: %v", requestErr, err) + return nil, entryVersion{}, serverSideWholeFileCopyAmbiguous, fmt.Errorf("%w; post-copy readback failed: %v", requestErr, err) } if entry == nil { - return nil, serverSideWholeFileCopyAmbiguous, fmt.Errorf("%w; destination %s was not readable after the ambiguous request", requestErr, copyRequest.dstPath) + return nil, entryVersion{}, serverSideWholeFileCopyAmbiguous, fmt.Errorf("%w; destination %s was not readable after the ambiguous request", requestErr, copyRequest.dstPath) } - return nil, serverSideWholeFileCopyAmbiguous, fmt.Errorf("%w; destination %s did not match the requested copy after the ambiguous request", requestErr, copyRequest.dstPath) + return nil, entryVersion{}, serverSideWholeFileCopyAmbiguous, fmt.Errorf("%w; destination %s did not match the requested copy after the ambiguous request", requestErr, copyRequest.dstPath) } func entryMatchesServerSideWholeFileCopy(copyRequest wholeFileServerCopyRequest, entry *filer_pb.Entry) bool { diff --git a/weed/mount/weedfs_file_copy_range_test.go b/weed/mount/weedfs_file_copy_range_test.go index 02c34409b..4bca12e94 100644 --- a/weed/mount/weedfs_file_copy_range_test.go +++ b/weed/mount/weedfs_file_copy_range_test.go @@ -21,7 +21,7 @@ func TestWholeFileServerCopyCandidate(t *testing.T) { srcInode := wfs.inodeToPath.Lookup(srcPath, 1, false, false, 0, true) dstInode := wfs.inodeToPath.Lookup(dstPath, 1, false, false, 0, true) - srcHandle := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{ + srcHandle, _ := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{ Name: "src.txt", Attributes: &filer_pb.FuseAttributes{ FileMode: 0100644, @@ -29,14 +29,14 @@ func TestWholeFileServerCopyCandidate(t *testing.T) { Inode: srcInode, }, Content: []byte("hello"), - }) - dstHandle := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{ + }, 0, 0) + dstHandle, _ := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{ Name: "dst.txt", Attributes: &filer_pb.FuseAttributes{ FileMode: 0100644, Inode: dstInode, }, - }) + }, 0, 0) srcHandle.RememberPath(srcPath) dstHandle.RememberPath(dstPath) @@ -86,7 +86,7 @@ func TestCopyFileRangeUsesServerSideWholeFileCopy(t *testing.T) { srcInode := wfs.inodeToPath.Lookup(srcPath, 1, false, false, 0, true) dstInode := wfs.inodeToPath.Lookup(dstPath, 1, false, false, 0, true) - srcHandle := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{ + srcHandle, _ := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{ Name: "src.txt", Attributes: &filer_pb.FuseAttributes{ FileMode: 0100644, @@ -94,14 +94,14 @@ func TestCopyFileRangeUsesServerSideWholeFileCopy(t *testing.T) { Inode: srcInode, }, Content: []byte("hello"), - }) - dstHandle := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{ + }, 0, 0) + dstHandle, _ := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{ Name: "dst.txt", Attributes: &filer_pb.FuseAttributes{ FileMode: 0100644, Inode: dstInode, }, - }) + }, 0, 0) srcHandle.RememberPath(srcPath) dstHandle.RememberPath(dstPath) @@ -112,7 +112,7 @@ func TestCopyFileRangeUsesServerSideWholeFileCopy(t *testing.T) { }() var called bool - performServerSideWholeFileCopy = func(cancel <-chan struct{}, gotWFS *WFS, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, serverSideWholeFileCopyOutcome, error) { + performServerSideWholeFileCopy = func(cancel <-chan struct{}, gotWFS *WFS, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, entryVersion, serverSideWholeFileCopyOutcome, error) { called = true if gotWFS != wfs { t.Fatalf("wfs = %p, want %p", gotWFS, wfs) @@ -131,7 +131,7 @@ func TestCopyFileRangeUsesServerSideWholeFileCopy(t *testing.T) { Mime: "text/plain; charset=utf-8", }, Content: []byte("hello"), - }, serverSideWholeFileCopyCommitted, nil + }, entryVersion{tsNs: 2000}, serverSideWholeFileCopyCommitted, nil } written, status := wfs.CopyFileRange(make(chan struct{}), &fuse.CopyFileRangeIn{ @@ -171,7 +171,7 @@ func TestCopyFileRangeDoesNotFallbackAfterCommittedServerCopyRefreshFailure(t *t srcInode := wfs.inodeToPath.Lookup(srcPath, 1, false, false, 0, true) dstInode := wfs.inodeToPath.Lookup(dstPath, 1, false, false, 0, true) - srcHandle := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{ + srcHandle, _ := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{ Name: "src.txt", Attributes: &filer_pb.FuseAttributes{ FileMode: 0100644, @@ -181,14 +181,14 @@ func TestCopyFileRangeDoesNotFallbackAfterCommittedServerCopyRefreshFailure(t *t Inode: srcInode, }, Content: []byte("hello"), - }) - dstHandle := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{ + }, 0, 0) + dstHandle, _ := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{ Name: "dst.txt", Attributes: &filer_pb.FuseAttributes{ FileMode: 0100600, Inode: dstInode, }, - }) + }, 0, 0) srcHandle.RememberPath(srcPath) dstHandle.RememberPath(dstPath) @@ -198,11 +198,11 @@ func TestCopyFileRangeDoesNotFallbackAfterCommittedServerCopyRefreshFailure(t *t performServerSideWholeFileCopy = originalCopy }() - performServerSideWholeFileCopy = func(cancel <-chan struct{}, gotWFS *WFS, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, serverSideWholeFileCopyOutcome, error) { + performServerSideWholeFileCopy = func(cancel <-chan struct{}, gotWFS *WFS, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, entryVersion, serverSideWholeFileCopyOutcome, error) { if gotWFS != wfs || copyRequest.srcPath != srcPath || copyRequest.dstPath != dstPath { t.Fatalf("unexpected server-side copy call: wfs=%p src=%q dst=%q", gotWFS, copyRequest.srcPath, copyRequest.dstPath) } - return nil, serverSideWholeFileCopyCommitted, errors.New("reload copied entry: transient filer read failure") + return nil, entryVersion{}, serverSideWholeFileCopyCommitted, errors.New("reload copied entry: transient filer read failure") } written, status := wfs.CopyFileRange(make(chan struct{}), &fuse.CopyFileRangeIn{ @@ -233,7 +233,7 @@ func TestCopyFileRangeDoesNotFallbackAfterCommittedServerCopyRefreshFailure(t *t t.Fatalf("destination content = %q, want %q", string(gotEntry.GetContent()), "hello") } - cachedEntry, err := wfs.metaCache.FindEntry(context.Background(), dstPath) + cachedEntry, _, err := wfs.metaCache.FindEntry(context.Background(), dstPath) if err != nil { t.Fatalf("metaCache find entry: %v", err) } @@ -253,7 +253,7 @@ func TestCopyFileRangeReturnsEIOForAmbiguousServerSideCopy(t *testing.T) { srcInode := wfs.inodeToPath.Lookup(srcPath, 1, false, false, 0, true) dstInode := wfs.inodeToPath.Lookup(dstPath, 1, false, false, 0, true) - srcHandle := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{ + srcHandle, _ := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{ Name: "src.txt", Attributes: &filer_pb.FuseAttributes{ FileMode: 0100644, @@ -261,14 +261,14 @@ func TestCopyFileRangeReturnsEIOForAmbiguousServerSideCopy(t *testing.T) { Inode: srcInode, }, Content: []byte("hello"), - }) - dstHandle := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{ + }, 0, 0) + dstHandle, _ := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{ Name: "dst.txt", Attributes: &filer_pb.FuseAttributes{ FileMode: 0100600, Inode: dstInode, }, - }) + }, 0, 0) srcHandle.RememberPath(srcPath) dstHandle.RememberPath(dstPath) @@ -278,11 +278,11 @@ func TestCopyFileRangeReturnsEIOForAmbiguousServerSideCopy(t *testing.T) { performServerSideWholeFileCopy = originalCopy }() - performServerSideWholeFileCopy = func(cancel <-chan struct{}, gotWFS *WFS, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, serverSideWholeFileCopyOutcome, error) { + performServerSideWholeFileCopy = func(cancel <-chan struct{}, gotWFS *WFS, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, entryVersion, serverSideWholeFileCopyOutcome, error) { if gotWFS != wfs || copyRequest.srcPath != srcPath || copyRequest.dstPath != dstPath { t.Fatalf("unexpected server-side copy call: wfs=%p src=%q dst=%q", gotWFS, copyRequest.srcPath, copyRequest.dstPath) } - return nil, serverSideWholeFileCopyAmbiguous, errors.New("transport timeout after request dispatch") + return nil, entryVersion{}, serverSideWholeFileCopyAmbiguous, errors.New("transport timeout after request dispatch") } written, status := wfs.CopyFileRange(make(chan struct{}), &fuse.CopyFileRangeIn{ @@ -346,7 +346,7 @@ func newCopyRangeTestWFSWithMetaCache(t *testing.T) *WFS { func(path util.FullPath) bool { return wfs.inodeToPath.IsChildrenCached(path) }, - func(util.FullPath, *filer_pb.Entry) {}, + func(meta_cache.EntryInvalidation) {}, nil, ) t.Cleanup(func() { diff --git a/weed/mount/weedfs_file_mkrm.go b/weed/mount/weedfs_file_mkrm.go index 57fda67ab..a7090b403 100644 --- a/weed/mount/weedfs_file_mkrm.go +++ b/weed/mount/weedfs_file_mkrm.go @@ -40,7 +40,7 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o entryFullPath := dirFullPath.Child(name) var inode uint64 - newEntry, code := wfs.maybeLoadEntry(entryFullPath) + newEntry, _, code := wfs.maybeLoadEntry(entryFullPath) if code == fuse.OK { if newEntry == nil || newEntry.Attributes == nil { return fuse.EIO @@ -76,7 +76,7 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o if code == fuse.Status(syscall.EEXIST) && in.Flags&syscall.O_EXCL == 0 { // Race: another process created the file between our check and create. // Reopen the winner's entry. - newEntry, code = wfs.maybeLoadEntry(entryFullPath) + newEntry, _, code = wfs.maybeLoadEntry(entryFullPath) if code != fuse.OK { return code } @@ -110,7 +110,12 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o // For deferred creates, bypass AcquireHandle (which calls maybeReadEntry // and would fail since the entry is not yet on the filer or in the meta cache). // We already have the entry from createRegularFile, so create the handle directly. - fileHandle := wfs.fhMap.AcquireFileHandle(wfs, inode, newEntry) + fileHandle, existed := wfs.fhMap.AcquireFileHandle(wfs, inode, newEntry, 0, 0) + if existed { + // A create is authoritative for the entry it just made, so it takes + // effect even on a handle that outlived a previous incarnation. + fileHandle.SetEntry(newEntry) + } fileHandle.RememberPath(entryFullPath) // Mark dirty so the deferred filer create happens on Flush, // even if the file is closed without any writes. @@ -181,7 +186,7 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin } entryFullPath := dirFullPath.Child(name) - entry, code := wfs.maybeLoadEntry(entryFullPath) + entry, _, code := wfs.maybeLoadEntry(entryFullPath) if code != fuse.OK { if code == fuse.ENOENT { return fuse.OK @@ -210,7 +215,7 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin // maybeLoadEntry path above. Do not fall back to the stale // pre-lock snapshot: proceeding with its HardLinkCounter would // reintroduce the stale-base update the lock is meant to prevent. - fresh, freshCode := wfs.maybeLoadEntry(entryFullPath) + fresh, _, freshCode := wfs.maybeLoadEntry(entryFullPath) if freshCode == fuse.ENOENT { return fuse.OK } @@ -221,7 +226,7 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin } // POSIX: enforce sticky bit on the parent directory. - if dirEntry, dirCode := wfs.maybeLoadEntry(dirFullPath); dirCode == fuse.OK && dirEntry != nil && dirEntry.Attributes != nil { + if dirEntry, _, dirCode := wfs.maybeLoadEntry(dirFullPath); dirCode == fuse.OK && dirEntry != nil && dirEntry.Attributes != nil { targetUid := uint32(0) if entry != nil && entry.Attributes != nil { targetUid = entry.Attributes.Uid @@ -324,7 +329,7 @@ func (wfs *WFS) createRegularFile(dirFullPath util.FullPath, name string, mode u // SkipCheckParentDirectory, so this is the only parent check). With // default_permissions the kernel already verified write+search on it before // Create/Mknod, so skip only the mode-bit check and its group lookup. - parentEntry, parentStatus := wfs.maybeLoadEntry(dirFullPath) + parentEntry, _, parentStatus := wfs.maybeLoadEntry(dirFullPath) if parentStatus != fuse.OK { return 0, nil, parentStatus } @@ -345,7 +350,7 @@ func (wfs *WFS) createRegularFile(dirFullPath util.FullPath, name string, mode u entryFullPath := dirFullPath.Child(name) if !skipExistenceCheck { - if _, status := wfs.maybeLoadEntry(entryFullPath); status == fuse.OK { + if _, _, status := wfs.maybeLoadEntry(entryFullPath); status == fuse.OK { return 0, nil, fuse.Status(syscall.EEXIST) } else if status != fuse.ENOENT { return 0, nil, status @@ -377,7 +382,7 @@ func (wfs *WFS) createRegularFile(dirFullPath util.FullPath, name string, mode u // create checks, stat, readdir). // We use InsertEntry directly instead of applyLocalMetadataEvent to avoid // triggering directory hot-threshold eviction that would wipe the entry. - if insertErr := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(string(dirFullPath), newEntry)); insertErr != nil { + if insertErr := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(string(dirFullPath), newEntry), 0); insertErr != nil { glog.Warningf("createFile %s: insert local entry: %v", entryFullPath, insertErr) } wfs.inodeToPath.TouchDirectory(dirFullPath) @@ -505,6 +510,7 @@ func (wfs *WFS) truncateEntry(entryFullPath util.FullPath, entry *filer_pb.Entry fhActiveLock := fh.wfs.fhLockTable.AcquireLock("truncateEntry", fh.fh, util.ExclusiveLock) fh.ResetDirtyPages() fh.SetEntry(entry) + fh.setAuthoritativeBase(proto.Clone(entry).(*filer_pb.Entry)) fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock) } } diff --git a/weed/mount/weedfs_file_mkrm_test.go b/weed/mount/weedfs_file_mkrm_test.go index ba3828a9c..6a8b34312 100644 --- a/weed/mount/weedfs_file_mkrm_test.go +++ b/weed/mount/weedfs_file_mkrm_test.go @@ -128,7 +128,7 @@ func newCreateTestWFS(t *testing.T) (*WFS, *createEntryTestServer) { func(path util.FullPath) bool { return wfs.inodeToPath.IsChildrenCached(path) }, - func(util.FullPath, *filer_pb.Entry) {}, + func(meta_cache.EntryInvalidation) {}, nil, ) wfs.inodeToPath.MarkChildrenCached(root) @@ -267,7 +267,7 @@ func TestTruncateEntryClearsDirtyPagesForOpenHandle(t *testing.T) { }, } - fh := wfs.fhMap.AcquireFileHandle(wfs, inode, entry) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, entry, 0, 0) fh.RememberPath(fullPath) if err := fh.dirtyPages.AddPage(0, []byte("hello"), true, time.Now().UnixNano()); err != nil { @@ -315,7 +315,7 @@ func TestAccessChecksPermissions(t *testing.T) { fullPath := util.FullPath("/visible.txt") inode := wfs.inodeToPath.Lookup(fullPath, 1, false, false, 0, true) - handle := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + handle, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ Name: "visible.txt", Attributes: &filer_pb.FuseAttributes{ FileMode: 0o640, @@ -323,7 +323,7 @@ func TestAccessChecksPermissions(t *testing.T) { Gid: 456, Inode: inode, }, - }) + }, 0, 0) handle.RememberPath(fullPath) if status := wfs.Access(make(chan struct{}), &fuse.AccessIn{ @@ -464,7 +464,7 @@ func TestCreateExistingFileIgnoresQuotaPreflight(t *testing.T) { Gid: 456, }, } - if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/", entry)); err != nil { + if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/", entry), 0); err != nil { t.Fatalf("InsertEntry: %v", err) } wfs.inodeToPath.Lookup(util.FullPath("/existing.txt"), entry.Attributes.Crtime, false, false, entry.Attributes.Inode, true) @@ -523,7 +523,7 @@ func TestAcquireHandleHonorsDefaultPermissions(t *testing.T) { Gid: 456, }, } - if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/", entry)); err != nil { + if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/", entry), 0); err != nil { t.Fatalf("InsertEntry: %v", err) } inode := wfs.inodeToPath.Lookup(util.FullPath("/secret.txt"), entry.Attributes.Crtime, false, false, entry.Attributes.Inode, true) diff --git a/weed/mount/weedfs_file_sync.go b/weed/mount/weedfs_file_sync.go index 57712d8a8..376426050 100644 --- a/weed/mount/weedfs_file_sync.go +++ b/weed/mount/weedfs_file_sync.go @@ -258,6 +258,9 @@ func (wfs *WFS) flushMetadataToFiler(ctx context.Context, fh *FileHandle, dir, n SkipCheckParentDirectory: true, } + // Snapshot with local ids before the request mapping mutates the clone: + // on ack this becomes the handle's base, judged against future events. + baseSnapshot := proto.Clone(requestEntry).(*filer_pb.Entry) wfs.mapPbIdFromLocalToFiler(request.Entry) resp, err := wfs.streamCreateEntry(ctx, request) @@ -269,7 +272,15 @@ func (wfs *WFS) flushMetadataToFiler(ctx context.Context, fh *FileHandle, dir, n event := resp.GetMetadataEvent() if event == nil { event = metadataUpdateEvent(string(dir), request.Entry) + if event != nil { + event.TsNs = ackVersionTsNs(resp) + } } + // The filer acknowledged this state at the event's log position (or, for + // a no-op create, at the response's log position); older queued + // subscription events must not roll the handle back. + fh.setAuthoritativeBase(baseSnapshot) + fh.advanceEntryVersion(ackVersionTsNs(resp), resp.GetLogSignature()) if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil { glog.Warningf("flush %s: best-effort metadata apply failed: %v", fileFullPath, applyErr) wfs.inodeToPath.InvalidateChildrenCache(util.FullPath(dir)) diff --git a/weed/mount/weedfs_filehandle.go b/weed/mount/weedfs_filehandle.go index 7fe61d1db..7009d982d 100644 --- a/weed/mount/weedfs_filehandle.go +++ b/weed/mount/weedfs_filehandle.go @@ -7,7 +7,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" - "github.com/seaweedfs/seaweedfs/weed/util" ) func (wfs *WFS) AcquireHandle(inode uint64, flags, uid, gid uint32) (fileHandle *FileHandle, status fuse.Status) { @@ -17,42 +16,70 @@ func (wfs *WFS) AcquireHandle(inode uint64, flags, uid, gid uint32) (fileHandle // data that was just written asynchronously. wfs.waitForPendingAsyncFlush(inode) - var entry *filer_pb.Entry - var path util.FullPath - path, _, entry, status = wfs.maybeReadEntry(inode) - if status == fuse.OK { - if wormEnforced, _ := wfs.wormEnforcedForEntry(path, entry); wormEnforced && flags&fuse.O_ANYWRITE != 0 { - return nil, fuse.EPERM - } - // Check unix permission bits for the requested access mode. With - // default_permissions the kernel already enforced them before this - // open, so the check (and its supplementary-group lookup) is redundant. - if !wfs.option.DefaultPermissions && entry != nil && entry.Attributes != nil { - fileUid, fileGid := entry.Attributes.Uid, entry.Attributes.Gid - if wfs.option.UidGidMapper != nil { - fileUid, fileGid = wfs.option.UidGidMapper.FilerToLocal(fileUid, fileGid) - } - if mask := openFlagsToAccessMask(flags); mask != 0 && !hasAccess(uid, gid, fileUid, fileGid, entry.Attributes.FileMode, mask) { - return nil, fuse.EACCES - } - } - // need to AcquireFileHandle again to ensure correct handle counter - fileHandle = wfs.fhMap.AcquireFileHandle(wfs, inode, entry) - fileHandle.RememberPath(path) + path, pathStatus := wfs.inodeToPath.GetPath(inode) + if pathStatus != fuse.OK { + return nil, pathStatus + } - // Acquire distributed lock for write opens. The lock is held with - // auto-renewal until the file handle is released (close). - // Use the filer path as the lock key since inode numbers are - // assigned per-mount and differ across mount instances. - if wfs.lockClient != nil && flags&fuse.O_ANYWRITE != 0 && fileHandle.dlmLock == nil { - owner := fmt.Sprintf("mount-%d", wfs.signature) - fileHandle.dlmLock = wfs.lockClient.NewBlockingLongLivedLock( - string(path), owner, lock_manager.LiveLockTTL, - ) - glog.V(1).Infof("DLM lock acquired for %s", path) + // A fresh lookup returns the entry with the filer log position it + // reflects; the handle takes that entry and version as one decision in + // AcquireFileHandle, so a slower concurrent opener cannot + // overwrite a newer install. An existing handle keeps its own entry and + // version — they did not come from this lookup. + var entry *filer_pb.Entry + var entryVersionTsNs entryVersion + freshLookup := false + if existingFh, found := wfs.fhMap.FindFileHandle(inode); found { + entry = existingFh.UpdateEntry(func(entry *filer_pb.Entry) { + if entry != nil && existingFh.entry.Attributes == nil { + entry.Attributes = &filer_pb.FuseAttributes{} + } + }) + entryVersionTsNs = entryVersion{tsNs: existingFh.entryVersionTsNs.Load(), signature: existingFh.entryVersionSignature.Load()} + } else { + entry, entryVersionTsNs, status = wfs.maybeLoadEntry(path) + if status != fuse.OK { + return nil, status + } + freshLookup = true + } + if wormEnforced, _ := wfs.wormEnforcedForEntry(path, entry); wormEnforced && flags&fuse.O_ANYWRITE != 0 { + return nil, fuse.EPERM + } + // Check unix permission bits for the requested access mode. With + // default_permissions the kernel already enforced them before this + // open, so the check (and its supplementary-group lookup) is redundant. + if !wfs.option.DefaultPermissions && entry != nil && entry.Attributes != nil { + fileUid, fileGid := entry.Attributes.Uid, entry.Attributes.Gid + if wfs.option.UidGidMapper != nil { + fileUid, fileGid = wfs.option.UidGidMapper.FilerToLocal(fileUid, fileGid) + } + if mask := openFlagsToAccessMask(flags); mask != 0 && !hasAccess(uid, gid, fileUid, fileGid, entry.Attributes.FileMode, mask) { + return nil, fuse.EACCES } } - return + // need to AcquireFileHandle again to ensure correct handle counter + var existed bool + fileHandle, existed = wfs.fhMap.AcquireFileHandle(wfs, inode, entry, entryVersionTsNs.tsNs, entryVersionTsNs.signature) + fileHandle.RememberPath(path) + if existed && freshLookup { + // Another opener created the handle while our lookup was in flight; + // install only state that provably improves on what it holds. + fileHandle.installAckedEntry(entry, entryVersionTsNs.tsNs, entryVersionTsNs.signature) + } + + // Acquire distributed lock for write opens. The lock is held with + // auto-renewal until the file handle is released (close). + // Use the filer path as the lock key since inode numbers are + // assigned per-mount and differ across mount instances. + if wfs.lockClient != nil && flags&fuse.O_ANYWRITE != 0 && fileHandle.dlmLock == nil { + owner := fmt.Sprintf("mount-%d", wfs.signature) + fileHandle.dlmLock = wfs.lockClient.NewBlockingLongLivedLock( + string(path), owner, lock_manager.LiveLockTTL, + ) + glog.V(1).Infof("DLM lock acquired for %s", path) + } + return fileHandle, fuse.OK } // ReleaseHandle is called from FUSE Release. For handles with a pending diff --git a/weed/mount/weedfs_invalidate_open_handle_test.go b/weed/mount/weedfs_invalidate_open_handle_test.go new file mode 100644 index 000000000..af991e10c --- /dev/null +++ b/weed/mount/weedfs_invalidate_open_handle_test.go @@ -0,0 +1,2077 @@ +package mount + +import ( + "context" + "net" + "path/filepath" + "strconv" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc/metadata" + + "github.com/seaweedfs/go-fuse/v2/fuse" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/mount/meta_cache" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func newInvalidateTestWFS(t *testing.T) *WFS { + t.Helper() + + // Map filer uid 2000 to local uid 1000 to verify the event entry gets the + // same id translation a filer lookup would apply. + uidGidMapper, err := meta_cache.NewUidGidMapper("1000:2000", "") + if err != nil { + t.Fatalf("create uid/gid mapper: %v", err) + } + + root := util.FullPath("/") + wfs := &WFS{ + signature: 1, + inodeToPath: NewInodeToPath(root, 0), + fhMap: NewFileHandleToInode(), + fhLockTable: util.NewLockTable[FileHandleId](), + hardLinkLockTable: util.NewLockTable[string](), + option: &Option{ + ChunkSizeLimit: 1024, + ConcurrentReaders: 1, + VolumeServerAccess: "filerProxy", + // Nothing listens here: a transient filer failure at any point + // must never leave a handle permanently stale. + FilerAddresses: []pb.ServerAddress{ + pb.NewServerAddressWithGrpcPort("127.0.0.1:1", 1), + }, + GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), + UidGidMapper: uidGidMapper, + }, + } + + wfs.metaCache = meta_cache.NewMetaCache( + filepath.Join(t.TempDir(), "meta"), + uidGidMapper, + root, + false, + func(path util.FullPath) { wfs.inodeToPath.MarkChildrenCached(path) }, + func(path util.FullPath) bool { return wfs.inodeToPath.IsChildrenCached(path) }, + wfs.invalidateOpenFileHandle, + nil, + ) + t.Cleanup(wfs.metaCache.Shutdown) + + return wfs +} + +type fakeFilerServer struct { + filer_pb.UnimplementedSeaweedFilerServer + listSnapshotTrailerTsNs int64 // when set, ListEntries returns empty with this trailer snapshot + lookupSize uint64 + lookupLogTsNs int64 + lookupSignature int32 // filer signature the lookup fence carries + lookupSize2 uint64 // when set, served to the second and later lookups + lookupLogTsNs2 int64 + cacheSize uint64 + cacheLogTsNs int64 + cacheUid uint32 // filer-side uid the cache response carries + updateEventless bool // UpdateEntry acks like a no-change update: no event, log position only + updateLogTsNs int64 + lookupCalls atomic.Int32 + lookupStarted chan struct{} // closed when the first lookup arrives + lookupGate chan struct{} // first lookup waits here when non-nil +} + +func (s *fakeFilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) { + call := s.lookupCalls.Add(1) + if s.lookupGate != nil && call == 1 { + close(s.lookupStarted) + <-s.lookupGate + } + size, logTsNs := s.lookupSize, s.lookupLogTsNs + if call > 1 && s.lookupSize2 != 0 { + size, logTsNs = s.lookupSize2, s.lookupLogTsNs2 + } + return &filer_pb.LookupDirectoryEntryResponse{ + Entry: &filer_pb.Entry{ + Name: req.Name, + Attributes: &filer_pb.FuseAttributes{FileSize: size, FileMode: 0100644}, + }, + LogTsNs: logTsNs, + LogSignature: s.lookupSignature, + }, nil +} + +func (s *fakeFilerServer) CacheRemoteObjectToLocalCluster(ctx context.Context, req *filer_pb.CacheRemoteObjectToLocalClusterRequest) (*filer_pb.CacheRemoteObjectToLocalClusterResponse, error) { + // No MetadataEvent: the object was already cached by another client. + return &filer_pb.CacheRemoteObjectToLocalClusterResponse{ + Entry: &filer_pb.Entry{ + Name: req.Name, + Attributes: &filer_pb.FuseAttributes{FileSize: s.cacheSize, FileMode: 0100644, Uid: s.cacheUid}, + }, + LogTsNs: s.cacheLogTsNs, + }, nil +} + +func (s *fakeFilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream filer_pb.SeaweedFiler_ListEntriesServer) error { + if s.listSnapshotTrailerTsNs != 0 { + stream.SetTrailer(metadata.Pairs(filer_pb.ListSnapshotTsNsTrailerKey, strconv.FormatInt(s.listSnapshotTrailerTsNs, 10))) + } + return nil +} + +func (s *fakeFilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) { + return &filer_pb.CreateEntryResponse{ + MetadataEvent: &filer_pb.SubscribeMetadataResponse{ + Directory: req.Directory, + TsNs: 3000, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: req.Entry.Name}, + NewEntry: req.Entry, + NewParentPath: req.Directory, + }, + }, + }, nil +} + +func (s *fakeFilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) { + if s.updateEventless { + return &filer_pb.UpdateEntryResponse{LogTsNs: s.updateLogTsNs}, nil + } + return &filer_pb.UpdateEntryResponse{ + MetadataEvent: &filer_pb.SubscribeMetadataResponse{ + Directory: req.Directory, + TsNs: 2000, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: req.Entry.Name}, + NewEntry: req.Entry, + NewParentPath: req.Directory, + }, + }, + }, nil +} + +// startFakeFiler serves fake on a local port and points wfs at it. +func startFakeFiler(t *testing.T, wfs *WFS, fake *fakeFilerServer) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + server := pb.NewGrpcServer() + filer_pb.RegisterSeaweedFilerServer(server, fake) + go server.Serve(listener) + t.Cleanup(server.Stop) + wfs.option.FilerAddresses = []pb.ServerAddress{ + pb.NewServerAddressWithGrpcPort("127.0.0.1:1", listener.Addr().(*net.TCPAddr).Port), + } +} + +func updateEventFor(name string, size uint64, tsNs int64) *filer_pb.SubscribeMetadataResponse { + return &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: tsNs, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: name}, + NewEntry: &filer_pb.Entry{ + Name: name, + Attributes: &filer_pb.FuseAttributes{FileSize: size}, + }, + NewParentPath: "/dir", + }, + } +} + +// An update event must refresh an open file handle from the entry the event +// itself carries: a second lookup can fail transiently, and with the +// subscription cursor already advanced, the handle would stay pinned to its +// old entry until an unrelated event arrives. +func TestUpdateEventRefreshesOpenFileHandle(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 88}, + }, 0, 0) + + updateResp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1000, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + NewEntry: &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 180020, Uid: 2000}, + Chunks: []*filer_pb.FileChunk{{FileId: "1,ab1", Size: 180020}}, + }, + NewParentPath: "/dir", + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateResp, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply update event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + entry := fh.GetEntry().GetEntry() + if entry.Attributes.FileSize != 180020 { + t.Fatalf("open handle file size = %d, want 180020", entry.Attributes.FileSize) + } + if len(entry.GetChunks()) != 1 { + t.Fatalf("open handle chunks = %d, want 1", len(entry.GetChunks())) + } + if entry.Attributes.Uid != 1000 { + t.Fatalf("open handle uid = %d, want filer uid 2000 mapped to local 1000", entry.Attributes.Uid) + } + + // A delete leaves the handle with its last entry so unlinked-but-open + // reads keep working. + deleteResp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1100, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), deleteResp, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply delete event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 180020 { + t.Fatalf("open handle file size after delete = %d, want 180020", size) + } +} + +// A queued invalidation must not roll the handle back over newer state a +// local flush installed while the event sat in the queue: for a cached +// directory the store entry is the ordered merge of both, and its version +// outranks the event's. +func TestQueuedEventDoesNotRollBackNewerLocalState(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/")) + wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false) + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir")) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 88}, + }, 0, 0) + + // Hold the handle lock so the queued invalidation cannot apply yet. + testLock := wfs.fhLockTable.AcquireLock("test", fh.fh, util.ExclusiveLock) + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 100, 1000), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + wfs.fhLockTable.ReleaseLock(fh.fh, testLock) + t.Fatalf("apply subscriber event: %v", err) + } + + // A local flush lands after the event was queued: newer state goes into + // the handle and, via the local apply, into the local store. + fh.SetEntry(&filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + }) + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 200, 2000), meta_cache.LocalMetadataResponseApplyOptions); err != nil { + wfs.fhLockTable.ReleaseLock(fh.fh, testLock) + t.Fatalf("apply local event: %v", err) + } + wfs.fhLockTable.ReleaseLock(fh.fh, testLock) + + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("open handle file size = %d, want 200 (queued size-100 event must not roll back the newer local state)", size) + } +} + +// During a directory build, an event touching the building directory is +// buffered: its store write is deferred while its invalidation runs against +// a mid-build store. Build completion versions the directory at the listing +// snapshot and re-invalidates, so the handle lands on the completed state. +func TestBufferedBuildEventReinvalidatesOnCompletion(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/")) + wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 88}, + }, 0, 0) + + if err := wfs.metaCache.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil { + t.Fatalf("begin build: %v", err) + } + + // Covered by the upcoming listing snapshot (TsNs 900 <= snapshot 1000); + // its immediate invalidation runs while the directory is read-through, + // so the handle picks up the event's state. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 100, 900), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply buffered event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 100 { + t.Fatalf("open handle file size mid-build = %d, want 100 (event state)", size) + } + + // The listing then inserts the newer entry the snapshot already covers, + // through the same batch path a real build uses. + if err := wfs.metaCache.InsertEntry(context.Background(), &filer.Entry{ + FullPath: "/dir/file", + Attr: filer.Attr{ + Crtime: time.Unix(1, 0), + Mtime: time.Unix(1, 0), + Mode: 0100644, + FileSize: 300, + }, + }, 2000); err != nil { + t.Fatalf("insert listing entry: %v", err) + } + + if err := wfs.metaCache.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 1000); err != nil { + t.Fatalf("complete build: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 300 { + t.Fatalf("open handle file size after build completion = %d, want 300 (snapshot-covered event must re-invalidate)", size) + } +} + +// A store hit only resolves an invalidation when the parent directory is +// cached. An uncached parent receives no store writes, so a leftover entry +// there is stale and must not mask the event. +func TestUncachedDirStaleStoreEntryDoesNotMaskEvent(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 88}, + }, 0, 0) + + // Leftover store entry under a parent that is not children-cached. + if err := wfs.metaCache.InsertEntry(context.Background(), &filer.Entry{ + FullPath: "/dir/file", + Attr: filer.Attr{ + Crtime: time.Unix(1, 0), + Mtime: time.Unix(1, 0), + Mode: 0100644, + FileSize: 88, + }, + }, 0); err != nil { + t.Fatalf("insert stale entry: %v", err) + } + + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 180020, 1000), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply update event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 180020 { + t.Fatalf("open handle file size = %d, want 180020 (stale store entry must not mask the event)", size) + } +} + +// In a read-through directory neither a local flush nor the event reaches the +// local store, so ordering falls to the versions: an event at or before the +// handle's last filer-acknowledged mutation is old news and must not roll the +// handle back. +func TestQueuedEventOlderThanFlushedStateIsIgnored(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 88}, + }, 0, 0) + + // Hold the handle lock so the queued invalidation cannot apply yet. + testLock := wfs.fhLockTable.AcquireLock("test", fh.fh, util.ExclusiveLock) + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 100, 1000), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + wfs.fhLockTable.ReleaseLock(fh.fh, testLock) + t.Fatalf("apply subscriber event: %v", err) + } + + // A local flush lands: the filer acknowledged it with a later log + // timestamp than the queued event. + fh.SetEntry(&filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + }) + fh.advanceEntryVersion(2000, 0) + wfs.fhLockTable.ReleaseLock(fh.fh, testLock) + + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("open handle file size = %d, want 200 (event at TsNs 1000 predates the flush at 2000)", size) + } +} + +// saveEntry (truncate, setattr) must advance the open handle's version from +// the acknowledged mutation's log timestamp, or an older queued event rolls +// the mutation back in a read-through directory. +func TestSaveEntryKeepsOpenHandleAheadOfOlderEvents(t *testing.T) { + wfs := newInvalidateTestWFS(t) + startFakeFiler(t, wfs, &fakeFilerServer{}) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 88}, + }, 0, 0) + + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 100, 1000), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply subscriber event: %v", err) + } + + // A truncate-style mutation: the filer acknowledges it at TsNs 2000 and + // installs the acknowledged state into the handle. Whichever order the + // queued event and the acknowledgment reach the handle, the newer + // acknowledged state wins. + saved := &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + } + if code := wfs.saveEntry(util.FullPath("/dir/file"), saved); code != fuse.OK { + t.Fatalf("saveEntry status = %v, want OK", code) + } + + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("open handle file size = %d, want 200 (saveEntry at TsNs 2000 outranks the queued event at 1000)", size) + } +} + +// A handle opened while an older event sits in the invalidation queue takes +// its version from the lookup response's log position, which covers every +// event the filer had committed — including the queued one. +func TestQueuedEventDoesNotRollBackHandleOpenedAfterEnqueue(t *testing.T) { + wfs := newInvalidateTestWFS(t) + startFakeFiler(t, wfs, &fakeFilerServer{lookupSize: 200, lookupLogTsNs: 2000}) + + // Stall the single invalidation worker on an unrelated handle's lock so + // queued events outlive the open below. + blockerInode := wfs.inodeToPath.Lookup(util.FullPath("/other/blocker"), time.Now().Unix(), false, false, 0, false) + blockerFh, _ := wfs.fhMap.AcquireFileHandle(wfs, blockerInode, &filer_pb.Entry{ + Name: "blocker", + Attributes: &filer_pb.FuseAttributes{FileSize: 1}, + }, 0, 0) + blockerLock := wfs.fhLockTable.AcquireLock("test", blockerFh.fh, util.ExclusiveLock) + blockerReleased := false + releaseBlocker := func() { + if !blockerReleased { + blockerReleased = true + wfs.fhLockTable.ReleaseLock(blockerFh.fh, blockerLock) + } + } + defer releaseBlocker() + blockerEvent := &filer_pb.SubscribeMetadataResponse{ + Directory: "/other", + TsNs: 500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "blocker"}, + NewEntry: &filer_pb.Entry{ + Name: "blocker", + Attributes: &filer_pb.FuseAttributes{FileSize: 2}, + }, + NewParentPath: "/other", + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), blockerEvent, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply blocker event: %v", err) + } + + // The event predates the open below. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 100, 1000), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply subscriber event: %v", err) + } + + // Open now: the lookup reaches the filer, which serves the newer + // size-200 state versioned at log position 2000. + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, status := wfs.AcquireHandle(inode, 0, 0, 0) + if status != fuse.OK { + t.Fatalf("AcquireHandle status = %v, want OK", status) + } + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("opened handle file size = %d, want 200", size) + } + + releaseBlocker() + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("open handle file size = %d, want 200 (event queued before the open must not roll it back)", size) + } +} + +// An event buffered for a building directory keeps its queued invalidation +// even when the build is aborted. A handle opened after the abort is fenced +// by its lookup response's log position, which covers the committed event. +func TestAbortedBuildEventDoesNotRollBackLaterOpen(t *testing.T) { + wfs := newInvalidateTestWFS(t) + startFakeFiler(t, wfs, &fakeFilerServer{lookupSize: 200, lookupLogTsNs: 2000}) + + blockerInode := wfs.inodeToPath.Lookup(util.FullPath("/other/blocker"), time.Now().Unix(), false, false, 0, false) + blockerFh, _ := wfs.fhMap.AcquireFileHandle(wfs, blockerInode, &filer_pb.Entry{ + Name: "blocker", + Attributes: &filer_pb.FuseAttributes{FileSize: 1}, + }, 0, 0) + blockerLock := wfs.fhLockTable.AcquireLock("test", blockerFh.fh, util.ExclusiveLock) + blockerReleased := false + releaseBlocker := func() { + if !blockerReleased { + blockerReleased = true + wfs.fhLockTable.ReleaseLock(blockerFh.fh, blockerLock) + } + } + defer releaseBlocker() + blockerEvent := &filer_pb.SubscribeMetadataResponse{ + Directory: "/other", + TsNs: 500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "blocker"}, + NewEntry: &filer_pb.Entry{ + Name: "blocker", + Attributes: &filer_pb.FuseAttributes{FileSize: 2}, + }, + NewParentPath: "/other", + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), blockerEvent, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply blocker event: %v", err) + } + + if err := wfs.metaCache.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil { + t.Fatalf("begin build: %v", err) + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 100, 1000), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply buffered event: %v", err) + } + if err := wfs.metaCache.AbortDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil { + t.Fatalf("abort build: %v", err) + } + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, status := wfs.AcquireHandle(inode, 0, 0, 0) + if status != fuse.OK { + t.Fatalf("AcquireHandle status = %v, want OK", status) + } + + releaseBlocker() + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("open handle file size = %d, want 200 (aborted-build event must not roll back the open)", size) + } +} + +// An event applied while the open's lookup is in flight is committed on the +// filer before the lookup is served, so the response's log position covers +// it and the fresh handle is not rolled back. +func TestEventDuringOpenLookupIsFenced(t *testing.T) { + wfs := newInvalidateTestWFS(t) + fake := &fakeFilerServer{ + lookupSize: 200, + lookupLogTsNs: 2000, + lookupStarted: make(chan struct{}), + lookupGate: make(chan struct{}), + } + startFakeFiler(t, wfs, fake) + + blockerInode := wfs.inodeToPath.Lookup(util.FullPath("/other/blocker"), time.Now().Unix(), false, false, 0, false) + blockerFh, _ := wfs.fhMap.AcquireFileHandle(wfs, blockerInode, &filer_pb.Entry{ + Name: "blocker", + Attributes: &filer_pb.FuseAttributes{FileSize: 1}, + }, 0, 0) + blockerLock := wfs.fhLockTable.AcquireLock("test", blockerFh.fh, util.ExclusiveLock) + blockerReleased := false + releaseBlocker := func() { + if !blockerReleased { + blockerReleased = true + wfs.fhLockTable.ReleaseLock(blockerFh.fh, blockerLock) + } + } + defer releaseBlocker() + blockerEvent := &filer_pb.SubscribeMetadataResponse{ + Directory: "/other", + TsNs: 500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "blocker"}, + NewEntry: &filer_pb.Entry{ + Name: "blocker", + Attributes: &filer_pb.FuseAttributes{FileSize: 2}, + }, + NewParentPath: "/other", + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), blockerEvent, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply blocker event: %v", err) + } + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + type openResult struct { + fh *FileHandle + status fuse.Status + } + opened := make(chan openResult, 1) + go func() { + fh, status := wfs.AcquireHandle(inode, 0, 0, 0) + opened <- openResult{fh, status} + }() + + // While the open's lookup is blocked in the filer, an event lands and + // its invalidation is queued behind the blocker. + <-fake.lookupStarted + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 100, 1500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply mid-lookup event: %v", err) + } + close(fake.lookupGate) + + result := <-opened + if result.status != fuse.OK { + t.Fatalf("AcquireHandle status = %v, want OK", result.status) + } + + releaseBlocker() + wfs.metaCache.WaitForEntryInvalidations() + + if size := result.fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("open handle file size = %d, want 200 (mid-lookup event must be fenced)", size) + } +} + +// The remote-cache response versions the freshly loaded state by the caching +// event or, for an already-cached object, by the response's log position — +// covering events committed but not yet delivered, regardless of which filer +// answered after a failover. +func TestRemoteCacheResponseVersionFencesUndeliveredEvents(t *testing.T) { + wfs := newInvalidateTestWFS(t) + fake := &fakeFilerServer{cacheSize: 200, cacheLogTsNs: 2000} + startFakeFiler(t, wfs, fake) + // First filer is unreachable; WithFilerClient fails over to the fake. + live := wfs.option.FilerAddresses[0] + wfs.option.FilerAddresses = []pb.ServerAddress{ + pb.NewServerAddressWithGrpcPort("127.0.0.1:1", 1), + live, + } + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 88}, + }, 0, 0) + + if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil { + t.Fatalf("downloadRemoteEntry: %v", err) + } + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("downloaded file size = %d, want 200", size) + } + + // An event committed before the download (TsNs 1500 < 2000) but + // delivered only now must not roll the handle back. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 100, 1500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply late event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("open handle file size = %d, want 200 (response-versioned state must fence the undelivered event)", size) + } +} + +// A no-change update returns success without an event; the response's log +// position must still fence the handle, or a delayed event already reflected +// by the confirmed state rolls it back. +func TestEventlessSaveAckStillFencesOlderEvents(t *testing.T) { + wfs := newInvalidateTestWFS(t) + startFakeFiler(t, wfs, &fakeFilerServer{updateEventless: true, updateLogTsNs: 2000}) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + }, 0, 0) + + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 100, 1000), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply subscriber event: %v", err) + } + + // The save confirms the handle's state without changing it: no event, + // only the acknowledged log position, which installs the confirmed + // state over whatever the queued event may have applied first. + saved := &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + } + if code := wfs.saveEntry(util.FullPath("/dir/file"), saved); code != fuse.OK { + t.Fatalf("saveEntry status = %v, want OK", code) + } + + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("open handle file size = %d, want 200 (no-op ack at log position 2000 outranks the queued event at 1000)", size) + } +} + +// A local mutation ack for one path must not inflate the version of store +// reads for other paths: the subscription may still owe those paths older +// events, and an inflated fence would discard them permanently. +func TestLocalAckDoesNotFenceUnrelatedDelayedEvents(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/")) + wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false) + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir")) + + if err := wfs.metaCache.InsertEntry(context.Background(), &filer.Entry{ + FullPath: "/dir/file", + Attr: filer.Attr{ + Crtime: time.Unix(1, 0), + Mtime: time.Unix(1, 0), + Mode: 0100644, + FileSize: 88, + }, + }, 0); err != nil { + t.Fatalf("insert cached entry: %v", err) + } + + // A local flush of an unrelated file acknowledges filer position 2000. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("other", 10, 2000), meta_cache.LocalMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply local event: %v", err) + } + // Open the file: the cached-store read must not claim position 2000 — + // the local ack versioned only its own path. + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, status := wfs.AcquireHandle(inode, 0, 0, 0) + if status != fuse.OK { + t.Fatalf("AcquireHandle status = %v, want OK", status) + } + + // The delayed subscription event for this file must still land. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 150, 1500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply delayed event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 150 { + t.Fatalf("open handle file size = %d, want 150 (unrelated local ack must not fence this file's event)", size) + } +} + +// Two concurrent first opens race: the slower opener's older lookup result +// must not overwrite the newer entry the faster opener installed, while the +// monotonic version keeps the newer timestamp — entry and version are one +// decision under the handle map lock. +func TestSlowerConcurrentOpenDoesNotOverwriteNewerHandle(t *testing.T) { + wfs := newInvalidateTestWFS(t) + fake := &fakeFilerServer{ + lookupSize: 100, + lookupLogTsNs: 1000, + lookupSize2: 200, + lookupLogTsNs2: 2000, + lookupStarted: make(chan struct{}), + lookupGate: make(chan struct{}), + } + startFakeFiler(t, wfs, fake) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + type openResult struct { + fh *FileHandle + status fuse.Status + } + slow := make(chan openResult, 1) + go func() { + fh, status := wfs.AcquireHandle(inode, 0, 0, 0) + slow <- openResult{fh, status} + }() + <-fake.lookupStarted + + // The faster opener completes with newer state while the slow lookup is + // still in flight. + fastFh, status := wfs.AcquireHandle(inode, 0, 0, 0) + if status != fuse.OK { + t.Fatalf("fast AcquireHandle status = %v, want OK", status) + } + if size := fastFh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("fast open file size = %d, want 200", size) + } + + close(fake.lookupGate) + result := <-slow + if result.status != fuse.OK { + t.Fatalf("slow AcquireHandle status = %v, want OK", result.status) + } + + if size := fastFh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("open handle file size = %d, want 200 (slower opener's older lookup must not overwrite)", size) + } + if got := fastFh.entryVersionTsNs.Load(); got != 2000 { + t.Fatalf("open handle version = %d, want 2000", got) + } +} + +// An event at or below a directory's listing floor is already reflected in +// the snapshot state; applying it would roll the store back while the floor +// keeps claiming the snapshot version, fencing out the correcting events. +func TestFloorProtectsSnapshotStateFromDelayedEvents(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/")) + wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false) + + if err := wfs.metaCache.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil { + t.Fatalf("begin build: %v", err) + } + if err := wfs.metaCache.InsertEntry(context.Background(), &filer.Entry{ + FullPath: "/dir/file", + Attr: filer.Attr{ + Crtime: time.Unix(1, 0), + Mtime: time.Unix(1, 0), + Mode: 0100644, + FileSize: 300, + }, + }, 2000); err != nil { + t.Fatalf("insert listing entry: %v", err) + } + if err := wfs.metaCache.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 2000); err != nil { + t.Fatalf("complete build: %v", err) + } + + // Delayed event the snapshot already covers: must not touch the store. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 100, 1500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply covered event: %v", err) + } + entry, _, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/file")) + if err != nil { + t.Fatalf("find entry: %v", err) + } + if entry.FileSize != 300 { + t.Fatalf("store file size = %d, want 300 (event at 1500 is covered by the floor at 2000)", entry.FileSize) + } + + // A genuinely new event still applies. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 400, 2500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply new event: %v", err) + } + entry, _, err = wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/file")) + if err != nil { + t.Fatalf("find entry: %v", err) + } + if entry.FileSize != 400 { + t.Fatalf("store file size = %d, want 400 (event above the floor must apply)", entry.FileSize) + } + wfs.metaCache.WaitForEntryInvalidations() +} + +// A fence is a lower bound: a listing or lookup can include a mutation whose +// event is delivered afterwards. Such an event carries state the handle +// already holds — it must advance the version without destroying dirty +// pages, or local writes are lost for a no-op. +func TestAlreadyReflectedEventDoesNotDestroyDirtyPages(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + }, 0, 0) + pagesBefore := fh.dirtyPages + + // The event re-delivers exactly the state the handle already reflects. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 200, 1500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if fh.dirtyPages != pagesBefore { + t.Fatal("dirty pages were destroyed for an already-reflected event") + } + if got := fh.entryVersionTsNs.Load(); got != 1500 { + t.Fatalf("handle version = %d, want 1500 (the no-op event still advances the version)", got) + } +} + +// A slower opener's install must not land on a dirty handle (local writes +// would be lost) and unversioned lookup results cannot outrank anything. +func TestSlowerOpenRejectedWhenHandleDirtyOrLookupUnversioned(t *testing.T) { + wfs := newInvalidateTestWFS(t) + fake := &fakeFilerServer{ + lookupSize: 100, + lookupLogTsNs: 3000, // newer than the fast open, but the handle is dirty + lookupSize2: 200, + lookupLogTsNs2: 2000, + lookupStarted: make(chan struct{}), + lookupGate: make(chan struct{}), + } + startFakeFiler(t, wfs, fake) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + type openResult struct { + fh *FileHandle + status fuse.Status + } + slow := make(chan openResult, 1) + go func() { + fh, status := wfs.AcquireHandle(inode, 0, 0, 0) + slow <- openResult{fh, status} + }() + <-fake.lookupStarted + + fastFh, status := wfs.AcquireHandle(inode, 0, 0, 0) + if status != fuse.OK { + t.Fatalf("fast AcquireHandle status = %v, want OK", status) + } + fastFh.dirtyMetadata = true + + close(fake.lookupGate) + result := <-slow + if result.status != fuse.OK { + t.Fatalf("slow AcquireHandle status = %v, want OK", result.status) + } + + if size := fastFh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("open handle file size = %d, want 200 (install on a dirty handle must be rejected)", size) + } + if !fastFh.dirtyMetadata { + t.Fatal("dirtyMetadata was cleared by the rejected install") + } +} + +// Legacy filers return no version; two racing opens both at version zero must +// not overwrite each other — the first install stands. +func TestUnversionedSlowerOpenDoesNotOverwrite(t *testing.T) { + wfs := newInvalidateTestWFS(t) + fake := &fakeFilerServer{ + lookupSize: 100, // slower, unversioned + lookupSize2: 200, // faster, unversioned + lookupStarted: make(chan struct{}), + lookupGate: make(chan struct{}), + } + startFakeFiler(t, wfs, fake) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + type openResult struct { + fh *FileHandle + status fuse.Status + } + slow := make(chan openResult, 1) + go func() { + fh, status := wfs.AcquireHandle(inode, 0, 0, 0) + slow <- openResult{fh, status} + }() + <-fake.lookupStarted + + fastFh, status := wfs.AcquireHandle(inode, 0, 0, 0) + if status != fuse.OK { + t.Fatalf("fast AcquireHandle status = %v, want OK", status) + } + + close(fake.lookupGate) + result := <-slow + if result.status != fuse.OK { + t.Fatalf("slow AcquireHandle status = %v, want OK", result.status) + } + + if size := fastFh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("open handle file size = %d, want 200 (an unversioned response cannot outrank the installed entry)", size) + } +} + +// The no-op judgment must use the immutable base snapshot, not the live +// entry: local writes diverge the live entry from the base, and an event +// re-delivering the base would otherwise look like a foreign change — +// destroying the dirty pages and rolling the entry back over nothing. +func TestAlreadyReflectedEventPreservesDirtyWrites(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + }, 0, 0) + + // A local write grows the live entry past the base. + fh.UpdateEntry(func(entry *filer_pb.Entry) { + entry.Attributes.FileSize = 205 + }) + fh.dirtyMetadata = true + pagesBefore := fh.dirtyPages + + // The delayed event re-delivers the base the handle was opened with. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 200, 1500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if fh.dirtyPages != pagesBefore { + t.Fatal("dirty pages were destroyed by an event re-delivering the handle's base state") + } + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 205 { + t.Fatalf("live entry file size = %d, want 205 (local write must survive the base re-delivery)", size) + } + if got := fh.entryVersionTsNs.Load(); got != 1500 { + t.Fatalf("handle version = %d, want 1500", got) + } +} + +// A versioned deletion is a fact about the path with no entry left to carry +// it. Without a tombstone, a delayed older event resurrects the deleted path +// permanently — the deletion's own redelivery is dedup-suppressed. +func TestDeleteTombstoneBlocksResurrection(t *testing.T) { + wfs := newInvalidateTestWFS(t) + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/")) + wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false) + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir")) + + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 100, 1000), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply create: %v", err) + } + deleteResp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 2000, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), deleteResp, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply delete: %v", err) + } + + // Delayed update the deletion supersedes: must not resurrect the path. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 150, 1500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply delayed update: %v", err) + } + if entry, _, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/file")); err == nil { + t.Fatalf("deleted path resurrected by a delayed event: %+v", entry) + } + + // A genuinely newer create still applies over the tombstone. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 250, 2500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply newer create: %v", err) + } + entry, _, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/file")) + if err != nil || entry.FileSize != 250 { + t.Fatalf("entry after newer create = %+v, %v; want size 250", entry, err) + } + wfs.metaCache.WaitForEntryInvalidations() +} + +// A server-side copy installs the copied entry into the destination handle; +// it must enroll in the versioned-base protocol, or the copy's own event +// differs from the stale pre-copy base and destroys writes made to the +// destination after the copy. +func TestServerSideCopyInstallEnrollsInBase(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 100}, + }, 0, 0) + + copied := &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + } + wfs.applyServerSideWholeFileCopyResult(fh, fh, util.FullPath("/dir/file"), copied, entryVersion{}, 200) + + // Writes land on the destination before the copy's event arrives. + fh.UpdateEntry(func(entry *filer_pb.Entry) { + entry.Attributes.FileSize = 205 + }) + fh.dirtyMetadata = true + pagesBefore := fh.dirtyPages + + // The copy's own event carries the copied state — a no-op for this base. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 200, 1500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply copy event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if fh.dirtyPages != pagesBefore { + t.Fatal("dirty pages destroyed by the copy's own event") + } + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 205 { + t.Fatalf("live entry file size = %d, want 205 (post-copy write must survive)", size) + } +} + +// A completed listing proves absences as well as presences: a delayed +// create for a name the snapshot omitted re-creates something the listing +// already saw deleted. +func TestAbsenceFloorBlocksGhostCreate(t *testing.T) { + wfs := newInvalidateTestWFS(t) + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/")) + wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false) + + if err := wfs.metaCache.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil { + t.Fatalf("begin build: %v", err) + } + if err := wfs.metaCache.InsertEntry(context.Background(), &filer.Entry{ + FullPath: "/dir/other", + Attr: filer.Attr{ + Crtime: time.Unix(1, 0), + Mtime: time.Unix(1, 0), + Mode: 0100644, + FileSize: 1, + }, + }, 0); err != nil { + t.Fatalf("insert listing entry: %v", err) + } + if err := wfs.metaCache.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 2000); err != nil { + t.Fatalf("complete build: %v", err) + } + + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("ghost", 100, 1500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply ghost create: %v", err) + } + if entry, _, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/ghost")); err == nil { + t.Fatalf("name absent at snapshot 2000 resurrected by event at 1500: %+v", entry) + } + + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("ghost", 250, 2500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply newer ghost create: %v", err) + } + entry, _, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/ghost")) + if err != nil || entry.FileSize != 250 { + t.Fatalf("ghost after newer create = %+v, %v; want size 250", entry, err) + } + wfs.metaCache.WaitForEntryInvalidations() +} + +// A deletion is a fact about the path, not about what the cache happened to +// hold: even when the store has no entry to delete, the versioned delete +// must leave a tombstone, or a delayed older event recreates the path. +func TestVersionedDeleteOfMissingEntryLeavesTombstone(t *testing.T) { + wfs := newInvalidateTestWFS(t) + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/")) + wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false) + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir")) + + // No entry inserted: the delete finds nothing to remove. + deleteResp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 2000, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), deleteResp, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply delete: %v", err) + } + + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 150, 1500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply delayed update: %v", err) + } + if entry, _, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/file")); err == nil { + t.Fatalf("path deleted at 2000 recreated by an event at 1500: %+v", entry) + } + + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 250, 2500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply newer create: %v", err) + } + entry, _, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/file")) + if err != nil || entry.FileSize != 250 { + t.Fatalf("entry after newer create = %+v, %v; want size 250", entry, err) + } + wfs.metaCache.WaitForEntryInvalidations() +} + +// A committed copy whose readback failed installs a synthesized base with +// local timestamps; the copy's real event legitimately differs from it and +// must be adopted as the base without invalidating writes made since — the +// event is ours, not a foreign change. +func TestCommittedCopyWithFailedReadbackAdoptsItsEvent(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 100}, + }, 0, 0) + + // Readback failed: nil entry forces the synthesized fallback. + wfs.applyServerSideWholeFileCopyResult(fh, fh, util.FullPath("/dir/file"), nil, entryVersion{}, 200) + + // Writes land on the destination before the copy's event arrives. + fh.UpdateEntry(func(entry *filer_pb.Entry) { + entry.Attributes.FileSize = 205 + }) + fh.dirtyMetadata = true + pagesBefore := fh.dirtyPages + + // The real copy event differs from the synthesized base in timestamps. + copyEvent := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + NewEntry: &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200, Mtime: 999}, + }, + NewParentPath: "/dir", + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), copyEvent, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply copy event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if fh.dirtyPages != pagesBefore { + t.Fatal("dirty pages destroyed by the committed copy's own event") + } + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 205 { + t.Fatalf("live entry file size = %d, want 205 (post-copy write must survive)", size) + } + if got := fh.entryVersionTsNs.Load(); got != 1500 { + t.Fatalf("handle version = %d, want 1500", got) + } + + // The adoption is one-shot: a genuinely foreign event still invalidates. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 300, 1600), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply foreign event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 300 { + t.Fatalf("live entry file size = %d, want 300 (foreign event after adoption must install)", size) + } + if fh.dirtyPages == pagesBefore { + t.Fatal("foreign event after adoption must invalidate dirty pages") + } +} + +// A directory snapshot newer than a tombstone confirms the name is still +// absent at the newer position; an event between the two must be fenced by +// the floor even though an older record exists. +func TestNewerAbsenceFloorOverridesOlderTombstone(t *testing.T) { + wfs := newInvalidateTestWFS(t) + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/")) + wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false) + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir")) + + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 100, 500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply create: %v", err) + } + deleteResp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1000, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), deleteResp, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply delete: %v", err) + } + + // A later listing confirms the name is still absent as of 3000. + if err := wfs.metaCache.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil { + t.Fatalf("begin build: %v", err) + } + if err := wfs.metaCache.CompleteDirectoryBuild(context.Background(), util.FullPath("/dir"), 3000); err != nil { + t.Fatalf("complete build: %v", err) + } + + // Newer than the tombstone, older than the snapshot: still fenced. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 150, 2000), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply mid event: %v", err) + } + if entry, _, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/file")); err == nil { + t.Fatalf("name absent at snapshot 3000 recreated by an event at 2000: %+v", entry) + } + + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 350, 3500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply newer create: %v", err) + } + entry, _, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/file")) + if err != nil || entry.FileSize != 350 { + t.Fatalf("entry after newer create = %+v, %v; want size 350", entry, err) + } + wfs.metaCache.WaitForEntryInvalidations() +} + +// A flush acknowledgment supersedes a pending copy-event adoption: the copy's +// event is version gated after the ack, so a surviving adoption flag would +// misfire on the next genuinely foreign event, silently swallowing it. +func TestFlushAckCancelsPendingCopyEventAdoption(t *testing.T) { + wfs := newInvalidateTestWFS(t) + startFakeFiler(t, wfs, &fakeFilerServer{}) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 100}, + }, 0, 0) + + // Committed copy, failed readback: adoption pending on a synthesized base. + wfs.applyServerSideWholeFileCopyResult(fh, fh, util.FullPath("/dir/file"), nil, entryVersion{}, 200) + + // A local flush lands: the ack at 3000 becomes the authoritative base. + fh.dirtyMetadata = true + if status := wfs.doFlush(context.Background(), fh, 0, 0, false); status != fuse.OK { + t.Fatalf("doFlush status = %v, want OK", status) + } + + // The copy's own delayed event is version gated by the ack. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 200, 1500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply copy event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + // A genuinely foreign event must install normally, not be adopted. + pagesBefore := fh.dirtyPages + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 300, 3500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply foreign event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 300 { + t.Fatalf("live entry file size = %d, want 300 (foreign event must install, not be silently adopted)", size) + } + if fh.dirtyPages == pagesBefore { + t.Fatal("foreign event must invalidate dirty pages, not be silently adopted") + } +} + +// A version must never advance without its value: a handle opened while a +// setattr was in flight holds the pre-mutation entry, and stamping it with +// the acknowledgment's version would fence out the events carrying the state +// it lacks. The acknowledged entry is installed with the version instead. +func TestAckedSaveInstallsIntoRacingOpenHandle(t *testing.T) { + wfs := newInvalidateTestWFS(t) + startFakeFiler(t, wfs, &fakeFilerServer{}) + + // The handle opened after the setattr path found none, before the ack. + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 88}, + }, 0, 0) + + saved := &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + } + if code := wfs.saveEntry(util.FullPath("/dir/file"), saved); code != fuse.OK { + t.Fatalf("saveEntry status = %v, want OK", code) + } + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("open handle file size = %d, want 200 (the acknowledged state must be installed with its version)", size) + } + if got := fh.entryVersionTsNs.Load(); got != 2000 { + t.Fatalf("handle version = %d, want 2000", got) + } + + // A dirty handle is left alone entirely: neither entry nor version. + dirtyInode := wfs.inodeToPath.Lookup(util.FullPath("/dir/dirty"), time.Now().Unix(), false, false, 0, false) + dirtyFh, _ := wfs.fhMap.AcquireFileHandle(wfs, dirtyInode, &filer_pb.Entry{ + Name: "dirty", + Attributes: &filer_pb.FuseAttributes{FileSize: 88}, + }, 0, 0) + dirtyFh.dirtyMetadata = true + if code := wfs.saveEntry(util.FullPath("/dir/dirty"), &filer_pb.Entry{ + Name: "dirty", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + }); code != fuse.OK { + t.Fatalf("saveEntry status = %v, want OK", code) + } + if size := dirtyFh.GetEntry().GetEntry().Attributes.FileSize; size != 88 { + t.Fatalf("dirty handle file size = %d, want 88 (local writes supersede the ack)", size) + } + if got := dirtyFh.entryVersionTsNs.Load(); got != 0 { + t.Fatalf("dirty handle version = %d, want 0 (no version without its value)", got) + } +} + +// An empty listing carries its snapshot in the stream trailer, so empty +// directories still gain an absence floor — and their stale tombstones +// still get pruned — instead of accumulating forever. +func TestEmptyListingTrailerSnapshotSetsAbsenceFloor(t *testing.T) { + wfs := newInvalidateTestWFS(t) + startFakeFiler(t, wfs, &fakeFilerServer{listSnapshotTrailerTsNs: 4000}) + + wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false) + if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, util.FullPath("/dir")); err != nil { + t.Fatalf("EnsureVisited: %v", err) + } + + // Absent at the trailer snapshot: an older create must be fenced. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("ghost", 100, 3000), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply covered create: %v", err) + } + if entry, _, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/ghost")); err == nil { + t.Fatalf("name absent at trailer snapshot 4000 created by event at 3000: %+v", entry) + } + + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("ghost", 450, 4500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply newer create: %v", err) + } + entry, _, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/ghost")) + if err != nil || entry.FileSize != 450 { + t.Fatalf("entry after newer create = %+v, %v; want size 450", entry, err) + } + wfs.metaCache.WaitForEntryInvalidations() +} + +// A foreign delete of a file with unflushed local writes must not destroy the +// dirty pages: POSIX lets a process keep writing to an unlinked-but-open file, +// and the writes were already acknowledged. +func TestVacateEventPreservesDirtyPages(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + }, 0, 0) + fh.dirtyMetadata = true + pagesBefore := fh.dirtyPages + + deleteResp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), deleteResp, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply delete event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if fh.dirtyPages != pagesBefore { + t.Fatal("dirty pages destroyed by a foreign delete of an open file with unflushed writes") + } + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("open handle file size after delete = %d, want 200", size) + } +} + +// A committed copy whose readback failed adopts only the copy's own event +// (same content). A foreign write to the destination that arrives first has +// different content and must install normally, not be swallowed by the +// pending adoption. +func TestCopyAdoptRejectsForeignEvent(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 100}, + }, 0, 0) + + // Readback failed: synthesized base at size 200, adoption pending. + wfs.applyServerSideWholeFileCopyResult(fh, fh, util.FullPath("/dir/file"), nil, entryVersion{}, 200) + + // A foreign write to the destination arrives before the copy's own event: + // different content (size 500), so it must install and not be adopted. + foreign := updateEventFor("file", 500, 1500) + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), foreign, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply foreign event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 500 { + t.Fatalf("live entry file size = %d, want 500 (foreign write must install, not be swallowed by the copy adoption)", size) + } +} + +// downloadRemoteEntry stores the handle's base in local uid/gid form, so a +// later re-delivery of unchanged content compares equal and does not +// force-destroy dirty pages under a non-identity UidGidMapper. +func TestRemoteDownloadBaseMappedToLocal(t *testing.T) { + wfs := newInvalidateTestWFS(t) + // The test mapper maps filer uid 2000 -> local 1000; the cache response + // carries filer uid 2000. + startFakeFiler(t, wfs, &fakeFilerServer{cacheSize: 200, cacheUid: 2000, cacheLogTsNs: 1000}) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 200}, + }, 0, 0) + + if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil { + t.Fatalf("downloadRemoteEntry: %v", err) + } + // The base — and the live entry — must be in local uid form. + if uid := fh.GetEntry().GetEntry().Attributes.Uid; uid != 1000 { + t.Fatalf("live entry uid = %d, want filer 2000 mapped to local 1000", uid) + } + + // The user writes to the handle (dirty pages, not flushed). + fh.dirtyMetadata = true + pagesBefore := fh.dirtyPages + + // A re-delivery of the same content (local-form candidate, uid 2000 in the + // event mapped to local 1000) must read as a no-op and preserve the writes. + reDeliver := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + NewEntry: &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: 200, FileMode: 0100644, Uid: 2000}}, + NewParentPath: "/dir", + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), reDeliver, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply re-delivery: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if fh.dirtyPages != pagesBefore { + t.Fatal("dirty pages destroyed by an unchanged re-delivery (base was not mapped to local form)") + } +} + +// A foreign delete of a dirty open file must mark the handle deleted, so a +// later flush does not recreate the remotely-unlinked name. +func TestForeignDeleteMarksHandleDeleted(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + }, 0, 0) + fh.dirtyMetadata = true + + del := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1500, + EventNotification: &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: "file"}}, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), del, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply delete: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if !fh.isDeleted { + t.Fatal("handle not marked deleted after a foreign delete; a flush would recreate the unlinked name") + } +} + +// A no-event acknowledgment (log fence only) must version the cache entry, so +// an older subscriber event cannot roll it back. +func TestEventlessSaveVersionsCacheEntry(t *testing.T) { + wfs := newInvalidateTestWFS(t) + startFakeFiler(t, wfs, &fakeFilerServer{updateEventless: true, updateLogTsNs: 2000}) + + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/")) + wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false) + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir")) + + if code := wfs.saveEntry(util.FullPath("/dir/file"), &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200, FileMode: 0100644}, + }); code != fuse.OK { + t.Fatalf("saveEntry status = %v, want OK", code) + } + + // An older subscriber event must be fenced out by the ack's log position. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 100, 1500), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply older event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + entry, _, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/file")) + if err != nil || entry.FileSize != 200 { + t.Fatalf("cache entry = %+v, %v; want size 200 (older event must not roll back the no-event ack)", entry, err) + } +} + +// A remote download response that lands after the handle was already +// populated must not roll the entry back: it is older than what the handle +// holds, and the monotonic version would keep the newer value. +func TestStaleRemoteDownloadDoesNotRollBack(t *testing.T) { + wfs := newInvalidateTestWFS(t) + startFakeFiler(t, wfs, &fakeFilerServer{cacheSize: 100, cacheLogTsNs: 1000}) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 200}, + }, 0, 0) + // While the download was in flight the handle was populated at a newer + // version — it now has local chunks and no longer needs the response. + fh.SetEntry(&filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + Chunks: []*filer_pb.FileChunk{{FileId: "1,ab1", Size: 200}}, + }) + fh.advanceEntryVersion(3000, 0) + + if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil { + t.Fatalf("downloadRemoteEntry: %v", err) + } + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("handle file size = %d, want 200 (a download at version 1000 must not overwrite the version-3000 handle)", size) + } + if got := fh.entryVersionTsNs.Load(); got != 3000 { + t.Fatalf("handle version = %d, want 3000", got) + } +} + +// A still-remote-only handle must take an older or unversioned response +// anyway — without it there are no local chunks to read — but must not claim +// the response's log position. +func TestRemoteOnlyHandleTakesUnversionedDownload(t *testing.T) { + wfs := newInvalidateTestWFS(t) + startFakeFiler(t, wfs, &fakeFilerServer{cacheSize: 100}) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 100}, + RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 100}, + }, 0, 0) + fh.advanceEntryVersion(3000, 0) + + if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil { + t.Fatalf("downloadRemoteEntry: %v", err) + } + + if fh.GetEntry().GetEntry().IsInRemoteOnly() { + t.Fatal("remote-only handle did not take the download; reads would have no chunks") + } + if got := fh.entryVersionTsNs.Load(); got != 3000 { + t.Fatalf("handle version = %d, want 3000 (an unversioned response must not claim a position)", got) + } +} + +// A foreign metadata-only change (chmod) with unchanged content must not be +// mistaken for a committed copy's own event and adopted; it must install. +func TestCopyAdoptRejectsForeignMetadataChange(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200, FileMode: 0100644}, + }, 0, 0) + + // Readback failed: synthesized base at size 200, mode 0644; adoption pending. + wfs.applyServerSideWholeFileCopyResult(fh, fh, util.FullPath("/dir/file"), nil, entryVersion{}, 200) + + // A foreign chmod: same content (size 200) but mode 0600. + chmod := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + NewEntry: &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: 200, FileMode: 0100600}}, + NewParentPath: "/dir", + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), chmod, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply chmod: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if mode := fh.GetEntry().GetEntry().Attributes.FileMode; mode != 0100600 { + t.Fatalf("live entry mode = %o, want 0100600 (foreign chmod must install, not be swallowed by copy adoption)", mode) + } +} + +// A foreign rename must not mark an open handle deleted: the file still +// exists at its new path, and marking it silently drops later writes through +// the already-open descriptor. +func TestForeignRenameDoesNotMarkHandleDeleted(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + }, 0, 0) + fh.dirtyMetadata = true + + rename := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + NewEntry: &filer_pb.Entry{Name: "renamed", Attributes: &filer_pb.FuseAttributes{FileSize: 200}}, + NewParentPath: "/dir", + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), rename, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply rename: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if fh.isDeleted { + t.Fatal("handle marked deleted by a foreign rename; later writes would be silently dropped") + } + // The handle follows the file to its new name, so a flush updates the + // renamed file instead of recreating the old one. + if got := fh.FullPath(); got != util.FullPath("/dir/renamed") { + t.Fatalf("handle path = %q, want /dir/renamed after the rename", got) + } + if name := fh.GetEntry().GetEntry().Name; name != "renamed" { + t.Fatalf("handle entry name = %q, want renamed", name) + } + + // A real delete of the file at its current name still marks the handle. + del := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 2500, + EventNotification: &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: "renamed"}}, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), del, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply delete: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + if !fh.isDeleted { + t.Fatal("handle not marked deleted by a real foreign delete") + } +} + +// A foreign touch arriving before a committed copy's own event must apply its +// timestamps to a clean handle, and must not consume anything the copy's own +// event still needs — the copy's event carries the same content, so it never +// destroys the post-copy writes. +func TestCopyAdoptAppliesForeignTouchToCleanHandle(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200, Mtime: 100}, + }, 0, 0) + + // Readback failed: synthesized base, adoption pending. + wfs.applyServerSideWholeFileCopyResult(fh, fh, util.FullPath("/dir/file"), nil, entryVersion{}, 200) + + // A foreign touch: identical content, new mtime. + touch := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + NewEntry: &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: 200, Mtime: 999}}, + NewParentPath: "/dir", + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), touch, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply touch: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if mtime := fh.GetEntry().GetEntry().Attributes.Mtime; mtime != 999 { + t.Fatalf("live entry mtime = %d, want 999 (foreign touch must apply to a clean handle, not be swallowed)", mtime) + } +} + +// An unversioned response (pre-upgrade filer) must not overwrite versioned +// state on a handle that already has local content. +func TestUnversionedRemoteDownloadDoesNotOverwriteVersioned(t *testing.T) { + wfs := newInvalidateTestWFS(t) + // No cacheLogTsNs and no metadata event: the response carries no version. + startFakeFiler(t, wfs, &fakeFilerServer{cacheSize: 100}) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 200}, + }, 0, 0) + // Populated at version 3000 while the download was in flight. + fh.SetEntry(&filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + Chunks: []*filer_pb.FileChunk{{FileId: "1,ab1", Size: 200}}, + }) + fh.advanceEntryVersion(3000, 0) + + if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil { + t.Fatalf("downloadRemoteEntry: %v", err) + } + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("handle file size = %d, want 200 (an unversioned response must not overwrite version-3000 state)", size) + } +} + +// A fence stamped by one filer says nothing about an event another filer +// logged: their clocks are independent. Skipping across that boundary would +// leave the handle stale for good, so the event is applied instead — a +// re-apply the base check absorbs when it turns out to be redundant. +func TestEventFromAnotherFilerIsNotFencedByForeignClock(t *testing.T) { + wfs := newInvalidateTestWFS(t) + // Filer A answers the open-time lookup and stamps its fence at 5000. + startFakeFiler(t, wfs, &fakeFilerServer{lookupSize: 200, lookupLogTsNs: 5000, lookupSignature: 11}) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, status := wfs.AcquireHandle(inode, 0, 0, 0) + if status != fuse.OK { + t.Fatalf("AcquireHandle status = %v, want OK", status) + } + if got := fh.entryVersionSignature.Load(); got != 11 { + t.Fatalf("handle fence signature = %d, want 11 from the lookup", got) + } + + // Filer B logged this event at 3000 on its own clock — below A's fence, + // but the two positions are not comparable. + fromOtherFiler := updateEventFor("file", 900, 3000) + fromOtherFiler.EventNotification.Signatures = []int32{22} + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), fromOtherFiler, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply cross-filer event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 900 { + t.Fatalf("handle file size = %d, want 900 (an event from another filer must not be fenced by this filer's clock)", size) + } + + // The same filer's own older event is still fenced: one clock, real order. + sameFiler := updateEventFor("file", 700, 4000) + sameFiler.EventNotification.Signatures = []int32{11} + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), sameFiler, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply same-filer event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size == 700 { + t.Fatal("an older event from the fencing filer was applied; within one clock domain it must be fenced") + } +} + +// A foreign touch landing before a committed copy's own event must not leave +// the copy's event to destroy the post-copy writes. +func TestTouchBeforeCopyEventKeepsPostCopyWrites(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 100, Mtime: 100}, + }, 0, 0) + + // Committed copy whose readback failed: the base is synthesized. + wfs.applyServerSideWholeFileCopyResult(fh, fh, util.FullPath("/dir/file"), nil, entryVersion{}, 200) + + // Post-copy writes. + fh.dirtyMetadata = true + pagesBefore := fh.dirtyPages + + // A foreign touch arrives first: same content, new mtime. + touch := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + NewEntry: &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: 200, Mtime: 555}}, + NewParentPath: "/dir", + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), touch, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply touch: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + // Then the copy's own event, with the server's timestamps. + copyEvent := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1600, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + NewEntry: &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: 200, Mtime: 777}}, + NewParentPath: "/dir", + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), copyEvent, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply copy event: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if fh.dirtyPages != pagesBefore { + t.Fatal("post-copy writes destroyed: a timestamp-only event must not invalidate the dirty overlay") + } +} + +// A rejected download must not publish its state to the metadata cache either. +func TestRejectedDownloadDoesNotRollBackCache(t *testing.T) { + wfs := newInvalidateTestWFS(t) + // Unversioned response carrying older content. + startFakeFiler(t, wfs, &fakeFilerServer{cacheSize: 100}) + + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/")) + wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false) + wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir")) + + // The cache holds the current state at version 3000. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("file", 200, 3000), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("seed cache: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + Chunks: []*filer_pb.FileChunk{{FileId: "1,ab1", Size: 200}}, + }, 0, 0) + fh.advanceEntryVersion(3000, 0) + + if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil { + t.Fatalf("downloadRemoteEntry: %v", err) + } + // The download publishes asynchronously; a synchronous apply behind it + // drains the FIFO so the assertion sees the final state either way. + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), updateEventFor("other", 1, 4000), meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("flush apply loop: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + entry, versionTsNs, err := wfs.metaCache.FindEntry(context.Background(), util.FullPath("/dir/file")) + if err != nil { + t.Fatalf("find cache entry: %v", err) + } + if entry.FileSize != 200 || versionTsNs != 3000 { + t.Fatalf("cache size/version = %d/%d, want 200/3000 (a rejected download must not publish)", entry.FileSize, versionTsNs) + } +} + +// A remote-only handle takes an unversioned response because it cannot read +// without chunks, but a response that is merely older is refused: its content +// predates the state the handle already reflects. +func TestRemoteOnlyHandleRefusesKnownOlderDownload(t *testing.T) { + wfs := newInvalidateTestWFS(t) + // Versioned at 1000 — older than the handle's 3000. + startFakeFiler(t, wfs, &fakeFilerServer{cacheSize: 100, cacheLogTsNs: 1000}) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200}, + RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 200}, + }, 0, 0) + fh.advanceEntryVersion(3000, 0) + + if err := fh.downloadRemoteEntry(fh.GetEntry()); err != nil { + t.Fatalf("downloadRemoteEntry: %v", err) + } + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("handle file size = %d, want 200 (a known-older response must be refused even when remote-only)", size) + } +} + +// A metadata-only foreign change (chmod) must not invalidate the dirty-page +// overlay: pages overlay content, and the content did not change. +func TestForeignChmodPreservesDirtyPages(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 200, FileMode: 0100644}, + }, 0, 0) + fh.dirtyMetadata = true + pagesBefore := fh.dirtyPages + + chmod := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "file"}, + NewEntry: &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: 200, FileMode: 0100600}}, + NewParentPath: "/dir", + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), chmod, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply chmod: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if fh.dirtyPages != pagesBefore { + t.Fatal("dirty pages destroyed by a metadata-only change") + } +} + +// A rename over an existing file destroys that file; its open handle must be +// marked deleted so a later flush cannot resurrect it over the renamed source. +func TestRenameOverExistingMarksReplacedHandleDeleted(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + srcInode := wfs.inodeToPath.Lookup(util.FullPath("/dir/src"), time.Now().Unix(), false, false, 0, false) + srcFh, _ := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{ + Name: "src", + Attributes: &filer_pb.FuseAttributes{FileSize: 100}, + }, 0, 0) + dstInode := wfs.inodeToPath.Lookup(util.FullPath("/dir/dst"), time.Now().Unix(), false, false, 0, false) + dstFh, _ := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{ + Name: "dst", + Attributes: &filer_pb.FuseAttributes{FileSize: 900}, + }, 0, 0) + dstFh.dirtyMetadata = true + + rename := &filer_pb.SubscribeMetadataResponse{ + Directory: "/dir", + TsNs: 1500, + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "src"}, + NewEntry: &filer_pb.Entry{Name: "dst", Attributes: &filer_pb.FuseAttributes{FileSize: 100}}, + NewParentPath: "/dir", + }, + } + if err := wfs.metaCache.ApplyMetadataResponse(context.Background(), rename, meta_cache.SubscriberMetadataResponseApplyOptions); err != nil { + t.Fatalf("apply rename: %v", err) + } + wfs.metaCache.WaitForEntryInvalidations() + + if !dstFh.isDeleted { + t.Fatal("replaced destination handle not marked deleted; its flush could resurrect it over the renamed source") + } + if srcFh.isDeleted { + t.Fatal("renamed source handle must stay live") + } +} + +// An acknowledgment from one filer must not be dropped behind a fence another +// filer stamped: their positions are not comparable, and dropping it leaves +// the handle holding exactly the state the mutation replaced. +func TestAckFromAnotherFilerIsNotDroppedBehindForeignFence(t *testing.T) { + wfs := newInvalidateTestWFS(t) + + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + Name: "file", + Attributes: &filer_pb.FuseAttributes{FileSize: 88}, + }, 0, 0) + // Filer A's fence at 5000. + fh.advanceEntryVersion(5000, 11) + + // Filer B acknowledges our mutation at 1000 on its own clock. + acked := &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: 200}} + fh.installAckedEntry(acked, 1000, 22) + + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 200 { + t.Fatalf("handle file size = %d, want 200 (a cross-filer ack must not be dropped behind a foreign fence)", size) + } + + // The same filer's own older ack is still refused. + older := &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{FileSize: 300}} + fh.installAckedEntry(older, 500, fh.entryVersionSignature.Load()) + if size := fh.GetEntry().GetEntry().Attributes.FileSize; size == 300 { + t.Fatal("an older ack from the fencing filer was installed; within one clock domain it must be refused") + } +} diff --git a/weed/mount/weedfs_link.go b/weed/mount/weedfs_link.go index 969d7d199..a41f31490 100644 --- a/weed/mount/weedfs_link.go +++ b/weed/mount/weedfs_link.go @@ -46,7 +46,7 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out * } oldParentPath, _ := oldEntryPath.DirAndName() - oldEntry, status := wfs.maybeLoadEntry(oldEntryPath) + oldEntry, _, status := wfs.maybeLoadEntry(oldEntryPath) if status != fuse.OK { return status } @@ -78,7 +78,7 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out * // Do not fall back to the pre-lock snapshot: if every alias // was deleted while we waited, abort instead of deriving the // next counter from a stale entry. - fresh, freshStatus := wfs.maybeLoadEntry(oldEntryPath) + fresh, _, freshStatus := wfs.maybeLoadEntry(oldEntryPath) if freshStatus != fuse.OK { return freshStatus } @@ -212,7 +212,7 @@ func (wfs *WFS) syncHardLinkSiblings(inode uint64, authoritativeEntry *filer_pb. if _, skipped := skip[p]; skipped { continue } - sibling, err := wfs.metaCache.FindEntry(ctx, p) + sibling, _, err := wfs.metaCache.FindEntry(ctx, p) if err != nil || sibling == nil { continue } diff --git a/weed/mount/weedfs_metadata_flush.go b/weed/mount/weedfs_metadata_flush.go index 621a60c45..3ff2fa9f7 100644 --- a/weed/mount/weedfs_metadata_flush.go +++ b/weed/mount/weedfs_metadata_flush.go @@ -169,6 +169,9 @@ func (wfs *WFS) flushFileMetadata(fh *FileHandle) error { SkipCheckParentDirectory: true, } + // Snapshot with local ids before the request mapping mutates the clone: + // on ack this becomes the handle's base, judged against future events. + baseSnapshot := proto.Clone(requestEntry).(*filer_pb.Entry) wfs.mapPbIdFromLocalToFiler(request.Entry) resp, err := wfs.streamCreateEntry(context.Background(), request) @@ -179,7 +182,12 @@ func (wfs *WFS) flushFileMetadata(fh *FileHandle) error { event := resp.GetMetadataEvent() if event == nil { event = metadataUpdateEvent(string(dir), request.Entry) + if event != nil { + event.TsNs = ackVersionTsNs(resp) + } } + fh.setAuthoritativeBase(baseSnapshot) + fh.advanceEntryVersion(ackVersionTsNs(resp), resp.GetLogSignature()) if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil { glog.Warningf("flushFileMetadata %s: best-effort metadata apply failed: %v", fileFullPath, applyErr) wfs.inodeToPath.InvalidateChildrenCache(util.FullPath(dir)) diff --git a/weed/mount/weedfs_metadata_flush_mtime_test.go b/weed/mount/weedfs_metadata_flush_mtime_test.go index a2fc3554d..1c31de0bb 100644 --- a/weed/mount/weedfs_metadata_flush_mtime_test.go +++ b/weed/mount/weedfs_metadata_flush_mtime_test.go @@ -22,9 +22,9 @@ import ( // the entry's mtime is unchanged. func TestFlushFileMetadataPreservesUserMtime(t *testing.T) { wfs := &WFS{ - inodeToPath: NewInodeToPath(util.FullPath("/"), 0), - fhLockTable: util.NewLockTable[FileHandleId](), - option: &Option{}, + inodeToPath: NewInodeToPath(util.FullPath("/"), 0), + fhLockTable: util.NewLockTable[FileHandleId](), + option: &Option{}, } const inode = uint64(42) diff --git a/weed/mount/weedfs_posix_lock_routed.go b/weed/mount/weedfs_posix_lock_routed.go index 489d6721f..502ec8f3b 100644 --- a/weed/mount/weedfs_posix_lock_routed.go +++ b/weed/mount/weedfs_posix_lock_routed.go @@ -95,7 +95,7 @@ func (wfs *WFS) posixLockKeyForInode(inode uint64) (string, bool) { if status != fuse.OK { return "", false } - if entry, st := wfs.maybeLoadEntry(path); st == fuse.OK && entry != nil && len(entry.HardLinkId) > 0 { + if entry, _, st := wfs.maybeLoadEntry(path); st == fuse.OK && entry != nil && len(entry.HardLinkId) > 0 { return posixLockKeyPrefix + "hl:" + hex.EncodeToString(entry.HardLinkId), true } return posixLockKeyPrefix + string(path), true diff --git a/weed/mount/weedfs_rename.go b/weed/mount/weedfs_rename.go index 0fdcd1820..34343e0fe 100644 --- a/weed/mount/weedfs_rename.go +++ b/weed/mount/weedfs_rename.go @@ -199,13 +199,13 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string } newPath := newDir.Child(newName) - oldEntry, status := wfs.maybeLoadEntry(oldPath) + oldEntry, _, status := wfs.maybeLoadEntry(oldPath) if status != fuse.OK { return status } // POSIX: enforce sticky bit on the source directory. - if oldDirEntry, dirCode := wfs.maybeLoadEntry(oldDir); dirCode == fuse.OK && oldDirEntry != nil && oldDirEntry.Attributes != nil { + if oldDirEntry, _, dirCode := wfs.maybeLoadEntry(oldDir); dirCode == fuse.OK && oldDirEntry != nil && oldDirEntry.Attributes != nil { targetUid := uint32(0) if oldEntry != nil && oldEntry.Attributes != nil { targetUid = oldEntry.Attributes.Uid @@ -217,8 +217,8 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string // POSIX: enforce sticky bit on the destination directory when replacing an existing entry. if in.Flags != RenameNoReplace { - if newEntry, newStatus := wfs.maybeLoadEntry(newPath); newStatus == fuse.OK && newEntry != nil { - if newDirEntry, dirCode := wfs.maybeLoadEntry(newDir); dirCode == fuse.OK && newDirEntry != nil && newDirEntry.Attributes != nil { + if newEntry, _, newStatus := wfs.maybeLoadEntry(newPath); newStatus == fuse.OK && newEntry != nil { + if newDirEntry, _, dirCode := wfs.maybeLoadEntry(newDir); dirCode == fuse.OK && newDirEntry != nil && newDirEntry.Attributes != nil { targetUid := uint32(0) if newEntry.Attributes != nil { targetUid = newEntry.Attributes.Uid diff --git a/weed/mount/weedfs_rename_test.go b/weed/mount/weedfs_rename_test.go index 75b1e19eb..406a093c3 100644 --- a/weed/mount/weedfs_rename_test.go +++ b/weed/mount/weedfs_rename_test.go @@ -30,7 +30,7 @@ func TestHandleRenameResponseLeavesUncachedTargetOutOfCache(t *testing.T) { func(path util.FullPath) bool { return inodeToPath.IsChildrenCached(path) }, - func(util.FullPath, *filer_pb.Entry) {}, + func(meta_cache.EntryInvalidation) {}, nil, ) defer mc.Shutdown() @@ -73,7 +73,7 @@ func TestHandleRenameResponseLeavesUncachedTargetOutOfCache(t *testing.T) { t.Fatalf("handle rename response: %v", err) } - entry, findErr := mc.FindEntry(context.Background(), targetPath) + entry, _, findErr := mc.FindEntry(context.Background(), targetPath) if findErr != filer_pb.ErrNotFound { t.Fatalf("find target entry error = %v, want %v", findErr, filer_pb.ErrNotFound) } diff --git a/weed/mount/weedfs_symlink.go b/weed/mount/weedfs_symlink.go index b0d4cd9cc..d9a413d4f 100644 --- a/weed/mount/weedfs_symlink.go +++ b/weed/mount/weedfs_symlink.go @@ -91,7 +91,7 @@ func (wfs *WFS) Readlink(cancel <-chan struct{}, header *fuse.InHeader) (out []b return } - entry, status := wfs.maybeLoadEntry(entryFullPath) + entry, _, status := wfs.maybeLoadEntry(entryFullPath) if status != fuse.OK { return nil, status } diff --git a/weed/mount/weedfs_xattr_test.go b/weed/mount/weedfs_xattr_test.go index d0eb11c3f..9b2e1b49b 100644 --- a/weed/mount/weedfs_xattr_test.go +++ b/weed/mount/weedfs_xattr_test.go @@ -26,13 +26,13 @@ func TestSetXAttrCopiesValueBuffer(t *testing.T) { path := util.FullPath("/aaa.txt") inode := wfs.inodeToPath.Lookup(path, 1, false, false, 0, true) - fh := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ + fh, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{ Name: "aaa.txt", Attributes: &filer_pb.FuseAttributes{ FileMode: 0100644, Inode: inode, }, - }) + }, 0, 0) fh.RememberPath(path) // Caller buffer aliases a pool that the kernel will overwrite on diff --git a/weed/mount/wfs_save.go b/weed/mount/wfs_save.go index 24f692ae2..a3f1a0093 100644 --- a/weed/mount/wfs_save.go +++ b/weed/mount/wfs_save.go @@ -7,6 +7,8 @@ import ( "time" "github.com/seaweedfs/go-fuse/v2/fuse" + "google.golang.org/protobuf/proto" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" @@ -51,9 +53,26 @@ func (wfs *WFS) saveEntry(path util.FullPath, entry *filer_pb.Entry) (code fuse. return fuseStatus } + // The mutation is acknowledged; bring any open handle for this path up + // to the acknowledged state — a handle opened while this save was in + // flight holds an older entry, and advancing its version alone would + // fence out the events carrying what it lacks. A no-change update + // returns no event but still carries the log position it confirmed. + ackVersion := ackVersionTsNs(resp) + if inode, found := wfs.inodeToPath.GetInode(path); found { + if fh, fhFound := wfs.fhMap.FindFileHandle(inode); fhFound { + ackedEntry := proto.Clone(entry).(*filer_pb.Entry) + wfs.mapPbIdFromFilerToLocal(ackedEntry) + fh.installAckedEntry(ackedEntry, ackVersion, resp.GetLogSignature()) + } + } + event := resp.GetMetadataEvent() if event == nil { event = metadataUpdateEvent(parentDir, entry) + if event != nil { + event.TsNs = ackVersion + } } if applyErr := wfs.applyLocalMetadataEvent(context.Background(), event); applyErr != nil { glog.Warningf("saveEntry %s: best-effort metadata apply failed: %v", path, applyErr) diff --git a/weed/mq/broker/broker_write.go b/weed/mq/broker/broker_write.go index 4b91e49e0..8c4b42dae 100644 --- a/weed/mq/broker/broker_write.go +++ b/weed/mq/broker/broker_write.go @@ -35,7 +35,7 @@ func (b *MessageQueueBroker) appendToFileWithBufferIndex(targetFile string, data // find out existing entry fullpath := util.FullPath(targetFile) dir, name := fullpath.DirAndName() - entry, err := filer_pb.GetEntry(context.Background(), b, fullpath) + entry, _, _, err := filer_pb.GetEntry(context.Background(), b, fullpath) var offset int64 = 0 if err == filer_pb.ErrNotFound { entry = &filer_pb.Entry{ diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index d350b117a..359f9f819 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -121,6 +121,12 @@ message LookupDirectoryEntryRequest { message LookupDirectoryEntryResponse { Entry entry = 1; + // filer log position stamped before the entry read: every event at or + // below it is reflected in the returned entry + int64 log_ts_ns = 2; + // signature of the filer whose clock stamped log_ts_ns; positions are + // only comparable with events that filer logged + int32 log_signature = 3; } message ListEntriesRequest { @@ -446,6 +452,10 @@ message CreateEntryResponse { string error = 1; // kept for human readability + backward compat SubscribeMetadataResponse metadata_event = 2; FilerError error_code = 3; // machine-readable error code + // filer log position stamped under the path lock before the write: + // every event at or below it is reflected in the acknowledged state + int64 log_ts_ns = 4; + int32 log_signature = 5; // filer whose clock stamped log_ts_ns } message UpdateEntryRequest { @@ -461,6 +471,10 @@ message UpdateEntryRequest { } message UpdateEntryResponse { SubscribeMetadataResponse metadata_event = 1; + // filer log position stamped under the path lock before the write: + // every event at or below it is reflected in the acknowledged state + int64 log_ts_ns = 2; + int32 log_signature = 3; // filer whose clock stamped log_ts_ns } message TouchAccessTimeRequest { @@ -763,6 +777,10 @@ message CacheRemoteObjectToLocalClusterRequest { message CacheRemoteObjectToLocalClusterResponse { Entry entry = 1; SubscribeMetadataResponse metadata_event = 2; + // filer log position stamped before the entry read: every event at or + // below it is reflected in the returned entry + int64 log_ts_ns = 3; + int32 log_signature = 4; // filer whose clock stamped log_ts_ns } ///////////////////////// diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 059dde3f3..ec4dcc375 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -369,8 +369,14 @@ func (x *LookupDirectoryEntryRequest) GetName() string { } type LookupDirectoryEntryResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - Entry *Entry `protobuf:"bytes,1,opt,name=entry,proto3" json:"entry,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Entry *Entry `protobuf:"bytes,1,opt,name=entry,proto3" json:"entry,omitempty"` + // filer log position stamped before the entry read: every event at or + // below it is reflected in the returned entry + LogTsNs int64 `protobuf:"varint,2,opt,name=log_ts_ns,json=logTsNs,proto3" json:"log_ts_ns,omitempty"` + // signature of the filer whose clock stamped log_ts_ns; positions are + // only comparable with events that filer logged + LogSignature int32 `protobuf:"varint,3,opt,name=log_signature,json=logSignature,proto3" json:"log_signature,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -412,6 +418,20 @@ func (x *LookupDirectoryEntryResponse) GetEntry() *Entry { return nil } +func (x *LookupDirectoryEntryResponse) GetLogTsNs() int64 { + if x != nil { + return x.LogTsNs + } + return 0 +} + +func (x *LookupDirectoryEntryResponse) GetLogSignature() int32 { + if x != nil { + return x.LogSignature + } + return 0 +} + type ListEntriesRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Directory string `protobuf:"bytes,1,opt,name=directory,proto3" json:"directory,omitempty"` @@ -2251,6 +2271,10 @@ type CreateEntryResponse struct { Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` // kept for human readability + backward compat MetadataEvent *SubscribeMetadataResponse `protobuf:"bytes,2,opt,name=metadata_event,json=metadataEvent,proto3" json:"metadata_event,omitempty"` ErrorCode FilerError `protobuf:"varint,3,opt,name=error_code,json=errorCode,proto3,enum=filer_pb.FilerError" json:"error_code,omitempty"` // machine-readable error code + // filer log position stamped under the path lock before the write: + // every event at or below it is reflected in the acknowledged state + LogTsNs int64 `protobuf:"varint,4,opt,name=log_ts_ns,json=logTsNs,proto3" json:"log_ts_ns,omitempty"` + LogSignature int32 `protobuf:"varint,5,opt,name=log_signature,json=logSignature,proto3" json:"log_signature,omitempty"` // filer whose clock stamped log_ts_ns unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2306,6 +2330,20 @@ func (x *CreateEntryResponse) GetErrorCode() FilerError { return FilerError_OK } +func (x *CreateEntryResponse) GetLogTsNs() int64 { + if x != nil { + return x.LogTsNs + } + return 0 +} + +func (x *CreateEntryResponse) GetLogSignature() int32 { + if x != nil { + return x.LogSignature + } + return 0 +} + type UpdateEntryRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Directory string `protobuf:"bytes,1,opt,name=directory,proto3" json:"directory,omitempty"` @@ -2396,6 +2434,10 @@ func (x *UpdateEntryRequest) GetCondition() *WriteCondition { type UpdateEntryResponse struct { state protoimpl.MessageState `protogen:"open.v1"` MetadataEvent *SubscribeMetadataResponse `protobuf:"bytes,1,opt,name=metadata_event,json=metadataEvent,proto3" json:"metadata_event,omitempty"` + // filer log position stamped under the path lock before the write: + // every event at or below it is reflected in the acknowledged state + LogTsNs int64 `protobuf:"varint,2,opt,name=log_ts_ns,json=logTsNs,proto3" json:"log_ts_ns,omitempty"` + LogSignature int32 `protobuf:"varint,3,opt,name=log_signature,json=logSignature,proto3" json:"log_signature,omitempty"` // filer whose clock stamped log_ts_ns unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2437,6 +2479,20 @@ func (x *UpdateEntryResponse) GetMetadataEvent() *SubscribeMetadataResponse { return nil } +func (x *UpdateEntryResponse) GetLogTsNs() int64 { + if x != nil { + return x.LogTsNs + } + return 0 +} + +func (x *UpdateEntryResponse) GetLogSignature() int32 { + if x != nil { + return x.LogSignature + } + return 0 +} + type TouchAccessTimeRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Directory string `protobuf:"bytes,1,opt,name=directory,proto3" json:"directory,omitempty"` @@ -5262,6 +5318,10 @@ type CacheRemoteObjectToLocalClusterResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Entry *Entry `protobuf:"bytes,1,opt,name=entry,proto3" json:"entry,omitempty"` MetadataEvent *SubscribeMetadataResponse `protobuf:"bytes,2,opt,name=metadata_event,json=metadataEvent,proto3" json:"metadata_event,omitempty"` + // filer log position stamped before the entry read: every event at or + // below it is reflected in the returned entry + LogTsNs int64 `protobuf:"varint,3,opt,name=log_ts_ns,json=logTsNs,proto3" json:"log_ts_ns,omitempty"` + LogSignature int32 `protobuf:"varint,4,opt,name=log_signature,json=logSignature,proto3" json:"log_signature,omitempty"` // filer whose clock stamped log_ts_ns unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5310,6 +5370,20 @@ func (x *CacheRemoteObjectToLocalClusterResponse) GetMetadataEvent() *SubscribeM return nil } +func (x *CacheRemoteObjectToLocalClusterResponse) GetLogTsNs() int64 { + if x != nil { + return x.LogTsNs + } + return 0 +} + +func (x *CacheRemoteObjectToLocalClusterResponse) GetLogSignature() int32 { + if x != nil { + return x.LogSignature + } + return 0 +} + // /////////////////////// // distributed lock management // /////////////////////// @@ -6843,9 +6917,11 @@ const file_filer_proto_rawDesc = "" + "\vfiler.proto\x12\bfiler_pb\"O\n" + "\x1bLookupDirectoryEntryRequest\x12\x1c\n" + "\tdirectory\x18\x01 \x01(\tR\tdirectory\x12\x12\n" + - "\x04name\x18\x02 \x01(\tR\x04name\"E\n" + + "\x04name\x18\x02 \x01(\tR\x04name\"\x86\x01\n" + "\x1cLookupDirectoryEntryResponse\x12%\n" + - "\x05entry\x18\x01 \x01(\v2\x0f.filer_pb.EntryR\x05entry\"\xe4\x01\n" + + "\x05entry\x18\x01 \x01(\v2\x0f.filer_pb.EntryR\x05entry\x12\x1a\n" + + "\tlog_ts_ns\x18\x02 \x01(\x03R\alogTsNs\x12#\n" + + "\rlog_signature\x18\x03 \x01(\x05R\flogSignature\"\xe4\x01\n" + "\x12ListEntriesRequest\x12\x1c\n" + "\tdirectory\x18\x01 \x01(\tR\tdirectory\x12\x16\n" + "\x06prefix\x18\x02 \x01(\tR\x06prefix\x12,\n" + @@ -7058,12 +7134,14 @@ const file_filer_proto_rawDesc = "" + "\x1dObjectTransactionBatchRequest\x12F\n" + "\ftransactions\x18\x01 \x03(\v2\".filer_pb.ObjectTransactionRequestR\ftransactions\"c\n" + "\x1eObjectTransactionBatchResponse\x12A\n" + - "\tresponses\x18\x01 \x03(\v2#.filer_pb.ObjectTransactionResponseR\tresponses\"\xac\x01\n" + + "\tresponses\x18\x01 \x03(\v2#.filer_pb.ObjectTransactionResponseR\tresponses\"\xed\x01\n" + "\x13CreateEntryResponse\x12\x14\n" + "\x05error\x18\x01 \x01(\tR\x05error\x12J\n" + "\x0emetadata_event\x18\x02 \x01(\v2#.filer_pb.SubscribeMetadataResponseR\rmetadataEvent\x123\n" + "\n" + - "error_code\x18\x03 \x01(\x0e2\x14.filer_pb.FilerErrorR\terrorCode\"\x8a\x03\n" + + "error_code\x18\x03 \x01(\x0e2\x14.filer_pb.FilerErrorR\terrorCode\x12\x1a\n" + + "\tlog_ts_ns\x18\x04 \x01(\x03R\alogTsNs\x12#\n" + + "\rlog_signature\x18\x05 \x01(\x05R\flogSignature\"\x8a\x03\n" + "\x12UpdateEntryRequest\x12\x1c\n" + "\tdirectory\x18\x01 \x01(\tR\tdirectory\x12%\n" + "\x05entry\x18\x02 \x01(\v2\x0f.filer_pb.EntryR\x05entry\x121\n" + @@ -7075,9 +7153,11 @@ const file_filer_proto_rawDesc = "" + "\tcondition\x18\x06 \x01(\v2\x18.filer_pb.WriteConditionR\tcondition\x1aC\n" + "\x15ExpectedExtendedEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\fR\x05value:\x028\x01\"a\n" + + "\x05value\x18\x02 \x01(\fR\x05value:\x028\x01\"\xa2\x01\n" + "\x13UpdateEntryResponse\x12J\n" + - "\x0emetadata_event\x18\x01 \x01(\v2#.filer_pb.SubscribeMetadataResponseR\rmetadataEvent\"r\n" + + "\x0emetadata_event\x18\x01 \x01(\v2#.filer_pb.SubscribeMetadataResponseR\rmetadataEvent\x12\x1a\n" + + "\tlog_ts_ns\x18\x02 \x01(\x03R\alogTsNs\x12#\n" + + "\rlog_signature\x18\x03 \x01(\x05R\flogSignature\"r\n" + "\x16TouchAccessTimeRequest\x12\x1c\n" + "\tdirectory\x18\x01 \x01(\tR\tdirectory\x12\x12\n" + "\x04name\x18\x02 \x01(\tR\x04name\x12&\n" + @@ -7333,10 +7413,12 @@ const file_filer_proto_rawDesc = "" + "\tdirectory\x18\x01 \x01(\tR\tdirectory\x12\x12\n" + "\x04name\x18\x02 \x01(\tR\x04name\x12+\n" + "\x11chunk_concurrency\x18\x03 \x01(\x05R\x10chunkConcurrency\x121\n" + - "\x14download_concurrency\x18\x04 \x01(\x05R\x13downloadConcurrency\"\x9c\x01\n" + + "\x14download_concurrency\x18\x04 \x01(\x05R\x13downloadConcurrency\"\xdd\x01\n" + "'CacheRemoteObjectToLocalClusterResponse\x12%\n" + "\x05entry\x18\x01 \x01(\v2\x0f.filer_pb.EntryR\x05entry\x12J\n" + - "\x0emetadata_event\x18\x02 \x01(\v2#.filer_pb.SubscribeMetadataResponseR\rmetadataEvent\"\x9b\x01\n" + + "\x0emetadata_event\x18\x02 \x01(\v2#.filer_pb.SubscribeMetadataResponseR\rmetadataEvent\x12\x1a\n" + + "\tlog_ts_ns\x18\x03 \x01(\x03R\alogTsNs\x12#\n" + + "\rlog_signature\x18\x04 \x01(\x05R\flogSignature\"\x9b\x01\n" + "\vLockRequest\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12&\n" + "\x0fseconds_to_lock\x18\x02 \x01(\x03R\rsecondsToLock\x12\x1f\n" + diff --git a/weed/pb/filer_pb/filer_client.go b/weed/pb/filer_pb/filer_client.go index 3e7f9859e..93f86a2ff 100644 --- a/weed/pb/filer_pb/filer_client.go +++ b/weed/pb/filer_pb/filer_client.go @@ -7,6 +7,7 @@ import ( "io" "math" "os" + "strconv" "strings" "time" @@ -25,7 +26,11 @@ type FilerClient interface { GetDataCenter() string } -func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath util.FullPath) (entry *Entry, err error) { +// GetEntry returns the entry together with the log position the filer stamped +// for it and the signature of the filer that stamped it — positions are only +// comparable within one filer's clock. Both are zero against a filer that does +// not stamp them. +func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath util.FullPath) (entry *Entry, logTsNs int64, logSignature int32, err error) { dir, name := fullFilePath.DirAndName() @@ -49,12 +54,19 @@ func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath util.Fu } entry = resp.Entry + logTsNs = resp.LogTsNs + logSignature = resp.LogSignature return nil }) return } +// ListSnapshotTsNsTrailerKey carries the listing snapshot in the ListEntries +// stream trailer, so empty listings can convey it without a response older +// consumers would mistake for an entry. +const ListSnapshotTsNsTrailerKey = "sw-list-snapshot-ts-ns" + type EachEntryFunction func(entry *Entry, isLast bool) error func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunction) (err error) { @@ -156,6 +168,16 @@ func DoSeaweedListWithSnapshot(ctx context.Context, client SeaweedFilerClient, f return actualSnapshotTsNs, err } } + // An empty listing carries no in-band snapshot (a snapshot-only + // response would be read as an entry by older consumers); newer + // filers send it in the stream trailer instead. + if actualSnapshotTsNs == 0 { + if values := stream.Trailer().Get(ListSnapshotTsNsTrailerKey); len(values) > 0 { + if trailerTsNs, parseErr := strconv.ParseInt(values[0], 10, 64); parseErr == nil { + actualSnapshotTsNs = trailerTsNs + } + } + } break } else { return actualSnapshotTsNs, recvErr diff --git a/weed/pb/filer_pb/filer_vtproto.pb.go b/weed/pb/filer_pb/filer_vtproto.pb.go index 9bdbaba8e..93995c539 100644 --- a/weed/pb/filer_pb/filer_vtproto.pb.go +++ b/weed/pb/filer_pb/filer_vtproto.pb.go @@ -96,6 +96,16 @@ func (m *LookupDirectoryEntryResponse) MarshalToSizedBufferVT(dAtA []byte) (int, i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.LogSignature != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LogSignature)) + i-- + dAtA[i] = 0x18 + } + if m.LogTsNs != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LogTsNs)) + i-- + dAtA[i] = 0x10 + } if m.Entry != nil { size, err := m.Entry.MarshalToSizedBufferVT(dAtA[:i]) if err != nil { @@ -1992,6 +2002,16 @@ func (m *CreateEntryResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.LogSignature != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LogSignature)) + i-- + dAtA[i] = 0x28 + } + if m.LogTsNs != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LogTsNs)) + i-- + dAtA[i] = 0x20 + } if m.ErrorCode != 0 { i = protohelpers.EncodeVarint(dAtA, i, uint64(m.ErrorCode)) i-- @@ -2157,6 +2177,16 @@ func (m *UpdateEntryResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.LogSignature != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LogSignature)) + i-- + dAtA[i] = 0x18 + } + if m.LogTsNs != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LogTsNs)) + i-- + dAtA[i] = 0x10 + } if m.MetadataEvent != nil { size, err := m.MetadataEvent.MarshalToSizedBufferVT(dAtA[:i]) if err != nil { @@ -5020,6 +5050,16 @@ func (m *CacheRemoteObjectToLocalClusterResponse) MarshalToSizedBufferVT(dAtA [] i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.LogSignature != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LogSignature)) + i-- + dAtA[i] = 0x20 + } + if m.LogTsNs != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LogTsNs)) + i-- + dAtA[i] = 0x18 + } if m.MetadataEvent != nil { size, err := m.MetadataEvent.MarshalToSizedBufferVT(dAtA[:i]) if err != nil { @@ -6198,6 +6238,12 @@ func (m *LookupDirectoryEntryResponse) SizeVT() (n int) { l = m.Entry.SizeVT() n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if m.LogTsNs != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.LogTsNs)) + } + if m.LogSignature != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.LogSignature)) + } n += len(m.unknownFields) return n } @@ -6967,6 +7013,12 @@ func (m *CreateEntryResponse) SizeVT() (n int) { if m.ErrorCode != 0 { n += 1 + protohelpers.SizeOfVarint(uint64(m.ErrorCode)) } + if m.LogTsNs != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.LogTsNs)) + } + if m.LogSignature != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.LogSignature)) + } n += len(m.unknownFields) return n } @@ -7022,6 +7074,12 @@ func (m *UpdateEntryResponse) SizeVT() (n int) { l = m.MetadataEvent.SizeVT() n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if m.LogTsNs != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.LogTsNs)) + } + if m.LogSignature != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.LogSignature)) + } n += len(m.unknownFields) return n } @@ -8162,6 +8220,12 @@ func (m *CacheRemoteObjectToLocalClusterResponse) SizeVT() (n int) { l = m.MetadataEvent.SizeVT() n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if m.LogTsNs != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.LogTsNs)) + } + if m.LogSignature != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.LogSignature)) + } n += len(m.unknownFields) return n } @@ -8817,6 +8881,44 @@ func (m *LookupDirectoryEntryResponse) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LogTsNs", wireType) + } + m.LogTsNs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LogTsNs |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LogSignature", wireType) + } + m.LogSignature = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LogSignature |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -14138,6 +14240,44 @@ func (m *CreateEntryResponse) UnmarshalVT(dAtA []byte) error { break } } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LogTsNs", wireType) + } + m.LogTsNs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LogTsNs |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LogSignature", wireType) + } + m.LogSignature = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LogSignature |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -14604,6 +14744,44 @@ func (m *UpdateEntryResponse) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LogTsNs", wireType) + } + m.LogTsNs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LogTsNs |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LogSignature", wireType) + } + m.LogSignature = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LogSignature |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -21902,6 +22080,44 @@ func (m *CacheRemoteObjectToLocalClusterResponse) UnmarshalVT(dAtA []byte) error return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LogTsNs", wireType) + } + m.LogTsNs = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LogTsNs |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LogSignature", wireType) + } + m.LogSignature = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LogSignature |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/weed/pb/filer_pb/filer_vtproto_test.go b/weed/pb/filer_pb/filer_vtproto_test.go index 4a422813f..5d1611a75 100644 --- a/weed/pb/filer_pb/filer_vtproto_test.go +++ b/weed/pb/filer_pb/filer_vtproto_test.go @@ -263,3 +263,44 @@ func BenchmarkMarshalIntoBuffer(b *testing.B) { }) } } + +// The log position fields must survive VT round-trips, or a VT-marshaled +// response silently loses the version the mount fences handles with. +func TestVtProtoPreservesLogTsNs(t *testing.T) { + messages := []interface { + MarshalVT() ([]byte, error) + }{ + &LookupDirectoryEntryResponse{LogTsNs: 42}, + &CacheRemoteObjectToLocalClusterResponse{LogTsNs: 43}, + &CreateEntryResponse{LogTsNs: 44}, + &UpdateEntryResponse{LogTsNs: 45}, + } + for i, m := range messages { + data, err := m.MarshalVT() + if err != nil { + t.Fatalf("MarshalVT %d: %v", i, err) + } + switch v := m.(type) { + case *LookupDirectoryEntryResponse: + out := &LookupDirectoryEntryResponse{} + if err := out.UnmarshalVT(data); err != nil || out.LogTsNs != v.LogTsNs { + t.Fatalf("LookupDirectoryEntryResponse round-trip LogTsNs = %d, %v; want %d", out.LogTsNs, err, v.LogTsNs) + } + case *CacheRemoteObjectToLocalClusterResponse: + out := &CacheRemoteObjectToLocalClusterResponse{} + if err := out.UnmarshalVT(data); err != nil || out.LogTsNs != v.LogTsNs { + t.Fatalf("CacheRemoteObjectToLocalClusterResponse round-trip LogTsNs = %d, %v; want %d", out.LogTsNs, err, v.LogTsNs) + } + case *CreateEntryResponse: + out := &CreateEntryResponse{} + if err := out.UnmarshalVT(data); err != nil || out.LogTsNs != v.LogTsNs { + t.Fatalf("CreateEntryResponse round-trip LogTsNs = %d, %v; want %d", out.LogTsNs, err, v.LogTsNs) + } + case *UpdateEntryResponse: + out := &UpdateEntryResponse{} + if err := out.UnmarshalVT(data); err != nil || out.LogTsNs != v.LogTsNs { + t.Fatalf("UpdateEntryResponse round-trip LogTsNs = %d, %v; want %d", out.LogTsNs, err, v.LogTsNs) + } + } + } +} diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index d9d553de8..845427c5c 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -517,7 +517,7 @@ func SourceSupersedes(ctx context.Context, filerSource *source.FilerSource, sour if filerSource == nil { return false } - sourceEntry, err := filer_pb.GetEntry(ctx, filerSource, sourcePath) + sourceEntry, _, _, err := filer_pb.GetEntry(ctx, filerSource, sourcePath) return sourceSupersedes(sourcePath, sourceEntry, err, mtimeNs) } diff --git a/weed/s3api/filer_multipart_etag_test.go b/weed/s3api/filer_multipart_etag_test.go index 9ff578316..9445853df 100644 --- a/weed/s3api/filer_multipart_etag_test.go +++ b/weed/s3api/filer_multipart_etag_test.go @@ -118,4 +118,4 @@ func mustDecodeHexETagForTest(t *testing.T, etag string) []byte { t.Fatalf("decode hex ETag %q: %v", etag, err) } return decoded -} \ No newline at end of file +} diff --git a/weed/s3api/filer_multipart_sse_s3_test.go b/weed/s3api/filer_multipart_sse_s3_test.go index 38f6f07f5..e4e39c909 100644 --- a/weed/s3api/filer_multipart_sse_s3_test.go +++ b/weed/s3api/filer_multipart_sse_s3_test.go @@ -136,7 +136,7 @@ func TestApplyMultipartSSES3HeadersFromUploadEntry(t *testing.T) { existingKey := []byte("existing-key-bytes") existingHeader := []byte("aws:kms") finalEntry := &filer_pb.Entry{Extended: map[string][]byte{ - s3_constants.SeaweedFSSSES3Key: existingKey, + s3_constants.SeaweedFSSSES3Key: existingKey, s3_constants.AmzServerSideEncryption: existingHeader, }} applyMultipartSSES3HeadersFromUploadEntry(finalEntry, sses3Info) diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go index 5b22434c3..9a883bbf3 100644 --- a/weed/s3api/filer_util.go +++ b/weed/s3api/filer_util.go @@ -169,7 +169,8 @@ func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isD func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *filer_pb.Entry, err error) { fullPath := util.NewFullPath(parentDirectoryPath, entryName) - return filer_pb.GetEntry(context.Background(), s3a, fullPath) + entry, _, _, err = filer_pb.GetEntry(context.Background(), s3a, fullPath) + return entry, err } func (s3a *S3ApiServer) updateEntry(parentDirectoryPath string, newEntry *filer_pb.Entry) error { diff --git a/weed/s3api/s3_sse_kms.go b/weed/s3api/s3_sse_kms.go index bc1542b30..25f911346 100644 --- a/weed/s3api/s3_sse_kms.go +++ b/weed/s3api/s3_sse_kms.go @@ -43,14 +43,14 @@ type SSEKMSKey struct { // SSEKMSMetadata represents the metadata stored with SSE-KMS objects type SSEKMSMetadata struct { - Algorithm string `json:"algorithm"` // "aws:kms" - KeyID string `json:"keyId"` // KMS key identifier - EncryptedDataKey string `json:"encryptedDataKey"` // Base64-encoded encrypted data key - EncryptionContext map[string]string `json:"encryptionContext"` // Encryption context - BucketKeyEnabled bool `json:"bucketKeyEnabled"` // S3 Bucket Key optimization - IV string `json:"iv"` // Base64-encoded initialization vector - PartOffset int64 `json:"partOffset"` // Offset within original multipart part (for IV calculation) - KeyCommitment string `json:"keyCommitment,omitempty"` // Base64-encoded HMAC key commitment + Algorithm string `json:"algorithm"` // "aws:kms" + KeyID string `json:"keyId"` // KMS key identifier + EncryptedDataKey string `json:"encryptedDataKey"` // Base64-encoded encrypted data key + EncryptionContext map[string]string `json:"encryptionContext"` // Encryption context + BucketKeyEnabled bool `json:"bucketKeyEnabled"` // S3 Bucket Key optimization + IV string `json:"iv"` // Base64-encoded initialization vector + PartOffset int64 `json:"partOffset"` // Offset within original multipart part (for IV calculation) + KeyCommitment string `json:"keyCommitment,omitempty"` // Base64-encoded HMAC key commitment } const ( diff --git a/weed/s3api/s3_sse_s3.go b/weed/s3api/s3_sse_s3.go index 9625f1117..8ce3542d2 100644 --- a/weed/s3api/s3_sse_s3.go +++ b/weed/s3api/s3_sse_s3.go @@ -271,11 +271,11 @@ func DeserializeSSES3Metadata(data []byte, keyManager *SSES3KeyManager) (*SSES3K // SSES3KeyManager manages SSE-S3 encryption keys using envelope encryption // Instead of storing keys in memory, it uses a super key (KEK) to encrypt/decrypt DEKs type SSES3KeyManager struct { - mu sync.RWMutex - superKey []byte // 256-bit master key (KEK - Key Encryption Key) - filerClient filer_pb.FilerClient // Filer client for KEK persistence - kekPath string // Path in filer where KEK is stored (e.g., /etc/s3/sse_kek) - kekPassphrase string // If set, KEK is encrypted at rest using a key derived from this passphrase + mu sync.RWMutex + superKey []byte // 256-bit master key (KEK - Key Encryption Key) + filerClient filer_pb.FilerClient // Filer client for KEK persistence + kekPath string // Path in filer where KEK is stored (e.g., /etc/s3/sse_kek) + kekPassphrase string // If set, KEK is encrypted at rest using a key derived from this passphrase } const ( @@ -321,7 +321,6 @@ var kekWrappedV2Magic = []byte{0x53, 0x57, 0x76, 0x32} // "SWv2" // 32 bytes matches the SHA-256 output and is the standard recommendation. const kekRandomSaltSize = 32 - // NewSSES3KeyManager creates a new SSE-S3 key manager with envelope encryption. // If kekPassphrase is non-empty, the KEK is encrypted at rest using a key derived from it. func NewSSES3KeyManager(kekPassphrase ...string) *SSES3KeyManager { @@ -583,7 +582,7 @@ func (km *SSES3KeyManager) loadSuperKeyFromFiler() error { } // Get the entry from filer - entry, err := filer_pb.GetEntry(context.Background(), km.filerClient, util.FullPath(km.kekPath)) + entry, _, _, err := filer_pb.GetEntry(context.Background(), km.filerClient, util.FullPath(km.kekPath)) if err != nil { return fmt.Errorf("failed to get KEK entry from filer: %w", err) } diff --git a/weed/s3api/s3api_embedded_iam_oidc.go b/weed/s3api/s3api_embedded_iam_oidc.go index e3f478d06..ccfdf9ee7 100644 --- a/weed/s3api/s3api_embedded_iam_oidc.go +++ b/weed/s3api/s3api_embedded_iam_oidc.go @@ -14,15 +14,15 @@ import ( // OIDC provider IAM actions handled by this file. const ( - actionGetOpenIDConnectProvider = "GetOpenIDConnectProvider" - actionListOpenIDConnectProviders = "ListOpenIDConnectProviders" - actionCreateOpenIDConnectProvider = "CreateOpenIDConnectProvider" - actionDeleteOpenIDConnectProvider = "DeleteOpenIDConnectProvider" - actionAddClientIDToOpenIDConnectProvider = "AddClientIDToOpenIDConnectProvider" - actionRemoveClientIDFromOpenIDConnectProvider = "RemoveClientIDFromOpenIDConnectProvider" - actionUpdateOpenIDConnectProviderThumbprint = "UpdateOpenIDConnectProviderThumbprint" - actionTagOpenIDConnectProvider = "TagOpenIDConnectProvider" - actionUntagOpenIDConnectProvider = "UntagOpenIDConnectProvider" + actionGetOpenIDConnectProvider = "GetOpenIDConnectProvider" + actionListOpenIDConnectProviders = "ListOpenIDConnectProviders" + actionCreateOpenIDConnectProvider = "CreateOpenIDConnectProvider" + actionDeleteOpenIDConnectProvider = "DeleteOpenIDConnectProvider" + actionAddClientIDToOpenIDConnectProvider = "AddClientIDToOpenIDConnectProvider" + actionRemoveClientIDFromOpenIDConnectProvider = "RemoveClientIDFromOpenIDConnectProvider" + actionUpdateOpenIDConnectProviderThumbprint = "UpdateOpenIDConnectProviderThumbprint" + actionTagOpenIDConnectProvider = "TagOpenIDConnectProvider" + actionUntagOpenIDConnectProvider = "UntagOpenIDConnectProvider" ) // isOIDCProviderAction reports whether an action belongs to the OIDC provider diff --git a/weed/s3api/s3api_object_routed_read.go b/weed/s3api/s3api_object_routed_read.go index ec5661064..1dc060726 100644 --- a/weed/s3api/s3api_object_routed_read.go +++ b/weed/s3api/s3api_object_routed_read.go @@ -25,7 +25,8 @@ func (s3a *S3ApiServer) getObjectEntryRoutedByKey(bucket, object string) (*filer owner := s3a.routableWriteOwner(bucket, object) if owner == "" || s3a.filerClient == nil { - return filer_pb.GetEntry(context.Background(), s3a, fullPath) + entry, _, _, err := filer_pb.GetEntry(context.Background(), s3a, fullPath) + return entry, err } // Skip an owner whose recent read hit a transport error; read local-first until diff --git a/weed/s3api/s3lifecycle/dailyrun/filer_list_func_test.go b/weed/s3api/s3lifecycle/dailyrun/filer_list_func_test.go index 7796a5da6..2e3083c12 100644 --- a/weed/s3api/s3lifecycle/dailyrun/filer_list_func_test.go +++ b/weed/s3api/s3lifecycle/dailyrun/filer_list_func_test.go @@ -148,8 +148,8 @@ func TestFilerListFunc_PropagatesTagsOnFlatFiles(t *testing.T) { func TestFilerListFunc_RecursesIntoSubdirs(t *testing.T) { mtime := time.Now() client := &fakeFiler{tree: map[string][]*filer_pb.Entry{ - "/buckets/bkt": {dir("logs"), file("root.txt", mtime, 1)}, - "/buckets/bkt/logs": {dir("2026"), file("a.log", mtime, 5)}, + "/buckets/bkt": {dir("logs"), file("root.txt", mtime, 1)}, + "/buckets/bkt/logs": {dir("2026"), file("a.log", mtime, 5)}, "/buckets/bkt/logs/2026": {file("b.log", mtime, 7)}, }} listFn := FilerListFunc(client, "/buckets") diff --git a/weed/s3api/s3lifecycle/dailyrun/process_matches_test.go b/weed/s3api/s3lifecycle/dailyrun/process_matches_test.go index 20cd91c96..c61330f96 100644 --- a/weed/s3api/s3lifecycle/dailyrun/process_matches_test.go +++ b/weed/s3api/s3lifecycle/dailyrun/process_matches_test.go @@ -7,12 +7,12 @@ import ( "testing" "time" + dto "github.com/prometheus/client_model/go" "github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router" "github.com/seaweedfs/seaweedfs/weed/stats" - dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/weed/s3api/s3lifecycle/dailyrun/run.go b/weed/s3api/s3lifecycle/dailyrun/run.go index 330ad48d6..0c845cacc 100644 --- a/weed/s3api/s3lifecycle/dailyrun/run.go +++ b/weed/s3api/s3lifecycle/dailyrun/run.go @@ -207,10 +207,10 @@ func Run(ctx context.Context, cfg Config) error { // the state. func summarizeShardCursorLag(ctx context.Context, cfg Config, runNow time.Time) string { var ( - maxLag time.Duration - maxAge time.Duration - anyCursor bool - anyWalked bool + maxLag time.Duration + maxAge time.Duration + anyCursor bool + anyWalked bool ) for _, sh := range cfg.Shards { c, found, err := cfg.Persister.Load(ctx, sh) diff --git a/weed/s3api/s3lifecycle/dailyrun/walker_interval_test.go b/weed/s3api/s3lifecycle/dailyrun/walker_interval_test.go index 26099ee83..e6af3d875 100644 --- a/weed/s3api/s3lifecycle/dailyrun/walker_interval_test.go +++ b/weed/s3api/s3lifecycle/dailyrun/walker_interval_test.go @@ -64,11 +64,11 @@ func TestWalkerDue(t *testing.T) { // (tests, in-repo integration) doesn't regress. func TestRunShard_WalkerThrottle(t *testing.T) { cases := []struct { - name string - interval time.Duration - secondPassAfter time.Duration - wantTotalCalls int - wantSecondAdvNs bool // did LastWalkedNs change between pass 1 and pass 2? + name string + interval time.Duration + secondPassAfter time.Duration + wantTotalCalls int + wantSecondAdvNs bool // did LastWalkedNs change between pass 1 and pass 2? }{ {"interval=0 fires every pass", 0, 30 * time.Second, 2, true}, {"throttled: second pass within interval", time.Hour, 30 * time.Second, 1, false}, diff --git a/weed/s3api/s3lifecycle/engine/engine_test.go b/weed/s3api/s3lifecycle/engine/engine_test.go index 829c7c915..18e6da714 100644 --- a/weed/s3api/s3lifecycle/engine/engine_test.go +++ b/weed/s3api/s3lifecycle/engine/engine_test.go @@ -379,4 +379,3 @@ func TestCompile_DelayGroupsDedupeAcrossBuckets(t *testing.T) { } } } - diff --git a/weed/s3api/s3lifecycle/engine/hashes_test.go b/weed/s3api/s3lifecycle/engine/hashes_test.go index ca849191c..e139ba5c3 100644 --- a/weed/s3api/s3lifecycle/engine/hashes_test.go +++ b/weed/s3api/s3lifecycle/engine/hashes_test.go @@ -155,8 +155,8 @@ func TestPromotedHash_ChangesOnWalkToReplayDemotion(t *testing.T) { } snap := buildSnapshotForViews(t, "b1", rule) - before := PromotedHash(snap, s3lifecycle.DaysToDuration(7)) // walk - after := PromotedHash(snap, s3lifecycle.DaysToDuration(365)) // demoted back to replay + before := PromotedHash(snap, s3lifecycle.DaysToDuration(7)) // walk + after := PromotedHash(snap, s3lifecycle.DaysToDuration(365)) // demoted back to replay var empty [32]byte assert.NotEqual(t, empty, before) diff --git a/weed/s3api/s3lifecycle/reader/cursor_test.go b/weed/s3api/s3lifecycle/reader/cursor_test.go index 111c121d2..d4f1e6718 100644 --- a/weed/s3api/s3lifecycle/reader/cursor_test.go +++ b/weed/s3api/s3lifecycle/reader/cursor_test.go @@ -21,7 +21,7 @@ func TestCursorAdvanceMonotonic(t *testing.T) { c := NewCursor() k := key("b", s3lifecycle.ActionKindExpirationDays) c.Advance(k, 100) - c.Advance(k, 50) // backward, ignored + c.Advance(k, 50) // backward, ignored c.Advance(k, 100) // equal, ignored c.Advance(k, 200) if got := c.Get(k); got != 200 { diff --git a/weed/s3api/s3lifecycle/reader/reader.go b/weed/s3api/s3lifecycle/reader/reader.go index e75a224d9..a82015d96 100644 --- a/weed/s3api/s3lifecycle/reader/reader.go +++ b/weed/s3api/s3lifecycle/reader/reader.go @@ -64,8 +64,8 @@ func (e *Event) IsCreate() bool { type Reader struct { // ShardID and ShardPredicate are alternatives — set at most one. // ShardPredicate wins if both are populated. - ShardID int // [0, s3lifecycle.ShardCount); used when ShardPredicate is nil - ShardPredicate func(int) bool // accepts an event when true; nil falls back to ShardID equality + ShardID int // [0, s3lifecycle.ShardCount); used when ShardPredicate is nil + ShardPredicate func(int) bool // accepts an event when true; nil falls back to ShardID equality BucketsPath string // e.g. "/buckets" // Cursor is the single-shard cursor used for SinceNs when StartTsNs is 0. diff --git a/weed/s3api/s3lifecycle/router/router.go b/weed/s3api/s3lifecycle/router/router.go index 6c894acf1..369172c9c 100644 --- a/weed/s3api/s3lifecycle/router/router.go +++ b/weed/s3api/s3lifecycle/router/router.go @@ -52,15 +52,15 @@ type Survivors struct { // so the dispatcher can locate the specific version, and VersionID // carries the AWS-visible version ID separately. type Match struct { - Key s3lifecycle.ActionKey - Action *engine.CompiledAction - Result s3lifecycle.EvalResult - EventTs time.Time - DueTime time.Time - Bucket string + Key s3lifecycle.ActionKey + Action *engine.CompiledAction + Result s3lifecycle.EvalResult + EventTs time.Time + DueTime time.Time + Bucket string ObjectKey string VersionID string - Identity *EntryIdentity + Identity *EntryIdentity } // EntryIdentity is the schedule-time CAS witness; the dispatcher serializes diff --git a/weed/s3api/s3lifecycle/router/router_test.go b/weed/s3api/s3lifecycle/router/router_test.go index a9b752a0c..ddce9a674 100644 --- a/weed/s3api/s3lifecycle/router/router_test.go +++ b/weed/s3api/s3lifecycle/router/router_test.go @@ -152,7 +152,7 @@ func TestRouteSkipsHardDelete(t *testing.T) { Bucket: "bk", Key: "gone.txt", OldEntry: &filer_pb.Entry{ - Name: "gone.txt", + Name: "gone.txt", Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 1}, }, } @@ -500,19 +500,19 @@ func markerEvent(bucket, logicalKey, versionID string, mtimeUnix, mtimeNs int64) // state to return; calls list is appended on each invocation so tests // can assert whether the lister was consulted at all. type recordingLister struct { - calls []string - survivors Survivors - err error - lookupCalls []string - lookupEntry *filer_pb.Entry - lookupErr error - listCalls []string - listVersions []*filer_pb.Entry - listErr error - nullCalls []string - nullEntry *filer_pb.Entry - nullExplicit bool - nullErr error + calls []string + survivors Survivors + err error + lookupCalls []string + lookupEntry *filer_pb.Entry + lookupErr error + listCalls []string + listVersions []*filer_pb.Entry + listErr error + nullCalls []string + nullEntry *filer_pb.Entry + nullExplicit bool + nullErr error } func (r *recordingLister) ListVersions(_ context.Context, bucket, key string) ([]*filer_pb.Entry, error) { @@ -854,7 +854,6 @@ func TestRouteVersionedAllVersionFolderPathsSkipped(t *testing.T) { } } - func bootstrapVersionEntry(versionID string, mtime time.Time, isDeleteMarker bool) *filer_pb.Entry { ext := map[string][]byte{ s3_constants.ExtVersionIdKey: []byte(versionID), @@ -1431,7 +1430,7 @@ func TestRoutePointerTransitionNewerNoncurrentExpansionFiresOnCrossingThreshold( // Successor mtime (cached on container) is "now" — the new latest. ev := versionsContainerEvent("bk", "logs/foo", "v-cur", "v-new", now.Unix()) allVersions := []*filer_pb.Entry{ - displacedVersionEntry("v-new", now.Unix()), // newest + displacedVersionEntry("v-new", now.Unix()), // newest displacedVersionEntry("v-cur", now.AddDate(0, 0, -1).Unix()), displacedVersionEntry("v-mid", now.AddDate(0, 0, -10).Unix()), displacedVersionEntry("v-old", now.AddDate(0, 0, -30).Unix()), // oldest diff --git a/weed/s3api/sses3_multipart_repro_test.go b/weed/s3api/sses3_multipart_repro_test.go index 035be7e21..fde2ee108 100644 --- a/weed/s3api/sses3_multipart_repro_test.go +++ b/weed/s3api/sses3_multipart_repro_test.go @@ -40,11 +40,11 @@ func TestMultipartSSES3RealisticEndToEnd(t *testing.T) { // Realistic mix of part sizes: small (one chunk), exact 8MB, >8MB (two // chunks), much larger (multiple chunks). partSizes := []int{ - 5 * 1024 * 1024, // 5MB (single chunk) - 8 * 1024 * 1024, // 8MB exactly (single chunk, full) - 8*1024*1024 + 123, // crosses chunk boundary (two chunks) - 17 * 1024 * 1024, // three chunks - 1234, // tiny + 5 * 1024 * 1024, // 5MB (single chunk) + 8 * 1024 * 1024, // 8MB exactly (single chunk, full) + 8*1024*1024 + 123, // crosses chunk boundary (two chunks) + 17 * 1024 * 1024, // three chunks + 1234, // tiny } parts := make([][]byte, len(partSizes)) @@ -57,7 +57,7 @@ func TestMultipartSSES3RealisticEndToEnd(t *testing.T) { // store per-chunk metadata IV = calculateIVWithOffset(baseIV, partLocalOff), // then assign GLOBAL offsets to the FileChunk. type chunkBlob struct { - fid string + fid string ciphertext []byte } var chunks []*filer_pb.FileChunk diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index f4d2025f9..02d857e32 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -23,16 +23,30 @@ import ( "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/wdclient" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" ) +// fencedFindEntry reads an entry with an exact log-position fence: the stamp +// and the read share the path lock mutations hold across write and notify, so +// every event at or below the fence is in the entry and none above it are. +// Any read whose result a client versions must go through here. +func (fs *FilerServer) fencedFindEntry(ctx context.Context, path util.FullPath) (entry *filer.Entry, logTsNs int64, err error) { + pathLock := fs.entryLockTable.AcquireLock("fencedFindEntry", path, util.SharedLock) + logTsNs = time.Now().UnixNano() + entry, err = fs.filer.FindEntry(ctx, path) + fs.entryLockTable.ReleaseLock(path, pathLock) + return +} + func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) { glog.V(4).InfofCtx(ctx, "LookupDirectoryEntry %s", filepath.Join(req.Directory, req.Name)) - entry, err := fs.filer.FindEntry(ctx, util.JoinPath(req.Directory, req.Name)) + entry, logTsNs, err := fs.fencedFindEntry(ctx, util.JoinPath(req.Directory, req.Name)) + if err == filer_pb.ErrNotFound { - return &filer_pb.LookupDirectoryEntryResponse{}, err + return &filer_pb.LookupDirectoryEntryResponse{LogTsNs: logTsNs, LogSignature: fs.filer.Signature}, err } if err != nil { glog.V(3).InfofCtx(ctx, "LookupDirectoryEntry %s: %+v, ", filepath.Join(req.Directory, req.Name), err) @@ -40,7 +54,9 @@ func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.L } return &filer_pb.LookupDirectoryEntryResponse{ - Entry: entry.ToProtoEntry(), + Entry: entry.ToProtoEntry(), + LogTsNs: logTsNs, + LogSignature: fs.filer.Signature, }, nil } @@ -102,12 +118,10 @@ func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream file } - // For empty directories we intentionally do NOT send a snapshot-only - // response (Entry == nil). Many consumers (Java FilerClient, S3 listing, - // etc.) treat any received response as an entry. The Go client-side - // DoSeaweedListWithSnapshot generates a client-side cutoff when the - // server sends no snapshot, so snapshot consistency is preserved - // without a server-side send. + // No snapshot-only response for empty directories: many consumers (Java + // FilerClient, S3 listing) treat any response as an entry. The trailer + // carries it instead; older clients ignore trailers. + stream.SetTrailer(metadata.Pairs(filer_pb.ListSnapshotTsNsTrailerKey, strconv.FormatInt(snapshotTsNs, 10))) return nil } @@ -188,6 +202,11 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr pathLock := fs.entryLockTable.AcquireLock("CreateEntry", fullpath, util.ExclusiveLock) defer fs.entryLockTable.ReleaseLock(fullpath, pathLock) + // Fence stamped under the lock: a no-op create returns no event, but the + // acknowledged state still reflects everything at or below this. + resp.LogTsNs = time.Now().UnixNano() + resp.LogSignature = fs.filer.Signature + // Evaluate the optional precondition against the current entry while the // path lock is held, so the check and the write are atomic on this filer. // The fetched entry is then handed to CreateEntry below so it does not look @@ -603,6 +622,10 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr pathLock := fs.entryLockTable.AcquireLock("UpdateEntry", lockPath, util.ExclusiveLock) defer fs.entryLockTable.ReleaseLock(lockPath, pathLock) + // Fence stamped under the lock: the no-change path below returns no event, + // but the acknowledged state still reflects everything at or below this. + logTsNs := time.Now().UnixNano() + entry, err := fs.filer.FindEntry(ctx, lockPath) if err != nil { return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("not found %s: %v", fullpath, err) @@ -629,11 +652,11 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr } if filer.EqualEntry(entry, newEntry) { - return &filer_pb.UpdateEntryResponse{}, err + return &filer_pb.UpdateEntryResponse{LogTsNs: logTsNs, LogSignature: fs.filer.Signature}, err } ctx, eventSink := filer.WithMetadataEventSink(ctx) - resp := &filer_pb.UpdateEntryResponse{} + resp := &filer_pb.UpdateEntryResponse{LogTsNs: logTsNs, LogSignature: fs.filer.Signature} if err = fs.filer.UpdateEntry(ctx, entry, newEntry); err == nil { fs.filer.DeleteChunksNotRecursive(garbage) diff --git a/weed/server/filer_grpc_server_remote.go b/weed/server/filer_grpc_server_remote.go index e7fd0b79d..930d5c455 100644 --- a/weed/server/filer_grpc_server_remote.go +++ b/weed/server/filer_grpc_server_remote.go @@ -64,8 +64,9 @@ func (fs *FilerServer) CacheRemoteObjectToLocalCluster(ctx context.Context, req // doCacheRemoteObjectToLocalCluster performs the actual caching operation. // This is called from singleflight, so only one instance runs per object. func (fs *FilerServer) doCacheRemoteObjectToLocalCluster(ctx context.Context, req *filer_pb.CacheRemoteObjectToLocalClusterRequest) (*filer_pb.CacheRemoteObjectToLocalClusterResponse, error) { - // find the entry first to check if already cached - entry, err := fs.filer.FindEntry(ctx, util.JoinPath(req.Directory, req.Name)) + lockPath := util.JoinPath(req.Directory, req.Name) + entry, logTsNs, err := fs.fencedFindEntry(ctx, lockPath) + if err == filer_pb.ErrNotFound { return nil, err } @@ -73,7 +74,7 @@ func (fs *FilerServer) doCacheRemoteObjectToLocalCluster(ctx context.Context, re return nil, fmt.Errorf("find entry %s/%s: %v", req.Directory, req.Name, err) } - resp := &filer_pb.CacheRemoteObjectToLocalClusterResponse{} + resp := &filer_pb.CacheRemoteObjectToLocalClusterResponse{LogTsNs: logTsNs, LogSignature: fs.filer.Signature} // Early return if not a remote-only object or already cached if entry.Remote == nil || entry.Remote.RemoteSize == 0 { @@ -297,6 +298,34 @@ func (fs *FilerServer) doCacheRemoteObjectToLocalCluster(ctx context.Context, re return nil, err } + // Commit under the mutation path lock so a fenced lookup cannot land + // between the store update and its notification, handing out + // under-versioned state. Re-read under it: the entry may have changed + // during the unlocked download, and the stale base would clobber it. + commitLock := fs.entryLockTable.AcquireLock("CacheRemoteObjectToLocalCluster", lockPath, util.ExclusiveLock) + defer fs.entryLockTable.ReleaseLock(lockPath, commitLock) + + commitLogTsNs := time.Now().UnixNano() + current, err := fs.filer.FindEntry(ctx, lockPath) + if err != nil { + fs.filer.DeleteUncommittedChunks(ctx, chunks) + if err == filer_pb.ErrNotFound { + // Deleted while the download ran; keep the sentinel so callers + // still surface a 404 rather than a generic failure. + return nil, err + } + return nil, fmt.Errorf("find entry %s before commit: %v", lockPath, err) + } + if !filer.EqualEntry(current, entry) { + // Changed during the download: that writer supersedes the cached + // content. Return the current state, fenced at this read. + fs.filer.DeleteUncommittedChunks(ctx, chunks) + resp.Entry = current.ToProtoEntry() + resp.LogTsNs = commitLogTsNs + resp.LogSignature = fs.filer.Signature + return resp, nil + } + garbage := entry.GetChunks() newEntry := entry.ShallowClone() @@ -317,6 +346,8 @@ func (fs *FilerServer) doCacheRemoteObjectToLocalCluster(ctx context.Context, re resp.Entry = newEntry.ToProtoEntry() resp.MetadataEvent = eventSink.Last() + resp.LogTsNs = commitLogTsNs + resp.LogSignature = fs.filer.Signature return resp, nil diff --git a/weed/server/filer_grpc_server_rename.go b/weed/server/filer_grpc_server_rename.go index 3b60e9e75..d44bad283 100644 --- a/weed/server/filer_grpc_server_rename.go +++ b/weed/server/filer_grpc_server_rename.go @@ -12,6 +12,26 @@ import ( "github.com/seaweedfs/seaweedfs/weed/util" ) +// acquireRenamePathLocks holds both paths across commit and notification so a +// fenced lookup cannot read renamed state under a fence preceding its events. +// Path order avoids deadlock with a reverse rename; descendants are not +// locked. The returned func releases both. +func (fs *FilerServer) acquireRenamePathLocks(intention string, oldPath, newPath util.FullPath) func() { + firstPath, secondPath := oldPath, newPath + if secondPath < firstPath { + firstPath, secondPath = secondPath, firstPath + } + firstLock := fs.entryLockTable.AcquireLock(intention, firstPath, util.ExclusiveLock) + if secondPath == firstPath { + return func() { fs.entryLockTable.ReleaseLock(firstPath, firstLock) } + } + secondLock := fs.entryLockTable.AcquireLock(intention, secondPath, util.ExclusiveLock) + return func() { + fs.entryLockTable.ReleaseLock(secondPath, secondLock) + fs.entryLockTable.ReleaseLock(firstPath, firstLock) + } +} + func (fs *FilerServer) AtomicRenameEntry(ctx context.Context, req *filer_pb.AtomicRenameEntryRequest) (*filer_pb.AtomicRenameEntryResponse, error) { glog.V(1).Infof("AtomicRenameEntry %v", req) @@ -23,6 +43,8 @@ func (fs *FilerServer) AtomicRenameEntry(ctx context.Context, req *filer_pb.Atom return nil, err } + defer fs.acquireRenamePathLocks("AtomicRenameEntry", oldParent.Child(req.OldName), newParent.Child(req.NewName))() + ctx, err := fs.filer.BeginTransaction(ctx) if err != nil { return nil, err @@ -71,6 +93,8 @@ func (fs *FilerServer) StreamRenameEntry(req *filer_pb.StreamRenameEntryRequest, return err } + defer fs.acquireRenamePathLocks("StreamRenameEntry", oldParent.Child(req.OldName), newParent.Child(req.NewName))() + ctx := context.Background() ctx, err = fs.filer.BeginTransaction(ctx) diff --git a/weed/server/filer_grpc_server_rename_test.go b/weed/server/filer_grpc_server_rename_test.go index 3fe9392a1..5c6e058ae 100644 --- a/weed/server/filer_grpc_server_rename_test.go +++ b/weed/server/filer_grpc_server_rename_test.go @@ -284,7 +284,7 @@ func TestAtomicRenameEntryEmitsLogicalRenameEvent(t *testing.T) { queue := &captureQueue{} swapNotificationQueue(t, queue) - server := &FilerServer{filer: newRenameTestFiler(store)} + server := &FilerServer{filer: newRenameTestFiler(store), entryLockTable: util.NewLockTable[util.FullPath]()} _, err := server.AtomicRenameEntry(context.Background(), &filer_pb.AtomicRenameEntryRequest{ OldDirectory: "/", OldName: "src.txt", @@ -334,7 +334,7 @@ func TestAtomicRenameEntryOverwriteEmitsDeleteThenRename(t *testing.T) { queue := &captureQueue{} swapNotificationQueue(t, queue) - server := &FilerServer{filer: newRenameTestFiler(store)} + server := &FilerServer{filer: newRenameTestFiler(store), entryLockTable: util.NewLockTable[util.FullPath]()} _, err := server.AtomicRenameEntry(context.Background(), &filer_pb.AtomicRenameEntryRequest{ OldDirectory: "/", OldName: "src.txt", @@ -398,7 +398,7 @@ func TestAtomicRenameEntryDoesNotEmitEventOnDeleteFailure(t *testing.T) { queue := &captureQueue{} swapNotificationQueue(t, queue) - server := &FilerServer{filer: newRenameTestFiler(store)} + server := &FilerServer{filer: newRenameTestFiler(store), entryLockTable: util.NewLockTable[util.FullPath]()} _, err := server.AtomicRenameEntry(context.Background(), &filer_pb.AtomicRenameEntryRequest{ OldDirectory: "/", OldName: "src.txt", @@ -422,7 +422,7 @@ func TestAtomicRenameEntryDoesNotEmitEventOnCommitFailure(t *testing.T) { queue := &captureQueue{} swapNotificationQueue(t, queue) - server := &FilerServer{filer: newRenameTestFiler(store)} + server := &FilerServer{filer: newRenameTestFiler(store), entryLockTable: util.NewLockTable[util.FullPath]()} _, err := server.AtomicRenameEntry(context.Background(), &filer_pb.AtomicRenameEntryRequest{ OldDirectory: "/", OldName: "src.txt", @@ -447,7 +447,7 @@ func TestAtomicRenameEntrySkipsDescendantTargetLookups(t *testing.T) { queue := &captureQueue{} swapNotificationQueue(t, queue) - server := &FilerServer{filer: newRenameTestFiler(store)} + server := &FilerServer{filer: newRenameTestFiler(store), entryLockTable: util.NewLockTable[util.FullPath]()} _, err := server.AtomicRenameEntry(context.Background(), &filer_pb.AtomicRenameEntryRequest{ OldDirectory: "/", OldName: "srcdir", diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 3000ac8f9..52b52859d 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -371,7 +371,7 @@ func (fs *WebDavFileSystem) stat(ctx context.Context, fullFilePath string) (os.F fullpath := util.FullPath(fullFilePath) var fi FileInfo - entry, err := filer_pb.GetEntry(context.Background(), fs, fullpath) + entry, _, _, err := filer_pb.GetEntry(context.Background(), fs, fullpath) if err != nil { if err == filer_pb.ErrNotFound { return nil, os.ErrNotExist @@ -435,7 +435,7 @@ func (f *WebDavFile) Write(buf []byte) (int, error) { var getErr error ctx := context.Background() if f.entry == nil { - f.entry, getErr = filer_pb.GetEntry(context.Background(), f.fs, fullPath) + f.entry, _, _, getErr = filer_pb.GetEntry(context.Background(), f.fs, fullPath) } if f.entry == nil { @@ -526,7 +526,7 @@ func (f *WebDavFile) Read(p []byte) (readSize int, err error) { glog.V(2).Infof("WebDavFileSystem.Read %v", f.name) if f.entry == nil { - f.entry, err = filer_pb.GetEntry(context.Background(), f.fs, util.FullPath(f.name)) + f.entry, _, _, err = filer_pb.GetEntry(context.Background(), f.fs, util.FullPath(f.name)) } if f.entry == nil { return 0, err diff --git a/weed/shell/command_fs_meta_save.go b/weed/shell/command_fs_meta_save.go index 24ed1e850..da88bf34e 100644 --- a/weed/shell/command_fs_meta_save.go +++ b/weed/shell/command_fs_meta_save.go @@ -176,7 +176,7 @@ func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, var hasErr atomic.Bool // also save the directory itself (path) if it exists in the filer - if e, getErr := filer_pb.GetEntry(ctx, filerClient, util.FullPath(path)); getErr != nil { + if e, _, _, getErr := filer_pb.GetEntry(ctx, filerClient, util.FullPath(path)); getErr != nil { // Entry not found is expected and can be ignored; log other errors. if !errors.Is(getErr, filer_pb.ErrNotFound) { fmt.Fprintf(writer, "failed to get entry %s: %v\n", path, getErr)