fix(mount): propagate hard-link nlink changes to sibling cache entries (#9062)

* fix(mount): propagate hard-link nlink changes to sibling cache entries

weed mount serves stat from its local metacache, and the kernel also
caches inode attrs from FUSE replies. When a hard link was unlinked or
a new link added, the filer updated the shared HardLink blob correctly,
but the sibling link entries in the mount's metacache still carried the
stale HardLinkCounter and the kernel attr cache on the shared inode was
not invalidated. Subsequent lstat on any sibling link returned the old
nlink — pjdfstest link/00.t caught this after `unlink n0` and on
`link n1 n2` stating n0.

Walk every path bound to the hard-linked inode via a new
InodeToPath.GetAllPaths, rewrite each cached sibling's HardLinkCounter
and ctime to the authoritative new value, and call
fuseServer.InodeNotify to invalidate the kernel attr cache for the
shared inode. Applied from both Link (bump) and Unlink (decrement).

Unblocks tests/link/00.t and tests/unlink/00.t in pjdfstest; full suite
(235 files, 8803 tests) passes end-to-end with no regressions.

* fix(mount): harden hard-link sibling sync against nil Attributes and id mismatch

Review follow-ups:
- Unlink: guard entry.Attributes for nil before reading Inode, with a
  fallback to inodeToPath.GetInode resolved before RemovePath. Fold the
  duplicated RemovePath into a single call.
- syncHardLinkSiblings: skip siblings whose HardLinkId does not match
  the authoritative entry. The shared-inode invariant normally
  guarantees a match, but a transient mismatch (e.g. a rename replaced
  one of the paths) would otherwise stamp an unrelated entry with the
  wrong counter.

Full pjdfstest suite still passes (235 files, 8803 tests).

test/pjdfstest/known_failures.txt

@@ -11,12 +11,3 @@
 # causes cascading test failures within the test file.
 tests/rename/21.t
 
-# ── Hard link nlink count inconsistencies ────────────────────────────
-# link/00.t and unlink/00.t fail nlink assertions (e.g. expected nlink=2,
-# got nlink=3) after hard link creation/removal. This is a filer-side hard
-# link counter issue, not a mount mtime/ctime problem. The failures are
-# deterministic and surfaced by caching changes that affect the order in
-# which entries are loaded into the local meta cache.
-tests/link/00.t
-tests/unlink/00.t
-

weed/mount/inode_to_path.go

@@ -166,6 +166,21 @@ func (i *InodeToPath) GetPath(inode uint64) (util.FullPath, fuse.Status) {
 	return path.paths[0], fuse.OK
 }
 
+// GetAllPaths returns a copy of all paths associated with an inode. For a
+// hard-linked file, this includes every link that the mount currently knows
+// about. Returns nil if the inode is unknown.
+func (i *InodeToPath) GetAllPaths(inode uint64) []util.FullPath {
+	i.RLock()
+	defer i.RUnlock()
+	ie, found := i.inode2path[inode]
+	if !found || len(ie.paths) == 0 {
+		return nil
+	}
+	out := make([]util.FullPath, len(ie.paths))
+	copy(out, ie.paths)
+	return out
+}
+
 func (i *InodeToPath) HasPath(path util.FullPath) bool {
 	i.RLock()
 	defer i.RUnlock()

weed/mount/weedfs_file_mkrm.go

@@ -248,8 +248,36 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
 	wfs.inodeToPath.TouchDirectory(dirFullPath)
 	wfs.touchDirMtimeCtimeBest(dirFullPath)
 
+	// For hard-linked files, the filer's DeleteHardLink decremented the
+	// shared blob, but the sibling link entries in the local metacache
+	// still carry the pre-unlink HardLinkCounter, and the kernel has
+	// cached attrs on the shared inode. Resolve the shared inode before
+	// removing the path from inodeToPath, then propagate the decrement
+	// to every sibling cache entry and invalidate the kernel attr cache
+	// so subsequent lstats on other links see the new nlink — pjdfstest
+	// link/00.t asserts this after `unlink n0` leaves n1/n2 behind.
+	isHardLink := entry != nil && entry.Attributes != nil && len(entry.HardLinkId) > 0 && entry.HardLinkCounter > 1
+	var sharedInode uint64
+	if isHardLink {
+		sharedInode = entry.Attributes.Inode
+		if sharedInode == 0 {
+			if resolved, found := wfs.inodeToPath.GetInode(entryFullPath); found {
+				sharedInode = resolved
+			}
+		}
+	}
+
 	wfs.inodeToPath.RemovePath(entryFullPath)
 
+	if isHardLink && sharedInode != 0 {
+		decremented := proto.Clone(entry).(*filer_pb.Entry)
+		decremented.HardLinkCounter = entry.HardLinkCounter - 1
+		now := time.Now()
+		decremented.Attributes.Ctime = now.Unix()
+		decremented.Attributes.CtimeNs = int32(now.Nanosecond())
+		wfs.syncHardLinkSiblings(sharedInode, decremented, entryFullPath)
+	}
+
 	return fuse.OK
 
 }

weed/mount/weedfs_link.go

@@ -1,6 +1,7 @@
 package mount
 
 import (
+	"bytes"
 	"context"
 	"syscall"
 	"time"
@@ -147,7 +148,63 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
 
 	wfs.inodeToPath.AddPath(oldEntry.Attributes.Inode, newEntryPath)
 
+	// Propagate the new HardLinkCounter to sibling cache entries and
+	// invalidate the kernel's inode attr cache. Without this, `stat` on any
+	// existing sibling link (other than the source we just wrote) returns
+	// the old nlink from the local metacache — pjdfstest link/00.t catches
+	// this after `link n1 n2` when it stats n0.
+	wfs.syncHardLinkSiblings(oldEntry.Attributes.Inode, oldEntry, oldEntryPath, newEntryPath)
+
 	wfs.outputPbEntry(out, oldEntry.Attributes.Inode, request.Entry)
 
 	return fuse.OK
 }
+
+// syncHardLinkSiblings rewrites the cached HardLinkCounter (and ctime) on
+// every sibling link of the given inode, and invalidates the kernel's inode
+// attr cache. `authoritativeEntry` carries the counter/ctime that should be
+// propagated. `skipPaths` are the link paths already updated by the caller
+// (typically the source and/or the newly created/removed link).
+func (wfs *WFS) syncHardLinkSiblings(inode uint64, authoritativeEntry *filer_pb.Entry, skipPaths ...util.FullPath) {
+	if authoritativeEntry == nil || len(authoritativeEntry.HardLinkId) == 0 {
+		return
+	}
+	paths := wfs.inodeToPath.GetAllPaths(inode)
+	if len(paths) == 0 {
+		return
+	}
+	skip := make(map[util.FullPath]struct{}, len(skipPaths))
+	for _, p := range skipPaths {
+		skip[p] = struct{}{}
+	}
+	ctx := context.Background()
+	for _, p := range paths {
+		if _, skipped := skip[p]; skipped {
+			continue
+		}
+		sibling, err := wfs.metaCache.FindEntry(ctx, p)
+		if err != nil || sibling == nil {
+			continue
+		}
+		// Only touch siblings that genuinely share the same hard-link id.
+		// inodeToPath's shared-inode invariant should already guarantee
+		// this, but a mismatch can occur transiently (e.g. a rename
+		// replaced one of the paths), and blindly stamping an unrelated
+		// entry's counter would corrupt it.
+		if !bytes.Equal(sibling.HardLinkId, authoritativeEntry.HardLinkId) {
+			continue
+		}
+		sibling.HardLinkCounter = authoritativeEntry.HardLinkCounter
+		if authoritativeEntry.Attributes != nil {
+			sibling.Attr.Ctime = time.Unix(authoritativeEntry.Attributes.Ctime, int64(authoritativeEntry.Attributes.CtimeNs))
+		}
+		if err := wfs.metaCache.UpdateEntry(ctx, sibling); err != nil {
+			glog.V(4).Infof("syncHardLinkSiblings update %s: %v", p, err)
+		}
+	}
+	if wfs.fuseServer != nil {
+		if status := wfs.fuseServer.InodeNotify(inode, 0, -1); status != fuse.OK {
+			glog.V(4).Infof("syncHardLinkSiblings invalidate inode %d: %v", inode, status)
+		}
+	}
+}