From b21c2633281636ea98635bd9a97b4b7db6555fb7 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 25 May 2026 16:20:23 -0700 Subject: [PATCH] test/fuse_dlm: cross-mount POSIX locks + survival across a ring change (#9677) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two FUSE integration tests on the existing dlm cluster harness (the -dlm mounts route advisory locks to the owner filer): - TestPosixLockCrossMount: an flock taken on one mount blocks the other, and is grantable after release — the routed-to-owner path end to end. - TestPosixLockSurvivesFilerLoss: hold flocks on many files, stop filer1 so keys it owned migrate to filer0; after the ring settles and the holding mount re-asserts, every lock is still honored. Asserts only the settled state; the transient migration window is unit-covered. Locks are taken on read-only fds so the -dlm whole-file write lock (a different mechanism, held until close) isn't involved. Skipped on non-Linux: only Linux forwards advisory locks (SETLK) to the FUSE server; macFUSE handles flock in-kernel per mount. --- test/fuse_dlm/posix_lock_ring_test.go | 232 ++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 test/fuse_dlm/posix_lock_ring_test.go diff --git a/test/fuse_dlm/posix_lock_ring_test.go b/test/fuse_dlm/posix_lock_ring_test.go new file mode 100644 index 000000000..62210c3fb --- /dev/null +++ b/test/fuse_dlm/posix_lock_ring_test.go @@ -0,0 +1,232 @@ +package fuse_dlm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "syscall" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/stretchr/testify/require" +) + +// posixLockKey returns the routed-lock key a mount uses for a file at the mount +// root. Mounts run with -filer.path=/, so the file's filer path is "/"+name; the +// prefix must match mount.posixLockKeyForInode ("s3.fuse.lock:"). +func posixLockKey(name string) string { return "s3.fuse.lock:/" + name } + +// These tests exercise cross-mount POSIX advisory locks (flock), which the +// mounts route to the inode's owner filer because they run with -dlm +// (crossMountLocks() == lockClient != nil). They reuse the dlmTestCluster: +// master + volume + 2 filers (forming the lock ring) + 2 mounts on filer0. +// +// All flock opens are O_RDONLY on purpose: a write open (O_RDWR/O_WRONLY) would +// also take the -dlm whole-file write lock (held until close), which is a +// different mechanism — opening read-only isolates the POSIX advisory flock. + +// requireForwardedLocks skips on platforms where the kernel does not forward +// advisory locks to the FUSE server. Only Linux forwards flock/fcntl (SETLK) to +// the filesystem; macFUSE handles flock in-kernel per mount, so cross-mount +// coordination can't be observed there even though the routed path is correct. +func requireForwardedLocks(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skipf("advisory locks are only forwarded to the FUSE server on Linux (GOOS=%s)", runtime.GOOS) + } +} + +// openFlock opens path read-only and takes a flock of type how (e.g. LOCK_EX). +// The file must already exist. The returned file holds the lock until closed. +func openFlock(path string, how int) (*os.File, error) { + f, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return nil, err + } + if err := syscall.Flock(int(f.Fd()), how); err != nil { + f.Close() + return nil, err + } + return f, nil +} + +// createFile creates an empty file; the brief -dlm write lock it takes is +// released on close, before any flock test runs. +func createFile(t *testing.T, path string) { + t.Helper() + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + require.NoError(t, err, "create %s", path) + require.NoError(t, f.Close()) +} + +// waitVisible waits for path to appear (cross-mount metadata propagation). +func waitVisible(t *testing.T, path string) { + t.Helper() + require.Eventually(t, func() bool { + _, err := os.Stat(path) + return err == nil + }, 15*time.Second, 200*time.Millisecond, "%s never became visible", path) +} + +// tryExclusiveFlock attempts a non-blocking exclusive flock on path and +// classifies the outcome: acquired (and released again), blocked by another +// owner (EWOULDBLOCK/EAGAIN), or an unexpected error. It distinguishes a real +// "held by another" from incidental errors so the latter can't masquerade as a +// satisfied "must be blocked" assertion. +func tryExclusiveFlock(path string) (acquired, blocked bool, err error) { + f, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return false, false, err + } + defer f.Close() + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + return false, true, nil + } + return false, false, err + } + syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + return true, false, nil +} + +// requireEventuallyBlocked asserts path becomes held by another owner. It polls +// because a routed lock RPC can hit a transient EIO (a cold or forwarded gRPC +// call under load) even while the lock is genuinely held; it still fails if the +// lock is never blocked — whether it can be acquired (a double-grant) or errors +// persistently — so an incidental error can't masquerade as "held". +func requireEventuallyBlocked(t *testing.T, path string, timeout time.Duration, msg string) { + t.Helper() + require.Eventuallyf(t, func() bool { return isBlocked(path) }, + timeout, 500*time.Millisecond, "%s", msg) +} + +// isBlocked / isAcquirable are lenient predicates for polling, where transient +// errors during a migration are expected and simply mean "not yet". +func isBlocked(path string) bool { _, b, _ := tryExclusiveFlock(path); return b } +func isAcquirable(path string) bool { + a, _, _ := tryExclusiveFlock(path) + return a +} + +// stopFiler stops filer idx and clears its command so cluster teardown does not +// try to stop it again. +func (c *dlmTestCluster) stopFiler(idx int) { + stopCmd(c.filerCmds[idx]) + c.filerCmds[idx] = nil +} + +// TestPosixLockCrossMount verifies a flock taken on one mount is seen by the +// other mount of the same cluster (the routed-to-owner-filer path end to end). +func TestPosixLockCrossMount(t *testing.T) { + requireForwardedLocks(t) + c := startDLMTestCluster(t) + + name := "posix-xmount.lock" + path0 := filepath.Join(c.mountPoints[0], name) + path1 := filepath.Join(c.mountPoints[1], name) + + createFile(t, path0) + waitVisible(t, path1) + + held, err := openFlock(path0, syscall.LOCK_EX) + require.NoError(t, err, "mount0 should acquire the exclusive flock") + defer held.Close() + + requireEventuallyBlocked(t, path1, 15*time.Second, "mount1 must be blocked while mount0 holds the flock") + + require.NoError(t, syscall.Flock(int(held.Fd()), syscall.LOCK_UN)) + require.Eventually(t, func() bool { return isAcquirable(path1) }, + 15*time.Second, 500*time.Millisecond, + "mount1 should acquire the flock after mount0 releases it") +} + +// TestPosixLockSurvivesFilerLoss verifies advisory locks held across mounts +// survive a filer leaving the ring: ownership of the affected keys migrates to +// the surviving filer and the holding mount re-asserts them there, so the locks +// stay honored. It locks many files so that — with 2 filers — several are owned +// by filer1 and thus actually migrate when filer1 stops. +func TestPosixLockSurvivesFilerLoss(t *testing.T) { + requireForwardedLocks(t) + c := startDLMTestCluster(t) + + const n = 12 + + // Select files via the same ring the filers run so the locked set provably + // spans both filers: filer1-owned keys must migrate when filer1 stops, while + // filer0-owned keys must keep working. Choosing by ownership (instead of + // hoping a sequential set happens to spread) keeps the migration path + // exercised on every run, independent of the cluster's dynamic ports. + ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount) + ring.SetServers([]pb.ServerAddress{ + pb.ServerAddress(c.filerAddress(0)), + pb.ServerAddress(c.filerAddress(1)), + }) + filer1 := pb.ServerAddress(c.filerAddress(1)) + var onFiler1, onFiler0 []string + for i := 0; (len(onFiler1) < n/2 || len(onFiler0) < n/2) && i < 1000; i++ { + name := fmt.Sprintf("posix-migrate-%d.lock", i) + if ring.GetPrimary(posixLockKey(name)) == filer1 { + onFiler1 = append(onFiler1, name) + } else { + onFiler0 = append(onFiler0, name) + } + } + require.GreaterOrEqualf(t, len(onFiler1), n/2, "need %d filer1-owned keys to exercise migration", n/2) + require.GreaterOrEqualf(t, len(onFiler0), n/2, "need %d filer0-owned keys", n/2) + names := append(onFiler1[:n/2:n/2], onFiler0[:n/2]...) + + held := make([]*os.File, n) + for i, name := range names { + createFile(t, filepath.Join(c.mountPoints[0], name)) + waitVisible(t, filepath.Join(c.mountPoints[1], name)) + f, err := openFlock(filepath.Join(c.mountPoints[0], name), syscall.LOCK_EX) + require.NoError(t, err, "mount0 should acquire flock %s", name) + held[i] = f + defer held[i].Close() + } + + require.Eventually(t, func() bool { + for _, name := range names { + if !isBlocked(filepath.Join(c.mountPoints[1], name)) { + return false + } + } + return true + }, 20*time.Second, time.Second, + "every lock must be held on mount1 before the ring change") + + // Drop filer1 from the ring; keys it owned migrate to filer0. + c.stopFiler(1) + require.NoError(t, c.waitForFilerCount(1, 30*time.Second), "ring should drop to one filer") + + // Poll until every lock is honored again on the surviving filer: the ring + // must propagate and the holding mount must re-assert its locks (keepalive is + // 5s). We assert only the settled state — the transient migration window is + // covered by unit tests. + require.Eventually(t, func() bool { + for _, name := range names { + if !isBlocked(filepath.Join(c.mountPoints[1], name)) { + return false + } + } + return true + }, 45*time.Second, time.Second, + "all locks must survive filer1 leaving the ring (migrate to filer0)") + + // Releasing on mount0 frees them all: mount1 can then acquire each. + for _, f := range held { + require.NoError(t, syscall.Flock(int(f.Fd()), syscall.LOCK_UN)) + } + require.Eventually(t, func() bool { + for _, name := range names { + if !isAcquirable(filepath.Join(c.mountPoints[1], name)) { + return false + } + } + return true + }, 20*time.Second, 500*time.Millisecond, + "mount1 should acquire every lock after mount0 releases them post-migration") +}