From 205cfbdf4ae19a5cf50fc4c790900d3b26da9f44 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Mon, 15 Jun 2026 11:37:48 -0700 Subject: [PATCH] Don't shut down the server when fencing a rid twice A node only needs to be fenced once, but scoutfs_fence_start() can be called for the same rid more than once. When a new leader starts it fences the previous leader as it removes it from the quorum (quorum_block_leader), and that same rid can also be a mounted client that then fails to recover within the timeout (client_recovery). The second fence call collides on that name, sysfs returns -EEXIST, and the error is propagated to fence_pending_recov_worker() which treats any error as fatal and shuts the server down. On the next mount a new leader hits the same stale set and the same collision, so the filesystem can never finish recovery. Jun 15 09:22:35 kernel: scoutfs f.000000.r.222222: fencing previous leader f.000000.r.111111 at term 183942 in slot 3 with address x.x.x.x:6000 Jun 15 09:22:36 scoutfs-fenced[9194]: [2026-06-15 09:22:36.588037842] server f.000000.r.222222 fencing rid 1111111111111111 at IP x.x.x.x for quorum_block_leader Jun 15 09:23:09 kernel: scoutfs f.000000.r.222222 error: 30000 ms recovery timeout expired for client rid 1111111111111111, fencing Jun 15 09:23:09 kernel: sysfs: cannot create duplicate filename '/fs/scoutfs/f.000000.r.222222/fence/1111111111111111' Jun 15 09:23:09 kernel: scoutfs f.000000.r.222222 error: fence returned err -17, shutting down server Check the list for the rid and skip the duplicate before creating sysfs. A pending fence can be freed once it is on the list, so a new fi->mutex serializes creation against the freeing path: the duplicate check, sysfs create, and list insert run as a unit, and a fence becomes visible on the list only once it is fully built. scoutfs_fence_free() and scoutfs_fence_stop() take the same mutex around removing a fence and tearing it down, and scoutfs_fence_destroy() drains through fence_stop() rather than walking the list unlocked. Signed-off-by: Auke Kok --- kmod/src/fence.c | 46 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/kmod/src/fence.c b/kmod/src/fence.c index 60799917..a3579228 100644 --- a/kmod/src/fence.c +++ b/kmod/src/fence.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "super.h" @@ -65,6 +66,7 @@ struct fence_info { struct kobject fence_dir_kobj; struct workqueue_struct *wq; wait_queue_head_t waitq; + struct mutex mutex; spinlock_t lock; struct list_head list; }; @@ -235,8 +237,10 @@ static void fence_timeout(struct timer_list *timer) int scoutfs_fence_start(struct super_block *sb, u64 rid, __be32 ipv4_addr, int reason) { DECLARE_FENCE_INFO(sb, fi); + struct pending_fence *existing; struct pending_fence *fence; - int ret; + bool duplicate = false; + int ret = 0; fence = kzalloc(sizeof(struct pending_fence), GFP_NOFS); if (!fence) { @@ -246,6 +250,7 @@ int scoutfs_fence_start(struct super_block *sb, u64 rid, __be32 ipv4_addr, int r fence->sb = sb; scoutfs_sysfs_init_attrs(sb, &fence->ssa); + timer_setup(&fence->timer, fence_timeout, 0); fence->start_kt = ktime_get(); fence->ipv4_addr = ipv4_addr; @@ -254,22 +259,39 @@ int scoutfs_fence_start(struct super_block *sb, u64 rid, __be32 ipv4_addr, int r fence->reason = reason; fence->rid = rid; + mutex_lock(&fi->mutex); + + spin_lock(&fi->lock); + list_for_each_entry(existing, &fi->list, entry) { + if (existing->rid == rid) { + duplicate = true; + break; + } + } + spin_unlock(&fi->lock); + + if (duplicate) + goto unlock; + ret = scoutfs_sysfs_create_attrs_parent(sb, &fi->kset->kobj, &fence->ssa, fence_attrs, "%016llx", rid); - if (ret < 0) { - kfree(fence); - goto out; - } + if (ret < 0) + goto unlock; - timer_setup(&fence->timer, fence_timeout, 0); fence->timer.expires = jiffies + msecs_to_jiffies(FENCE_TIMEOUT_MS); add_timer(&fence->timer); spin_lock(&fi->lock); list_add_tail(&fence->entry, &fi->list); spin_unlock(&fi->lock); + + fence = NULL; +unlock: + mutex_unlock(&fi->mutex); out: + if (fence) + destroy_fence(fence); return ret; } @@ -324,6 +346,8 @@ int scoutfs_fence_free(struct super_block *sb, u64 rid) struct pending_fence *fence; int ret = -ENOENT; + mutex_lock(&fi->mutex); + spin_lock(&fi->lock); list_for_each_entry(fence, &fi->list, entry) { if (fence->rid == rid) { @@ -339,6 +363,8 @@ int scoutfs_fence_free(struct super_block *sb, u64 rid) wake_up(&fi->waitq); } + mutex_unlock(&fi->mutex); + return ret; } @@ -413,6 +439,7 @@ int scoutfs_fence_setup(struct super_block *sb) } init_waitqueue_head(&fi->waitq); + mutex_init(&fi->mutex); spin_lock_init(&fi->lock); INIT_LIST_HEAD(&fi->list); @@ -446,6 +473,7 @@ void scoutfs_fence_stop(struct super_block *sb) DECLARE_FENCE_INFO(sb, fi); struct pending_fence *fence; + mutex_lock(&fi->mutex); do { spin_lock(&fi->lock); fence = list_first_entry_or_null(&fi->list, struct pending_fence, entry); @@ -458,20 +486,18 @@ void scoutfs_fence_stop(struct super_block *sb) wake_up(&fi->waitq); } } while (fence); + mutex_unlock(&fi->mutex); } void scoutfs_fence_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct fence_info *fi = SCOUTFS_SB(sb)->fence_info; - struct pending_fence *fence; - struct pending_fence *tmp; if (fi) { if (fi->wq) destroy_workqueue(fi->wq); - list_for_each_entry_safe(fence, tmp, &fi->list, entry) - destroy_fence(fence); + scoutfs_fence_stop(sb); if (fi->kset) kset_unregister(fi->kset); kfree(fi);