Merge pull request #42 from versity/zab/fencing_and_reclaiming

Zab/fencing and reclaiming

kmod/src/Makefile

@@ -18,6 +18,7 @@ scoutfs-y +=			\
 	dir.o			\
 	export.o		\
 	ext.o			\
+	fence.o			\
 	file.o			\
 	forest.o		\
 	inode.o			\

kmod/src/block.c

@@ -200,7 +200,9 @@ static void block_free(struct super_block *sb, struct block_private *bp)
 	else
 		BUG();
 
-	WARN_ON_ONCE(!list_empty(&bp->dirty_entry));
+	/* ok to tear down dirty blocks when forcing unmount */
+	WARN_ON_ONCE(!scoutfs_forcing_unmount(sb) && !list_empty(&bp->dirty_entry));
+
 	WARN_ON_ONCE(atomic_read(&bp->refcount));
 	WARN_ON_ONCE(atomic_read(&bp->io_count));
 	kfree(bp);
@@ -485,6 +487,9 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp,
 	sector_t sector;
 	int ret = 0;
 
+	if (scoutfs_forcing_unmount(sb))
+		return -EIO;
+
 	sector = bp->bl.blkno << (SCOUTFS_BLOCK_LG_SHIFT - 9);
 
 	WARN_ON_ONCE(bp->bl.blkno == U64_MAX);
@@ -1148,7 +1153,7 @@ static void sm_block_bio_end_io(struct bio *bio, int err)
  * only layer that sees the full block buffer so we pass the calculated
  * crc to the caller for them to check in their context.
  */
-static int sm_block_io(struct block_device *bdev, int rw, u64 blkno,
+static int sm_block_io(struct super_block *sb, struct block_device *bdev, int rw, u64 blkno,
 		       struct scoutfs_block_header *hdr, size_t len,
 		       __le32 *blk_crc)
 {
@@ -1160,6 +1165,9 @@ static int sm_block_io(struct block_device *bdev, int rw, u64 blkno,
 
 	BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_BLOCK_SM_SIZE);
 
+	if (scoutfs_forcing_unmount(sb))
+		return -EIO;
+
 	if (WARN_ON_ONCE(len > SCOUTFS_BLOCK_SM_SIZE) ||
 	    WARN_ON_ONCE(!(rw & WRITE) && !blk_crc))
 		return -EINVAL;
@@ -1212,14 +1220,14 @@ int scoutfs_block_read_sm(struct super_block *sb,
 			  struct scoutfs_block_header *hdr, size_t len,
 			  __le32 *blk_crc)
 {
-	return sm_block_io(bdev, READ, blkno, hdr, len, blk_crc);
+	return sm_block_io(sb, bdev, READ, blkno, hdr, len, blk_crc);
 }
 
 int scoutfs_block_write_sm(struct super_block *sb,
 			   struct block_device *bdev, u64 blkno,
 			   struct scoutfs_block_header *hdr, size_t len)
 {
-	return sm_block_io(bdev, WRITE, blkno, hdr, len, NULL);
+	return sm_block_io(sb, bdev, WRITE, blkno, hdr, len, NULL);
 }
 
 int scoutfs_block_setup(struct super_block *sb)

kmod/src/btree.c

@@ -1101,6 +1101,10 @@ static int btree_walk(struct super_block *sb,
 	if (WARN_ON_ONCE((flags & BTW_DIRTY) && (!alloc || !wri)))
 		return -EINVAL;
 
+	/* all ops come through walk and walk calls all reads */
+	if (scoutfs_forcing_unmount(sb))
+		return -EIO;
+
 	scoutfs_inc_counter(sb, btree_walk);
 
 restart:

kmod/src/client.c

@@ -48,6 +48,7 @@ struct client_info {
 
 	struct workqueue_struct *workq;
 	struct delayed_work connect_dwork;
+	unsigned long connect_delay_jiffies;
 
 	u64 server_term;
 
@@ -349,6 +350,7 @@ static int client_greeting(struct super_block *sb,
 	scoutfs_net_client_greeting(sb, conn, new_server);
 
 	client->server_term = le64_to_cpu(gr->server_term);
+	client->connect_delay_jiffies = 0;
 	ret = 0;
 out:
 	return ret;
@@ -398,6 +400,20 @@ out:
 	return ret;
 }
 
+/*
+ * If we're not seeing successful connections we want to back off.  Each
+ * connection attempt starts by setting a long connection work delay.
+ * We only set a shorter delay if we see a greeting response from the
+ * server.  At that point we'll try to immediately reconnect if the
+ * connection is broken.
+ */
+static void queue_connect_dwork(struct super_block *sb, struct client_info *client)
+{
+	if (!atomic_read(&client->shutting_down) && !scoutfs_forcing_unmount(sb))
+		queue_delayed_work(client->workq, &client->connect_dwork,
+				   client->connect_delay_jiffies);
+}
+
 /*
  * This work is responsible for maintaining a connection from the client
  * to the server.  It's queued on mount and disconnect and we requeue
@@ -437,6 +453,9 @@ static void scoutfs_client_connect_worker(struct work_struct *work)
 		goto out;
 	}
 
+	/* always wait a bit until a greeting response sets a lower delay */
+	client->connect_delay_jiffies = msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS);
+
 	ret = scoutfs_quorum_server_sin(sb, &sin);
 	if (ret < 0)
 		goto out;
@@ -464,11 +483,8 @@ static void scoutfs_client_connect_worker(struct work_struct *work)
 	if (ret)
 		scoutfs_net_shutdown(sb, client->conn);
 out:
-
-	/* always have a small delay before retrying to avoid storms */
-	if (ret && !atomic_read(&client->shutting_down))
-		queue_delayed_work(client->workq, &client->connect_dwork,
-				   msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS));
+	if (ret)
+		queue_connect_dwork(sb, client);
 }
 
 static scoutfs_net_request_t client_req_funcs[] = {
@@ -487,8 +503,7 @@ static void client_notify_down(struct super_block *sb,
 {
 	struct client_info *client = SCOUTFS_SB(sb)->client_info;
 
-	if (!atomic_read(&client->shutting_down))
-		queue_delayed_work(client->workq, &client->connect_dwork, 0);
+	queue_connect_dwork(sb, client);
 }
 
 int scoutfs_client_setup(struct super_block *sb)
@@ -523,7 +538,7 @@ int scoutfs_client_setup(struct super_block *sb)
 		goto out;
 	}
 
-	queue_delayed_work(client->workq, &client->connect_dwork, 0);
+	queue_connect_dwork(sb, client);
 	ret = 0;
 
 out:
@@ -580,7 +595,7 @@ void scoutfs_client_destroy(struct super_block *sb)
 	if (client == NULL)
 		return;
 
-	if (client->server_term != 0) {
+	if (client->server_term != 0 && !scoutfs_forcing_unmount(sb)) {
 		client->sending_farewell = true;
 		ret = scoutfs_net_submit_request(sb, client->conn,
 						 SCOUTFS_NET_CMD_FAREWELL,

kmod/src/fence.c

@@ -0,0 +1,480 @@
+/*
+ * Copyright (C) 2019 Versity Software, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/kobject.h>
+#include <linux/sysfs.h>
+#include <linux/device.h>
+#include <linux/timer.h>
+#include <asm/barrier.h>
+
+#include "super.h"
+#include "msg.h"
+#include "sysfs.h"
+#include "server.h"
+#include "fence.h"
+
+/*
+ * Fencing ensures that a given mount can no longer write to the
+ * metadata or data devices.  It's necessary to ensure that it's safe to
+ * give another mount access to a resource that is currently owned by a
+ * mount that has stopped responding.
+ *
+ * Fencing is performed in collaboration between the currently elected
+ * quorum leader mount and userspace running on its host.  The kernel
+ * creates fencing requests as it notices that mounts have stopped
+ * participating.  The fence requests are published as directories in
+ * sysfs.  Userspace agents watch for directories, take action, and
+ * write to files in the directory to indicate that the mount has been
+ * fenced.  Once the mount is fenced the server can reclaim the
+ * resources previously held by the fenced mount.
+ *
+ * The fence requests contain metadata identifying the specific instance
+ * of the mount that needs to be fenced.  This lets a fencing agent
+ * ensure that a specific mount has been fenced without necessarily
+ * destroying the node that was hosting it.  Maybe the node had rebooted
+ * and the mount is no longer there, maybe the mount can be force
+ * unmounted, maybe the node can be configured to isolate the mount from
+ * the devices.
+ *
+ * The fencing mechanism is asynchronous and can fail but the server
+ * cannot make progress until it completes.  If a fence request times
+ * out the server shuts down in the hope that another instance of a
+ * server might have more luck fencing a non-responsive mount.
+ *
+ * Sources of fencing are fundamentally anchored in shared persistent
+ * state.  It is possible, though unlikely, that servers can fence a
+ * node and then themselves fail, leaving the next server to try and
+ * fence the mount again.
+ */
+
+struct fence_info {
+	struct kset *kset;
+	struct kobject fence_dir_kobj;
+	struct workqueue_struct *wq;
+	wait_queue_head_t waitq;
+	spinlock_t lock;
+	struct list_head list;
+};
+
+#define DECLARE_FENCE_INFO(sb, name) \
+	struct fence_info *name = SCOUTFS_SB(sb)->fence_info
+
+struct pending_fence {
+	struct super_block *sb;
+	struct scoutfs_sysfs_attrs ssa;
+	struct list_head entry;
+	struct timer_list timer;
+
+	ktime_t start_kt;
+	__be32 ipv4_addr;
+	bool fenced;
+	bool error;
+	int reason;
+	u64 rid;
+};
+
+#define FENCE_FROM_KOBJ(kobj)					\
+	container_of(SCOUTFS_SYSFS_ATTRS(kobj), struct pending_fence, ssa)
+#define DECLARE_FENCE_FROM_KOBJ(name, kobj)				\
+	struct pending_fence *name = FENCE_FROM_KOBJ(kobj)
+
+static void destroy_fence(struct pending_fence *fence)
+{
+	struct super_block *sb = fence->sb;
+
+	scoutfs_sysfs_destroy_attrs(sb, &fence->ssa);
+	del_timer_sync(&fence->timer);
+	kfree(fence);
+}
+
+static ssize_t elapsed_secs_show(struct kobject *kobj,
+				 struct kobj_attribute *attr, char *buf)
+{
+	DECLARE_FENCE_FROM_KOBJ(fence, kobj);
+	ktime_t now = ktime_get();
+	struct timeval tv = { 0, };
+
+	if (ktime_after(now, fence->start_kt))
+		tv = ktime_to_timeval(ktime_sub(now, fence->start_kt));
+
+	return snprintf(buf, PAGE_SIZE, "%llu", (long long)tv.tv_sec);
+}
+SCOUTFS_ATTR_RO(elapsed_secs);
+
+static ssize_t fenced_show(struct kobject *kobj, struct kobj_attribute *attr,
+			   char *buf)
+{
+	DECLARE_FENCE_FROM_KOBJ(fence, kobj);
+
+	return snprintf(buf, PAGE_SIZE, "%u", !!fence->fenced);
+}
+
+/*
+ * any write to the fenced file from userspace indicates that the mount
+ * has been safely fenced and can no longer write to the shared device.
+ */
+static ssize_t fenced_store(struct kobject *kobj, struct kobj_attribute *attr,
+			    const char *buf, size_t count)
+{
+	DECLARE_FENCE_FROM_KOBJ(fence, kobj);
+	DECLARE_FENCE_INFO(fence->sb, fi);
+
+	if (!fence->fenced) {
+		del_timer_sync(&fence->timer);
+		fence->fenced = true;
+		wake_up(&fi->waitq);
+	}
+
+	return count;
+}
+SCOUTFS_ATTR_RW(fenced);
+
+static ssize_t error_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	DECLARE_FENCE_FROM_KOBJ(fence, kobj);
+
+	return snprintf(buf, PAGE_SIZE, "%u", !!fence->error);
+}
+
+/*
+ * Fencing can tell us that they were unable to fence the given mount.
+ * We can't continue if the mount can't be isolated so we shut down the
+ * server.
+ */
+static ssize_t error_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf,
+			   size_t count)
+{
+	DECLARE_FENCE_FROM_KOBJ(fence, kobj);
+	struct super_block *sb = fence->sb;
+	DECLARE_FENCE_INFO(fence->sb, fi);
+
+	if (!fence->error) {
+		fence->error = true;
+		scoutfs_err(sb, "error indicated by fence action for rid %016llx", fence->rid);
+		wake_up(&fi->waitq);
+	}
+
+	return count;
+}
+SCOUTFS_ATTR_RW(error);
+
+static ssize_t ipv4_addr_show(struct kobject *kobj,
+			      struct kobj_attribute *attr, char *buf)
+{
+	DECLARE_FENCE_FROM_KOBJ(fence, kobj);
+
+	return snprintf(buf, PAGE_SIZE, "%pI4", &fence->ipv4_addr);
+}
+SCOUTFS_ATTR_RO(ipv4_addr);
+
+static ssize_t reason_show(struct kobject *kobj, struct kobj_attribute *attr,
+			   char *buf)
+{
+	DECLARE_FENCE_FROM_KOBJ(fence, kobj);
+	unsigned r = fence->reason;
+	char *str = "unknown";
+	static char *reasons[] = {
+		[SCOUTFS_FENCE_CLIENT_RECOVERY] = "client_recovery",
+		[SCOUTFS_FENCE_CLIENT_RECONNECT] = "client_reconnect",
+		[SCOUTFS_FENCE_QUORUM_BLOCK_LEADER] = "quorum_block_leader",
+	};
+
+	if (r < ARRAY_SIZE(reasons) && reasons[r])
+		str = reasons[r];
+
+	return snprintf(buf, PAGE_SIZE, "%s", str);
+}
+SCOUTFS_ATTR_RO(reason);
+
+static ssize_t rid_show(struct kobject *kobj, struct kobj_attribute *attr,
+			char *buf)
+{
+	DECLARE_FENCE_FROM_KOBJ(fence, kobj);
+
+	return snprintf(buf, PAGE_SIZE, "%016llx", fence->rid);
+}
+SCOUTFS_ATTR_RO(rid);
+
+static struct attribute *fence_attrs[] = {
+	SCOUTFS_ATTR_PTR(elapsed_secs),
+	SCOUTFS_ATTR_PTR(fenced),
+	SCOUTFS_ATTR_PTR(error),
+	SCOUTFS_ATTR_PTR(ipv4_addr),
+	SCOUTFS_ATTR_PTR(reason),
+	SCOUTFS_ATTR_PTR(rid),
+	NULL,
+};
+
+#define FENCE_TIMEOUT_MS (MSEC_PER_SEC * 30)
+
+static void fence_timeout(struct timer_list *timer)
+{
+	struct pending_fence *fence = from_timer(fence, timer, timer);
+	struct super_block *sb = fence->sb;
+	DECLARE_FENCE_INFO(sb, fi);
+
+	fence->error = true;
+	scoutfs_err(sb, "fence request for rid %016llx was not serviced in %lums, raising error",
+		    fence->rid, FENCE_TIMEOUT_MS);
+	wake_up(&fi->waitq);
+}
+
+int scoutfs_fence_start(struct super_block *sb, u64 rid, __be32 ipv4_addr, int reason)
+{
+	DECLARE_FENCE_INFO(sb, fi);
+	struct pending_fence *fence;
+	int ret;
+
+	fence = kzalloc(sizeof(struct pending_fence), GFP_NOFS);
+	if (!fence) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	fence->sb = sb;
+	scoutfs_sysfs_init_attrs(sb, &fence->ssa);
+
+	fence->start_kt = ktime_get();
+	fence->ipv4_addr = ipv4_addr;
+	fence->fenced = false;
+	fence->error = false;
+	fence->reason = reason;
+	fence->rid = rid;
+
+	ret = scoutfs_sysfs_create_attrs_parent(sb, &fi->kset->kobj,
+						&fence->ssa, fence_attrs,
+						"%016llx", rid);
+	if (ret < 0) {
+		kfree(fence);
+		goto out;
+	}
+
+	timer_setup(&fence->timer, fence_timeout, 0);
+	fence->timer.expires = jiffies + msecs_to_jiffies(FENCE_TIMEOUT_MS);
+	add_timer(&fence->timer);
+
+	spin_lock(&fi->lock);
+	list_add_tail(&fence->entry, &fi->list);
+	spin_unlock(&fi->lock);
+out:
+	return ret;
+}
+
+/*
+ * Give the caller the rid of the next fence request which has been
+ * fenced.  This doesn't have a position from which to return the next
+ * because the caller either frees the fence request it's given or shuts
+ * down.
+ */
+int scoutfs_fence_next(struct super_block *sb, u64 *rid, int *reason, bool *error)
+{
+	DECLARE_FENCE_INFO(sb, fi);
+	struct pending_fence *fence;
+	int ret = -ENOENT;
+
+	spin_lock(&fi->lock);
+	list_for_each_entry(fence, &fi->list, entry) {
+		if (fence->fenced || fence->error) {
+			*rid = fence->rid;
+			*reason = fence->reason;
+			*error = fence->error;
+			ret = 0;
+			break;
+		}
+	}
+	spin_unlock(&fi->lock);
+
+	return ret;
+}
+
+int scoutfs_fence_reason_pending(struct super_block *sb, int reason)
+{
+	DECLARE_FENCE_INFO(sb, fi);
+	struct pending_fence *fence;
+	bool pending = false;
+
+	spin_lock(&fi->lock);
+	list_for_each_entry(fence, &fi->list, entry) {
+		if (fence->reason == reason) {
+			pending = true;
+			break;
+		}
+	}
+	spin_unlock(&fi->lock);
+
+	return pending;
+}
+
+int scoutfs_fence_free(struct super_block *sb, u64 rid)
+{
+	DECLARE_FENCE_INFO(sb, fi);
+	struct pending_fence *fence;
+	int ret = -ENOENT;
+
+	spin_lock(&fi->lock);
+	list_for_each_entry(fence, &fi->list, entry) {
+		if (fence->rid == rid) {
+			list_del_init(&fence->entry);
+			ret = 0;
+			break;
+		}
+	}
+	spin_unlock(&fi->lock);
+
+	if (ret == 0) {
+		destroy_fence(fence);
+		wake_up(&fi->waitq);
+	}
+
+	return ret;
+}
+
+static bool all_fenced(struct fence_info *fi, bool *error)
+{
+	struct pending_fence *fence;
+	bool all = true;
+
+	*error = false;
+
+	spin_lock(&fi->lock);
+	list_for_each_entry(fence, &fi->list, entry) {
+		if (fence->error) {
+			*error = true;
+			all = true;
+			break;
+		}
+		if (!fence->fenced) {
+			all = false;
+			break;
+		}
+	}
+	spin_unlock(&fi->lock);
+
+	return all;
+}
+
+/*
+ * The caller waits for all the current requests to be fenced, but not
+ * necessarily reclaimed.
+ */
+int scoutfs_fence_wait_fenced(struct super_block *sb, long timeout_jiffies)
+{
+	DECLARE_FENCE_INFO(sb, fi);
+	bool error;
+	long ret;
+
+	ret = wait_event_interruptible_timeout(fi->waitq, all_fenced(fi, &error), timeout_jiffies);
+	if (ret == 0)
+		ret = -ETIMEDOUT;
+	else if (ret > 0)
+		ret = 0;
+	else if (error)
+		ret = -EIO;
+
+	return ret;
+}
+
+/*
+ * This must be called early during startup so that it is guaranteed that
+ * no other subsystems will try and call fence_start while we're waiting
+ * for testing fence requests to complete.
+ */
+int scoutfs_fence_setup(struct super_block *sb)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	struct mount_options *opts = &sbi->opts;
+	struct fence_info *fi;
+	int ret;
+
+	/* can only fence if we can be elected by quorum */
+	if (opts->quorum_slot_nr == -1) {
+		ret = 0;
+		goto out;
+	}
+
+	fi = kzalloc(sizeof(struct fence_info), GFP_KERNEL);
+	if (!fi) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	init_waitqueue_head(&fi->waitq);
+	spin_lock_init(&fi->lock);
+	INIT_LIST_HEAD(&fi->list);
+
+	sbi->fence_info = fi;
+
+	fi->kset = kset_create_and_add("fence", NULL, scoutfs_sysfs_sb_dir(sb));
+	if (!fi->kset) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	fi->wq = alloc_workqueue("scoutfs_fence",
+				 WQ_UNBOUND | WQ_NON_REENTRANT, 0);
+	if (!fi->wq) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ret = 0;
+out:
+	if (ret)
+		scoutfs_fence_destroy(sb);
+
+	return ret;
+}
+
+/*
+ * Tear down all pending fence requests because the server is shutting down.
+ */
+void scoutfs_fence_stop(struct super_block *sb)
+{
+	DECLARE_FENCE_INFO(sb, fi);
+	struct pending_fence *fence;
+
+	do {
+		spin_lock(&fi->lock);
+		fence = list_first_entry_or_null(&fi->list, struct pending_fence, entry);
+		if (fence)
+			list_del_init(&fence->entry);
+		spin_unlock(&fi->lock);
+
+		if (fence) {
+			destroy_fence(fence);
+			wake_up(&fi->waitq);
+		}
+	} while (fence);
+}
+
+void scoutfs_fence_destroy(struct super_block *sb)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	struct fence_info *fi = SCOUTFS_SB(sb)->fence_info;
+	struct pending_fence *fence;
+	struct pending_fence *tmp;
+
+	if (fi) {
+		if (fi->wq)
+			destroy_workqueue(fi->wq);
+		list_for_each_entry_safe(fence, tmp, &fi->list, entry)
+			destroy_fence(fence);
+		if (fi->kset)
+			kset_unregister(fi->kset);
+		kfree(fi);
+		sbi->fence_info = NULL;
+	}
+}

kmod/src/fence.h

@@ -0,0 +1,20 @@
+#ifndef _SCOUTFS_FENCE_H_
+#define _SCOUTFS_FENCE_H_
+
+enum {
+	SCOUTFS_FENCE_CLIENT_RECOVERY,
+	SCOUTFS_FENCE_CLIENT_RECONNECT,
+	SCOUTFS_FENCE_QUORUM_BLOCK_LEADER,
+};
+
+int scoutfs_fence_start(struct super_block *sb, u64 rid, __be32 ipv4_addr, int reason);
+int scoutfs_fence_next(struct super_block *sb, u64 *rid, int *reason, bool *error);
+int scoutfs_fence_reason_pending(struct super_block *sb, int reason);
+int scoutfs_fence_free(struct super_block *sb, u64 rid);
+int scoutfs_fence_wait_fenced(struct super_block *sb, long timeout_jiffies);
+
+int scoutfs_fence_setup(struct super_block *sb);
+void scoutfs_fence_stop(struct super_block *sb);
+void scoutfs_fence_destroy(struct super_block *sb);
+
+#endif

kmod/src/format.h

@@ -327,7 +327,9 @@ struct scoutfs_alloc_root {
 #define SCOUTFS_ALLOC_OWNER_SRCH	3
 
 struct scoutfs_mounted_client_btree_val {
+	union scoutfs_inet_addr addr;
 	__u8 flags;
+	__u8 __pad[7];
 };
 
 #define SCOUTFS_MOUNTED_CLIENT_QUORUM	(1 << 0)
@@ -586,6 +588,12 @@ struct scoutfs_xattr {
 #define SCOUTFS_QUORUM_HB_IVAL_MS	100
 #define SCOUTFS_QUORUM_HB_TIMEO_MS	(5 * MSEC_PER_SEC)
 
+/*
+ * A newly elected leader will give fencing some time before giving up and
+ * shutting down.
+ */
+#define SCOUTFS_QUORUM_FENCE_TO_MS	(15 * MSEC_PER_SEC)
+
 struct scoutfs_quorum_message {
 	__le64 fsid;
 	__le64 version;
@@ -617,18 +625,24 @@ struct scoutfs_quorum_config {
 	} slots[SCOUTFS_QUORUM_MAX_SLOTS];
 };
 
-struct scoutfs_quorum_block {
-	struct scoutfs_block_header hdr;
-	__le64 term;
-	__le64 random_write_mark;
-	__le64 flags;
-	struct scoutfs_quorum_block_event {
-		__le64 rid;
-		struct scoutfs_timespec ts;
-	} write, update_term, set_leader, clear_leader, fenced;
+enum {
+	SCOUTFS_QUORUM_EVENT_BEGIN,		/* quorum service starting up */
+	SCOUTFS_QUORUM_EVENT_TERM,		/* updated persistent term */
+	SCOUTFS_QUORUM_EVENT_ELECT,		/* won election */
+	SCOUTFS_QUORUM_EVENT_FENCE,		/* server fenced others */
+	SCOUTFS_QUORUM_EVENT_STOP,		/* server stopped */
+	SCOUTFS_QUORUM_EVENT_END,		/* quorum service shutting down */
+	SCOUTFS_QUORUM_EVENT_NR,
 };
 
-#define SCOUTFS_QUORUM_BLOCK_LEADER (1 << 0)
+struct scoutfs_quorum_block {
+	struct scoutfs_block_header hdr;
+	struct scoutfs_quorum_block_event {
+		__le64 rid;
+		__le64 term;
+		struct scoutfs_timespec ts;
+	} events[SCOUTFS_QUORUM_EVENT_NR];
+};
 
 /*
  * Tunable options that apply to the entire system.  They can be set in

kmod/src/lock_server.c

@@ -486,7 +486,7 @@ static int process_waiting_requests(struct super_block *sb,
 
 	/* processing waits for all invalidation responses or recovery */
 	if (!list_empty(&snode->invalidated) ||
-	    scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_LOCKS) != 0) {
+	    scoutfs_recov_next_pending(sb, 0, SCOUTFS_RECOV_LOCKS) != 0) {
 		ret = 0;
 		goto out;
 	}

kmod/src/net.c

@@ -30,6 +30,7 @@
 #include "net.h"
 #include "endian_swap.h"
 #include "tseq.h"
+#include "fence.h"
 
 /*
  * scoutfs networking delivers requests and responses between nodes.
@@ -330,6 +331,9 @@ static int submit_send(struct super_block *sb,
 	    WARN_ON_ONCE(id == 0 && (flags & SCOUTFS_NET_FLAG_RESPONSE)))
 		return -EINVAL;
 
+	if (scoutfs_forcing_unmount(sb))
+		return -EIO;
+
 	msend = kmalloc(offsetof(struct message_send,
 				 nh.data[data_len]), GFP_NOFS);
 	if (!msend)
@@ -420,6 +424,16 @@ static int process_request(struct scoutfs_net_connection *conn,
 			mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len));
 }
 
+static int call_resp_func(struct super_block *sb, struct scoutfs_net_connection *conn,
+			  scoutfs_net_response_t resp_func, void *resp_data,
+			  void *resp, unsigned int resp_len, int error)
+{
+	if (resp_func)
+		return resp_func(sb, conn, resp, resp_len, error, resp_data);
+	else
+		return 0;
+}
+
 /*
  * An incoming response finds the queued request and calls its response
  * function.  The response function for a given request will only be
@@ -434,7 +448,6 @@ static int process_response(struct scoutfs_net_connection *conn,
 	struct message_send *msend;
 	scoutfs_net_response_t resp_func = NULL;
 	void *resp_data;
-	int ret = 0;
 
 	spin_lock(&conn->lock);
 
@@ -449,11 +462,8 @@ static int process_response(struct scoutfs_net_connection *conn,
 
 	spin_unlock(&conn->lock);
 
-	if (resp_func)
-		ret = resp_func(sb, conn, mrecv->nh.data,
-				le16_to_cpu(mrecv->nh.data_len),
-				net_err_to_host(mrecv->nh.error), resp_data);
-	return ret;
+	return call_resp_func(sb, conn, resp_func, resp_data, mrecv->nh.data,
+			      le16_to_cpu(mrecv->nh.data_len), net_err_to_host(mrecv->nh.error));
 }
 
 /*
@@ -823,9 +833,15 @@ static void scoutfs_net_destroy_worker(struct work_struct *work)
 	if (conn->listening_conn && conn->notify_down)
 		conn->notify_down(sb, conn, conn->info, conn->rid);
 
-	/* free all messages, refactor and complete for forced unmount? */
+	/*
+	 * Usually networking is idle and we destroy pending sends, but when forcing unmount
+	 * we can have to wake up waiters by failing pending sends.
+	 */
 	list_splice_init(&conn->resend_queue, &conn->send_queue);
 	list_for_each_entry_safe(msend, tmp, &conn->send_queue, head) {
+		if (scoutfs_forcing_unmount(sb))
+			call_resp_func(sb, conn, msend->resp_func, msend->resp_data,
+				       NULL, 0, -ECONNABORTED);
 		free_msend(ninf, msend);
 	}
 
@@ -925,6 +941,8 @@ static int sock_opts_and_names(struct scoutfs_net_connection *conn,
 		ret = -EAFNOSUPPORT;
 	if (ret)
 		goto out;
+
+	conn->last_peername = conn->peername;
 out:
 	return ret;
 }
@@ -1205,6 +1223,7 @@ static void scoutfs_net_reconn_free_worker(struct work_struct *work)
 	unsigned long now = jiffies;
 	unsigned long deadline = 0;
 	bool requeue = false;
+	int ret;
 
 	trace_scoutfs_net_reconn_free_work_enter(sb, 0, 0);
 
@@ -1218,10 +1237,18 @@ restart:
 		     time_after_eq(now, acc->reconn_deadline))) {
 			set_conn_fl(acc, reconn_freeing);
 			spin_unlock(&conn->lock);
-			if (!test_conn_fl(conn, shutting_down))
-				scoutfs_info(sb, "client timed out "SIN_FMT" -> "SIN_FMT", can not reconnect",
-					     SIN_ARG(&acc->sockname),
-					     SIN_ARG(&acc->peername));
+			if (!test_conn_fl(conn, shutting_down)) {
+				scoutfs_info(sb, "client "SIN_FMT" reconnect timed out, fencing",
+					     SIN_ARG(&acc->last_peername));
+				ret = scoutfs_fence_start(sb, acc->rid,
+						acc->last_peername.sin_addr.s_addr,
+						SCOUTFS_FENCE_CLIENT_RECONNECT);
+				if (ret) {
+					scoutfs_err(sb, "client fence returned err %d, shutting down server",
+						    ret);
+					scoutfs_server_abort(sb);
+				}
+			}
 			destroy_conn(acc);
 			goto restart;
 		}
@@ -1292,6 +1319,7 @@ scoutfs_net_alloc_conn(struct super_block *sb,
 	init_waitqueue_head(&conn->waitq);
 	conn->sockname.sin_family = AF_INET;
 	conn->peername.sin_family = AF_INET;
+	conn->last_peername.sin_family = AF_INET;
 	INIT_LIST_HEAD(&conn->accepted_head);
 	INIT_LIST_HEAD(&conn->accepted_list);
 	conn->next_send_seq = 1;

kmod/src/net.h

@@ -49,6 +49,7 @@ struct scoutfs_net_connection {
 	u64 greeting_id;
 	struct sockaddr_in sockname;
 	struct sockaddr_in peername;
+	struct sockaddr_in last_peername;
 
 	struct list_head accepted_head;
 	struct scoutfs_net_connection *listening_conn;
@@ -99,6 +100,16 @@ static inline void scoutfs_addr_to_sin(struct sockaddr_in *sin,
 	sin->sin_port = cpu_to_be16(le16_to_cpu(addr->v4.port));
 }
 
+static inline void scoutfs_sin_to_addr(union scoutfs_inet_addr *addr, struct sockaddr_in *sin)
+{
+	BUG_ON(sin->sin_family != AF_INET);
+
+	memset(addr, 0, sizeof(union scoutfs_inet_addr));
+	addr->v4.family = cpu_to_le16(SCOUTFS_AF_IPV4);
+	addr->v4.addr = be32_to_le32(sin->sin_addr.s_addr);
+	addr->v4.port = be16_to_le16(sin->sin_port);
+}
+
 struct scoutfs_net_connection *
 scoutfs_net_alloc_conn(struct super_block *sb,
 		       scoutfs_net_notify_t notify_up,

kmod/src/omap.c

@@ -485,6 +485,10 @@ static int remove_rid_from_reqs(struct omap_info *ominf, u64 rid, u64 *resp_rid,
  * response if it was the last rid waiting for a response.
  *
  * If this returns an error then the server will shut down.
+ *
+ * This can be called multiple times by different servers if there are
+ * errors reclaiming an evicted mount, so we allow asking to remove a
+ * rid that hasn't been added.
  */
 int scoutfs_omap_remove_rid(struct super_block *sb, u64 rid)
 {
@@ -495,21 +499,20 @@ int scoutfs_omap_remove_rid(struct super_block *sb, u64 rid)
 	u64 resp_id = 0;
 	int ret;
 
-	map = kmalloc(sizeof(struct scoutfs_open_ino_map), GFP_NOFS);
-	if (!map) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
 	spin_lock(&ominf->lock);
 	entry = find_rid(&ominf->rids, rid);
 	if (entry)
 		free_rid(&ominf->rids, entry);
 	spin_unlock(&ominf->lock);
 
-	/* the server really shouldn't be removing a rid it never added */
-	if (WARN_ON_ONCE(!entry)) {
-		ret = -ENOENT;
+	if (!entry) {
+		ret = 0;
+		goto out;
+	}
+
+	map = kmalloc(sizeof(struct scoutfs_open_ino_map), GFP_NOFS);
+	if (!map) {
+		ret = -ENOMEM;
 		goto out;
 	}
 
@@ -616,7 +619,7 @@ static int handle_requests(struct super_block *sb)
 	int ret;
 	int err;
 
-	if (scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_GREETING))
+	if (scoutfs_recov_next_pending(sb, 0, SCOUTFS_RECOV_GREETING))
 		return 0;
 
 	ret = 0;

kmod/src/quorum.c

@@ -32,6 +32,7 @@
 #include "block.h"
 #include "net.h"
 #include "sysfs.h"
+#include "fence.h"
 #include "scoutfs_trace.h"
 
 /*
@@ -60,10 +61,9 @@
  * running (maybe they've deadlocked, or lost network communications).
  * In addition to a configuration slot in the super block, each quorum
  * member also has a known block location that represents their slot.
- * They set a flag in their block indicating that they've been elected
- * leader, then read slots for all the other blocks looking for
- * previously active leaders to fence.  After that it can start the
- * server.
+ * The block contains an array of events which are updated during the life
+ * time of the quorum agent.  The elected leader set its elected event
+ * and can then start the server.
  *
  * It's critical to raft elections that a participant's term not go
  * backwards in time so each mount also uses its quorum block to store
@@ -334,17 +334,18 @@ static int recv_msg(struct super_block *sb, struct quorum_host_msg *msg,
 }
 
 /*
- * The caller can provide a mark that they're using to track their
- * written blocks.  It's updated as they write the block and we can
- * compare it with what we read to see if there have been unexpected
- * intervening writes to the block -- the caller is supposed to have
- * exclusive access to the block (or was fenced).
+ * Read and verify block fields before giving it to the caller.  We
+ * should have exclusive write access to the block.  We know that
+ * something has gone horribly wrong if we don't see our rid in the
+ * begin event after we've written it as we started up.
  */
-static int read_quorum_block(struct super_block *sb, u64 blkno,
-			     struct scoutfs_quorum_block *blk, __le64 *mark)
+static int read_quorum_block(struct super_block *sb, u64 blkno, struct scoutfs_quorum_block *blk,
+			     bool check_rid)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
 	struct scoutfs_super_block *super = &sbi->super;
+	const u64 rid = sbi->rid;
+	char msg[150];
 	__le32 crc;
 	int ret;
 
@@ -355,162 +356,208 @@ static int read_quorum_block(struct super_block *sb, u64 blkno,
 
 	ret = scoutfs_block_read_sm(sb, sbi->meta_bdev, blkno,
 				     &blk->hdr, sizeof(*blk), &crc);
+	if (ret < 0) {
+		scoutfs_err(sb, "quorum block read error %d", ret);
+		goto out;
+	}
 
 	/* detect invalid blocks */
-	if (ret == 0 &&
-	    ((blk->hdr.crc != crc) ||
-	     (le32_to_cpu(blk->hdr.magic) != SCOUTFS_BLOCK_MAGIC_QUORUM) ||
-	     (blk->hdr.fsid != super->hdr.fsid) ||
-	     (le64_to_cpu(blk->hdr.blkno) != blkno))) {
-		scoutfs_inc_counter(sb, quorum_read_invalid_block);
+	if (blk->hdr.crc != crc)
+		snprintf(msg, sizeof(msg), "blk crc %08x != %08x",
+			 le32_to_cpu(blk->hdr.crc), le32_to_cpu(crc));
+	else if (le32_to_cpu(blk->hdr.magic) != SCOUTFS_BLOCK_MAGIC_QUORUM) 
+		snprintf(msg, sizeof(msg), "blk magic %08x != %08x",
+			 le32_to_cpu(blk->hdr.magic), SCOUTFS_BLOCK_MAGIC_QUORUM);
+	else if (blk->hdr.fsid != super->hdr.fsid)
+		snprintf(msg, sizeof(msg), "blk fsid %016llx != %016llx",
+			 le64_to_cpu(blk->hdr.fsid), le64_to_cpu(super->hdr.fsid));
+	else if (le64_to_cpu(blk->hdr.blkno) != blkno)
+		snprintf(msg, sizeof(msg), "blk blkno %llu != %llu",
+			 le64_to_cpu(blk->hdr.blkno), blkno);
+	else if (check_rid && le64_to_cpu(blk->events[SCOUTFS_QUORUM_EVENT_BEGIN].rid) != rid)
+		snprintf(msg, sizeof(msg), "quorum block begin rid %016llx != our rid %016llx, are multiple mounts configured with this slot?",
+		le64_to_cpu(blk->events[SCOUTFS_QUORUM_EVENT_BEGIN].rid), rid);
+	else
+		msg[0] = '\0';
+
+	if (msg[0] != '\0') {
+		scoutfs_err(sb, "read invalid quorum block, %s", msg);
 		ret = -EIO;
+		goto out;
 	}
 
-	if (mark && *mark != 0 && blk->random_write_mark != *mark) {
-		scoutfs_err(sb, "read unexpected quorum block write mark, are multiple mounts configured with the same slot?");
-		ret = -EIO;
-	}
-
-	if (ret < 0)
-		scoutfs_err(sb, "quorum block read error %d", ret);
-
+out:
 	return ret;
 }
 
-static void set_quorum_block_event(struct super_block *sb,
-				   struct scoutfs_quorum_block *blk,
-				   struct scoutfs_quorum_block_event *ev)
+static void set_quorum_block_event(struct super_block *sb, struct scoutfs_quorum_block *blk,
+				   int event, u64 term)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	struct scoutfs_quorum_block_event *ev;
 	struct timespec64 ts;
 
+	if (WARN_ON_ONCE(event < 0 || event >= SCOUTFS_QUORUM_EVENT_NR))
+		return;
+
 	getnstimeofday64(&ts);
 
+	ev = &blk->events[event];
 	ev->rid = cpu_to_le64(sbi->rid);
+	ev->term = cpu_to_le64(term);
 	ev->ts.sec = cpu_to_le64(ts.tv_sec);
 	ev->ts.nsec = cpu_to_le32(ts.tv_nsec);
 }
 
-/*
- * Every time we write a block we update the write stamp and random
- * write mark so readers can see our write.
- */
-static int write_quorum_block(struct super_block *sb, u64 blkno,
-			      struct scoutfs_quorum_block *blk, __le64 *mark)
+static int write_quorum_block(struct super_block *sb, u64 blkno, struct scoutfs_quorum_block *blk)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
-	int ret;
 
 	if (WARN_ON_ONCE(blkno < SCOUTFS_QUORUM_BLKNO) ||
 	    WARN_ON_ONCE(blkno >= (SCOUTFS_QUORUM_BLKNO +
 				   SCOUTFS_QUORUM_BLOCKS)))
 		return -EINVAL;
 
-	do {
-		get_random_bytes(&blk->random_write_mark,
-				 sizeof(blk->random_write_mark));
-	} while (blk->random_write_mark == 0);
-
-	if (mark)
-		*mark = blk->random_write_mark;
-
-	set_quorum_block_event(sb, blk, &blk->write);
-
-	ret = scoutfs_block_write_sm(sb, sbi->meta_bdev, blkno,
-				      &blk->hdr, sizeof(*blk));
-	if (ret < 0)
-		scoutfs_err(sb, "quorum block write error %d", ret);
-
-	return ret;
+	return scoutfs_block_write_sm(sb, sbi->meta_bdev, blkno, &blk->hdr, sizeof(*blk));
 }
 
 /*
- * Read the caller's slot's current quorum block, make a change, and
- * write it back out.  If the caller provides a mark it can cause read
- * errors if we read a mark that doesn't match the last mark that the
- * caller wrote.
+ * Read the caller's slot's quorum block, make a change, and write it
+ * back out.
  */
-static int update_quorum_block(struct super_block *sb, u64 blkno,
-			       __le64 *mark, int role, u64 term)
+static int update_quorum_block(struct super_block *sb, int event, u64 term, bool check_rid)
 {
+	struct mount_options *opts = &SCOUTFS_SB(sb)->opts;
+	u64 blkno = SCOUTFS_QUORUM_BLKNO + opts->quorum_slot_nr;
 	struct scoutfs_quorum_block blk;
-	u64 flags;
-	u64 bits;
-	u64 set;
 	int ret;
 
-	ret = read_quorum_block(sb, blkno, &blk, mark);
+	ret = read_quorum_block(sb, blkno, &blk, check_rid);
 	if (ret == 0) {
-		if (blk.term != cpu_to_le64(term)) {
-			blk.term = cpu_to_le64(term);
-			set_quorum_block_event(sb, &blk, &blk.update_term);
-		}
-
-		flags = le64_to_cpu(blk.flags);
-		bits = SCOUTFS_QUORUM_BLOCK_LEADER;
-		set = role == LEADER ? SCOUTFS_QUORUM_BLOCK_LEADER : 0;
-		if ((flags & bits) != set)
-			set_quorum_block_event(sb, &blk,
-					       set ? &blk.set_leader :
-					             &blk.clear_leader);
-		blk.flags = cpu_to_le64((flags & ~bits) | set);
-
-		ret = write_quorum_block(sb, blkno, &blk, mark);
+		set_quorum_block_event(sb, &blk, event, term);
+		ret = write_quorum_block(sb, blkno, &blk);
+		if (ret < 0)
+			scoutfs_err(sb, "error %d reading quorum block %llu to update event %d term %llu",
+				    ret, blkno, event, term);
+	} else {
+		scoutfs_err(sb, "error %d writing quorum block %llu after updating event %d term %llu",
+			    ret, blkno, event, term);
 	}
 
 	return ret;
 }
 
+/*
+ * The calling server has fenced previous leaders and reclaimed their
+ * resources.  We can now update our fence event with a greater term to
+ * stop future leaders from doing the same.
+ */
+int scoutfs_quorum_fence_complete(struct super_block *sb, u64 term)
+{
+	return update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_FENCE, term, true);
+}
 
 /*
- * The calling server has been elected and updated their block, but
- * can't yet assume that it has exclusive access to the metadata device.
- * We read all the quorum blocks looking for previously elected leaders
- * to fence so that we're the only leader running.
+ * The calling server has been elected and has started running but can't
+ * yet assume that it has exclusive access to the metadata device.  We
+ * read all the quorum blocks looking for previously elected leaders to
+ * fence so that we're the only leader running.
+ *
+ * We're relying on the invariant that there can't be two mounts running
+ * with the same slot nr at the same time.  With this constraint there
+ * can be at most two previous leaders per slot that need to be fenced:
+ * a persistent record of an old mount on the slot, and an active mount.
+ *
+ * If we start fence requests then we only wait for them to complete
+ * before returning.  The server will reclaim their resources once it is
+ * up and running and will call us to update the fence event.  If we
+ * don't start fence requests then we update the fence event
+ * immediately, the server has nothing more to do.
+ *
+ * Quorum will be sending heartbeats while we wait for fencing.  That
+ * keeps us from being fenced while we allow userspace fencing to take a
+ * reasonably long time.  We still want to timeout eventually.
  */
-static int fence_leader_blocks(struct super_block *sb)
+int scoutfs_quorum_fence_leaders(struct super_block *sb, u64 term)
 {
+#define NR_OLD 2
+	struct scoutfs_quorum_block_event old[SCOUTFS_QUORUM_MAX_SLOTS][NR_OLD] = {{{0,}}};
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
 	struct scoutfs_super_block *super = &sbi->super;
-	struct mount_options *opts = &sbi->opts;
 	struct scoutfs_quorum_block blk;
 	struct sockaddr_in sin;
-	u64 blkno;
+	const u64 rid = sbi->rid;
+	bool fence_started = false;
+	u64 fenced = 0;
+	__le64 fence_rid;
 	int ret = 0;
+	int err;
 	int i;
+	int j;
 
 	BUILD_BUG_ON(SCOUTFS_QUORUM_BLOCKS < SCOUTFS_QUORUM_MAX_SLOTS);
 
 	for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
-		if (i == opts->quorum_slot_nr)
+		if (!quorum_slot_present(super, i))
 			continue;
 
-		blkno = SCOUTFS_QUORUM_BLKNO + i;
-		ret = read_quorum_block(sb, blkno, &blk, NULL);
+		ret = read_quorum_block(sb, SCOUTFS_QUORUM_BLKNO + i, &blk, false);
 		if (ret < 0)
 			goto out;
 
-		if (!(le64_to_cpu(blk.flags) & SCOUTFS_QUORUM_BLOCK_LEADER))
-			continue;
+		/* elected leader still running */
+		if (le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_ELECT].term) >
+		    le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_STOP].term))
+			old[i][0] = blk.events[SCOUTFS_QUORUM_EVENT_ELECT];
 
-		scoutfs_inc_counter(sb, quorum_fence_leader);
-		scoutfs_quorum_slot_sin(super, i, &sin);
+		/* persistent record of previous server before elected */
+		if ((le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_FENCE].term) >
+		     le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_STOP].term)) &&
+		    (le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_FENCE].term) <
+		     le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_ELECT].term)))
+			old[i][1] = blk.events[SCOUTFS_QUORUM_EVENT_FENCE];
 
-		scoutfs_err(sb, "fencing "SCSBF" at "SIN_FMT,
-			    SCSB_LEFR_ARGS(super->hdr.fsid, blk.set_leader.rid),
-			    SIN_ARG(&sin));
+		/* find greatest term that has fenced everything before it */
+		fenced = max(fenced, le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_FENCE].term));
+	}
 
-		blk.flags &= ~cpu_to_le64(SCOUTFS_QUORUM_BLOCK_LEADER);
-		set_quorum_block_event(sb, &blk, &blk.fenced);
+	/* now actually fence any old leaders which haven't been fenced yet */
+	for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
+		for (j = 0; j < NR_OLD; j++) {
+			if (le64_to_cpu(old[i][j].term) == 0 ||		/* uninitialized */
+			    le64_to_cpu(old[i][j].term) < fenced ||	/* already fenced */
+			    le64_to_cpu(old[i][j].term) > term ||	/* newer than us */
+			    le64_to_cpu(old[i][j].rid) == rid)		/* us */
+				continue;
 
-		ret = write_quorum_block(sb, blkno, &blk, NULL);
-		if (ret < 0)
-			goto out;
+			scoutfs_inc_counter(sb, quorum_fence_leader);
+			scoutfs_quorum_slot_sin(super, i, &sin);
+			fence_rid = old[i][j].rid;
+
+			scoutfs_info(sb, "fencing previous leader "SCSBF" at term %llu in slot %u with address "SIN_FMT,
+				     SCSB_LEFR_ARGS(super->hdr.fsid, fence_rid),
+				     le64_to_cpu(old[i][j].term), i, SIN_ARG(&sin));
+			ret = scoutfs_fence_start(sb, le64_to_cpu(fence_rid), sin.sin_addr.s_addr,
+						  SCOUTFS_FENCE_QUORUM_BLOCK_LEADER);
+			if (ret < 0)
+				goto out;
+			fence_started = true;
+		}
 	}
 
 out:
+	if (fence_started) {
+		err = scoutfs_fence_wait_fenced(sb, msecs_to_jiffies(SCOUTFS_QUORUM_FENCE_TO_MS));
+		if (ret == 0)
+			ret = err;
+	} else {
+		err = scoutfs_quorum_fence_complete(sb, term);
+		if (ret == 0)
+			ret = err;
+	}
+
 	if (ret < 0) {
-		scoutfs_err(sb, "error %d fencing active", ret);
+		scoutfs_err(sb, "error %d attempting to find and fence previous leaders", ret);
 		scoutfs_inc_counter(sb, quorum_fence_error);
 	}
 
@@ -533,23 +580,22 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 	struct sockaddr_in unused;
 	struct quorum_host_msg msg;
 	struct quorum_status qst;
-	__le64 mark;
 	u64 blkno;
 	int ret;
+	int err;
 
 	/* recording votes from slots as native single word bitmap */
 	BUILD_BUG_ON(SCOUTFS_QUORUM_MAX_SLOTS > BITS_PER_LONG);
 
 	/* get our starting term from our persistent block */
-	mark = 0;
 	blkno = SCOUTFS_QUORUM_BLKNO + opts->quorum_slot_nr;
-	ret = read_quorum_block(sb, blkno, &blk, &mark);
+	ret = read_quorum_block(sb, blkno, &blk, false);
 	if (ret < 0)
 		goto out;
 
 	/* start out as a follower */
 	qst.role = FOLLOWER;
-	qst.term = le64_to_cpu(blk.term);
+	qst.term = le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_TERM].term);
 	qst.vote_for = -1;
 	qst.vote_bits = 0;
 
@@ -559,6 +605,11 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 	else
 		qst.timeout = election_timeout();
 
+	/* record that we're up and running, readers check that it isn't updated */
+	ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_BEGIN, qst.term, false);
+	if (ret < 0)
+		goto out;
+
 	while (!qinf->shutdown) {
 
 		ret = recv_msg(sb, &msg, qst.timeout);
@@ -589,11 +640,6 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 			send_msg_others(sb, SCOUTFS_QUORUM_MSG_RESIGNATION,
 					qst.term);
 			scoutfs_inc_counter(sb, quorum_send_resignation);
-
-			ret = update_quorum_block(sb, blkno, &mark,
-						  qst.role, qst.term);
-			if (ret < 0)
-				goto out;
 		}
 
 		spin_lock(&qinf->show_lock);
@@ -624,8 +670,7 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 				qst.timeout = election_timeout();
 
 			/* store our increased term */
-			ret = update_quorum_block(sb, blkno, &mark,
-						  qst.role, qst.term);
+			ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_TERM, qst.term, true);
 			if (ret < 0)
 				goto out;
 		}
@@ -642,6 +687,11 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 					qst.term);
 			qst.timeout = election_timeout();
 			scoutfs_inc_counter(sb, quorum_send_request);
+
+			/* store our increased term */
+			ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_TERM, qst.term, true);
+			if (ret < 0)
+				goto out;
 		}
 
 		/* candidates count votes in their term */
@@ -670,10 +720,8 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 					qst.term);
 			qst.timeout = heartbeat_interval();
 
-			/* set our leader flag and fence */
-			ret = update_quorum_block(sb, blkno, &mark,
-						  qst.role, qst.term) ?:
-			      fence_leader_blocks(sb);
+			/* record that we've been elected before starting up server */
+			ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_ELECT, qst.term, true);
 			if (ret < 0)
 				goto out;
 
@@ -684,8 +732,13 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 
 			ret = scoutfs_server_start(sb, qst.term);
 			if (ret < 0) {
-				scoutfs_err(sb, "server startup failed with %d",
-					    ret);
+				clear_bit(QINF_FLAG_SERVER, &qinf->flags);
+				scoutfs_err(sb, "server startup failed with %d", ret);
+				/* store our increased term */
+				err = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_STOP, qst.term,
+							  true);
+				if (err < 0 && ret == 0)
+					ret = err;
 				goto out;
 			}
 		}
@@ -727,17 +780,13 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 	/* always try to stop a running server as we stop */
 	if (test_bit(QINF_FLAG_SERVER, &qinf->flags)) {
 		scoutfs_server_stop(sb);
+		scoutfs_fence_stop(sb);
 		send_msg_others(sb, SCOUTFS_QUORUM_MSG_RESIGNATION,
 				qst.term);
 	}
 
-	/* always try to clear leader block as we stop to avoid fencing */
-	if (qst.role == LEADER) {
-		ret = update_quorum_block(sb, blkno, &mark,
-					  FOLLOWER, qst.term);
-		if (ret < 0)
-			goto out;
-	}
+	/* informational event that we're shutting down, nothing relies on it */
+	update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_END, qst.term, true);
 out:
 	if (ret < 0) {
 		scoutfs_err(sb, "quorum service saw error %d, shutting down.  Cluster will be degraded until this slot is remounted to restart the quorum service",
@@ -746,58 +795,60 @@ out:
 }
 
 /*
- * Set a flag for the quorum work's next iteration to indicate that the
- * server has shutdown and that it should step down as leader, update
- * quorum blocks, and stop sending heartbeats.
+ * The calling server has shutdown and is no longer using shared
+ * resources.  Clear the bit so that we stop sending heartbeats and
+ * allow the next server to be elected.  Update the stop event so that
+ * it won't be considered available by clients or fenced by the next
+ * leader.
  */
-void scoutfs_quorum_server_shutdown(struct super_block *sb)
+void scoutfs_quorum_server_shutdown(struct super_block *sb, u64 term)
 {
 	DECLARE_QUORUM_INFO(sb, qinf);
 
-	set_bit(QINF_FLAG_SERVER, &qinf->flags);
+	clear_bit(QINF_FLAG_SERVER, &qinf->flags);
+	update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_STOP, term, true);
 }
 
 /*
  * Clients read quorum blocks looking for the leader with a server whose
  * address it can try and connect to.
  *
- * There can be multiple running servers if a client checks before a
- * server has had a chance to fence any old servers.  We try to use the
- * block with the most recent timestamp.  If we get it wrong the
- * connection will timeout and the client will try again, presumably
- * finding a single server block.
+ * There can be records of multiple previous elected leaders if the
+ * current server hasn't yet fenced any old servers.  We use the elected
+ * leader with the greatest elected term.  If we get it wrong the
+ * connection will timeout and the client will try again.
  */
 int scoutfs_quorum_server_sin(struct super_block *sb, struct sockaddr_in *sin)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
 	struct scoutfs_super_block *super = &sbi->super;
 	struct scoutfs_quorum_block blk;
-	struct timespec64 recent = {0,};
-	struct timespec64 ts;
-	int ret;
+	u64 elect_term;
+	u64 term = 0;
+	int ret = 0;
 	int i;
 
 	for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
-		ret = read_quorum_block(sb, SCOUTFS_QUORUM_BLKNO + i, &blk,
-					NULL);
+		if (!quorum_slot_present(super, i))
+			continue;
+
+		ret = read_quorum_block(sb, SCOUTFS_QUORUM_BLKNO + i, &blk, false);
 		if (ret < 0) {
 			scoutfs_err(sb, "error reading quorum block nr %u: %d",
 				    i, ret);
 			goto out;
 		}
 
-		ts.tv_sec = le64_to_cpu(blk.set_leader.ts.sec);
-		ts.tv_nsec = le32_to_cpu(blk.set_leader.ts.nsec);
-
-		if ((le64_to_cpu(blk.flags) & SCOUTFS_QUORUM_BLOCK_LEADER) &&
-		    (timespec64_to_ns(&ts) > timespec64_to_ns(&recent))) {
-			recent = ts;
+		elect_term = le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_ELECT].term);
+		if (elect_term > term &&
+		    elect_term > le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_STOP].term)) {
+			term = elect_term;
 			scoutfs_quorum_slot_sin(super, i, sin);
 			continue;
 		}
 	}
 
-	if (timespec64_to_ns(&recent) == 0)
+	if (term == 0)
 		ret = -ENOENT;
 
 out:

kmod/src/quorum.h

@@ -2,12 +2,15 @@
 #define _SCOUTFS_QUORUM_H_
 
 int scoutfs_quorum_server_sin(struct super_block *sb, struct sockaddr_in *sin);
-void scoutfs_quorum_server_shutdown(struct super_block *sb);
+void scoutfs_quorum_server_shutdown(struct super_block *sb, u64 term);
 
 u8 scoutfs_quorum_votes_needed(struct super_block *sb);
 void scoutfs_quorum_slot_sin(struct scoutfs_super_block *super, int i,
 			     struct sockaddr_in *sin);
 
+int scoutfs_quorum_fence_leaders(struct super_block *sb, u64 term);
+int scoutfs_quorum_fence_complete(struct super_block *sb, u64 term);
+
 int scoutfs_quorum_setup(struct super_block *sb);
 void scoutfs_quorum_shutdown(struct super_block *sb);
 void scoutfs_quorum_destroy(struct super_block *sb);

kmod/src/recov.c

@@ -16,9 +16,11 @@
 #include <linux/sched.h>
 #include <linux/rhashtable.h>
 #include <linux/rcupdate.h>
+#include <linux/list_sort.h>
 
 #include "super.h"
 #include "recov.h"
+#include "cmp.h"
 
 /*
  * There are a few server messages which can't be processed until they
@@ -47,18 +49,41 @@ struct recov_pending {
 	int which;
 };
 
-static struct recov_pending *find_pending(struct recov_info *recinf, u64 rid, int which)
+static struct recov_pending *next_pending(struct recov_info *recinf, u64 rid, int which)
 {
 	struct recov_pending *pend;
 
 	list_for_each_entry(pend, &recinf->pending, head) {
-		if ((rid == 0 || pend->rid == rid) && (pend->which & which))
+		if (pend->rid > rid && pend->which & which)
 			return pend;
 	}
 
 	return NULL;
 }
 
+static struct recov_pending *lookup_pending(struct recov_info *recinf, u64 rid, int which)
+{
+	struct recov_pending *pend;
+
+	pend = next_pending(recinf, rid - 1, which);
+	if (pend && pend->rid == rid)
+		return pend;
+
+	return NULL;
+}
+
+/*
+ * We keep the pending list sorted by rid so that we can iterate over
+ * them.  The list should be small and shouldn't be used often.
+ */
+static int cmp_pending_rid(void *priv, struct list_head *A, struct list_head *B)
+{
+	struct recov_pending *a = list_entry(A, struct recov_pending, head);
+	struct recov_pending *b = list_entry(B, struct recov_pending, head);
+
+	return scoutfs_cmp_u64s(a->rid, b->rid);
+}
+
 /*
  * Record that we'll be waiting for a client to recover something.
  * _finished will eventually be called for every _prepare, either
@@ -80,14 +105,15 @@ int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which)
 
 	spin_lock(&recinf->lock);
 
-	pend = find_pending(recinf, rid, SCOUTFS_RECOV_ALL);
+	pend = lookup_pending(recinf, rid, SCOUTFS_RECOV_ALL);
 	if (pend) {
 		pend->which |= which;
 	} else {
 		swap(pend, alloc);
 		pend->rid = rid;
 		pend->which = which;
-		list_add(&pend->head, &recinf->pending);
+		list_add_tail(&pend->head, &recinf->pending);
+		list_sort(NULL, &recinf->pending, cmp_pending_rid);
 	}
 
 	spin_unlock(&recinf->lock);
@@ -159,7 +185,7 @@ int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which)
 
 	spin_lock(&recinf->lock);
 
-	pend = find_pending(recinf, rid, which);
+	pend = lookup_pending(recinf, rid, which);
 	if (pend) {
 		pend->which &= ~which;
 		if (pend->which) {
@@ -190,29 +216,28 @@ bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which)
 	bool is_pending;
 
 	spin_lock(&recinf->lock);
-	is_pending = find_pending(recinf, rid, which) != NULL;
+	is_pending = lookup_pending(recinf, rid, which) != NULL;
 	spin_unlock(&recinf->lock);
 
 	return is_pending;
 }
 
 /*
- * Returns 0 if there are no rids waiting for the given state to be
- * recovered.  Returns the rid of a client still waiting if there are
- * any, in no specified order.
+ * Return the next rid after the given rid of a client waiting for the
+ * given state to be recovered.  Start with rid 0, returns 0 when there
+ * are no more clients waiting for recovery.
  *
  * This is inherently racey.  Callers are responsible for resolving any
  * actions taken based on pending with the recovery finishing, perhaps
  * before we return.
  */
-u64 scoutfs_recov_next_pending(struct super_block *sb, int which)
+u64 scoutfs_recov_next_pending(struct super_block *sb, u64 rid, int which)
 {
 	DECLARE_RECOV_INFO(sb, recinf);
 	struct recov_pending *pend;
-	u64 rid;
 
 	spin_lock(&recinf->lock);
-	pend = find_pending(recinf, 0, which);
+	pend = next_pending(recinf, rid, which);
 	rid = pend ? pend->rid : 0;
 	spin_unlock(&recinf->lock);
 

kmod/src/recov.h

@@ -14,7 +14,7 @@ int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_
 			unsigned int timeout_ms);
 int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which);
 bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which);
-u64 scoutfs_recov_next_pending(struct super_block *sb, int which);
+u64 scoutfs_recov_next_pending(struct super_block *sb, u64 rid, int which);
 void scoutfs_recov_shutdown(struct super_block *sb);
 
 int scoutfs_recov_setup(struct super_block *sb);

kmod/src/server.c

@@ -40,6 +40,7 @@
 #include "forest.h"
 #include "recov.h"
 #include "omap.h"
+#include "fence.h"
 
 /*
  * Every active mount can act as the server that listens on a net
@@ -106,6 +107,8 @@ struct server_info {
 
 	/* recovery timeout fences from work */
 	struct work_struct fence_pending_recov_work;
+	/* while running we check for fenced mounts to reclaim */
+	struct delayed_work reclaim_dwork;
 };
 
 #define DECLARE_SERVER_INFO(sb, name) \
@@ -157,10 +160,21 @@ struct commit_waiter {
 	int ret;
 };
 
+static bool test_shutting_down(struct server_info *server)
+{
+	smp_rmb();
+	return server->shutting_down;
+}
+
+static void set_shutting_down(struct server_info *server, bool val)
+{
+	server->shutting_down = val;
+	smp_rmb();
+}
+
 static void stop_server(struct server_info *server)
 {
-	/* wait_event/wake_up provide barriers */
-	server->shutting_down = true;
+	set_shutting_down(server, true);
 	wake_up(&server->waitq);
 }
 
@@ -284,6 +298,11 @@ static void scoutfs_server_commit_func(struct work_struct *work)
 
 	down_write(&server->commit_rwsem);
 
+	if (scoutfs_forcing_unmount(sb)) {
+		ret = -EIO;
+		goto out;
+	}
+
 	/* make sure next avail has sufficient blocks */
 	ret = scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri,
 				      server->other_avail,
@@ -1434,14 +1453,19 @@ static void init_mounted_client_key(struct scoutfs_key *key, u64 rid)
 	};
 }
 
+static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref)
+{
+	return (iref->val_len != sizeof(struct scoutfs_mounted_client_btree_val));
+}
+
 /*
  * Insert a new mounted client item for a client that is sending us a
  * greeting that hasn't yet seen a response.  The greeting can be
  * retransmitted to a new server after the previous inserted the item so
  * it's acceptable to see -EEXIST.
  */
-static int insert_mounted_client(struct super_block *sb, u64 rid,
-				 u64 gr_flags)
+static int insert_mounted_client(struct super_block *sb, u64 rid, u64 gr_flags,
+				 struct sockaddr_in *sin)
 {
 	DECLARE_SERVER_INFO(sb, server);
 	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
@@ -1450,6 +1474,7 @@ static int insert_mounted_client(struct super_block *sb, u64 rid,
 	int ret;
 
 	init_mounted_client_key(&key, rid);
+	scoutfs_sin_to_addr(&mcv.addr, sin);
 	mcv.flags = 0;
 	if (gr_flags & SCOUTFS_NET_GREETING_FLAG_QUORUM)
 		mcv.flags |= SCOUTFS_MOUNTED_CLIENT_QUORUM;
@@ -1465,6 +1490,34 @@ static int insert_mounted_client(struct super_block *sb, u64 rid,
 	return ret;
 }
 
+static int lookup_mounted_client_addr(struct super_block *sb, u64 rid,
+				      union scoutfs_inet_addr *addr)
+{
+	DECLARE_SERVER_INFO(sb, server);
+	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
+	struct scoutfs_mounted_client_btree_val *mcv;
+	SCOUTFS_BTREE_ITEM_REF(iref);
+	struct scoutfs_key key;
+	int ret;
+
+	init_mounted_client_key(&key, rid);
+
+	mutex_lock(&server->mounted_clients_mutex);
+	ret = scoutfs_btree_lookup(sb, &super->mounted_clients, &key, &iref);
+	if (ret == 0) {
+		if (invalid_mounted_client_item(&iref)) {
+			ret = -EIO;
+		} else {
+			mcv = iref.val;
+			*addr = mcv->addr;
+		}
+		scoutfs_btree_put_iref(&iref);
+	}
+	mutex_unlock(&server->mounted_clients_mutex);
+
+	return ret;
+}
+
 /*
  * Remove the record of a mounted client.  The record can already be
  * removed if we're processing a farewell on behalf of a client that
@@ -1542,7 +1595,7 @@ static int cancel_srch_compact(struct super_block *sb, u64 rid)
  */
 static void queue_farewell_work(struct server_info *server)
 {
-	if (!server->shutting_down)
+	if (!test_shutting_down(server))
 		queue_work(server->wq, &server->farewell_work);
 }
 
@@ -1603,8 +1656,8 @@ static int server_greeting(struct super_block *sb,
 		if (ret < 0)
 			goto send_err;
 
-		ret = insert_mounted_client(sb, le64_to_cpu(gr->rid),
-					    le64_to_cpu(gr->flags));
+		ret = insert_mounted_client(sb, le64_to_cpu(gr->rid), le64_to_cpu(gr->flags),
+					    &conn->peername);
 
 		ret = scoutfs_server_apply_commit(sb, ret);
 		queue_work(server->wq, &server->farewell_work);
@@ -1661,12 +1714,15 @@ struct farewell_request {
 	u64 rid;
 };
 
-static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref)
-{
-	return (iref->val_len !=
-			sizeof(struct scoutfs_mounted_client_btree_val));
-}
 
+/*
+ * Reclaim all the resources for a mount which has gone away.  It's sent
+ * us a farewell promising to leave or we actively fenced it.
+ *
+ * It's safe to call this multiple times for a given rid.  Each
+ * individual action knows to recognize that it's already been performed
+ * and return success.
+ */
 static int reclaim_rid(struct super_block *sb, u64 rid)
 {
 	int ret;
@@ -1675,13 +1731,13 @@ static int reclaim_rid(struct super_block *sb, u64 rid)
 	if (ret < 0)
 		return ret;
 
-	/* delete mounted client last, client reconnect looks for it */
+	/* delete mounted client last, recovery looks for it */
 	ret = scoutfs_lock_server_farewell(sb, rid) ?:
 	      remove_trans_seq(sb, rid) ?:
 	      reclaim_log_trees(sb, rid) ?:
 	      cancel_srch_compact(sb, rid) ?:
-	      delete_mounted_client(sb, rid) ?:
-	      scoutfs_omap_remove_rid(sb, rid);
+	      scoutfs_omap_remove_rid(sb, rid) ?:
+	      delete_mounted_client(sb, rid);
 
 	return scoutfs_server_apply_commit(sb, ret);
 }
@@ -1811,7 +1867,7 @@ static void farewell_worker(struct work_struct *work)
 		}
 	}
 
-	/* process and send farewell responses */
+	/* clean up resources for mounts before sending responses */
 	list_for_each_entry_safe(fw, tmp, &send, entry) {
 		ret = reclaim_rid(sb, fw->rid);
 		if (ret)
@@ -2012,29 +2068,37 @@ static void fence_pending_recov_worker(struct work_struct *work)
 	struct server_info *server = container_of(work, struct server_info,
 						  fence_pending_recov_work);
 	struct super_block *sb = server->sb;
-	u64 rid;
-	int ret;
+	union scoutfs_inet_addr addr;
+	u64 rid = 0;
+	int ret = 0;
 
-	while ((rid = scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_ALL)) > 0) {
+	while ((rid = scoutfs_recov_next_pending(sb, rid, SCOUTFS_RECOV_ALL)) > 0) {
 		scoutfs_err(sb, "%lu ms recovery timeout expired for client rid %016llx, fencing",
 			    SERVER_RECOV_TIMEOUT_MS, rid);
 
-		ret = reclaim_rid(sb, rid);
+		ret = lookup_mounted_client_addr(sb, rid, &addr);
 		if (ret < 0) {
-			scoutfs_err(sb, "error %d reclaiming rid %016llx, shutting down", ret, rid);
-			stop_server(server);
+			scoutfs_err(sb, "client rid addr lookup err %d, shutting down server", ret);
 			break;
 		}
 
-		scoutfs_server_recov_finish(sb, rid, SCOUTFS_RECOV_ALL);
+		ret = scoutfs_fence_start(sb, rid, le32_to_be32(addr.v4.addr),
+					  SCOUTFS_FENCE_CLIENT_RECOVERY);
+		if (ret < 0) {
+			scoutfs_err(sb, "fence returned err %d, shutting down server", ret);
+			break;
+		}
 	}
+
+	if (ret < 0)
+		scoutfs_server_abort(sb);
 }
 
 static void recovery_timeout(struct super_block *sb)
 {
 	DECLARE_SERVER_INFO(sb, server);
 
-	if (!server->shutting_down)
+	if (!test_shutting_down(server))
 		queue_work(server->wq, &server->fence_pending_recov_work);
 }
 
@@ -2097,6 +2161,78 @@ out:
 	return ret;
 }
 
+static void queue_reclaim_work(struct server_info *server, unsigned long delay)
+{
+	if (!test_shutting_down(server))
+		queue_delayed_work(server->wq, &server->reclaim_dwork, delay);
+}
+
+#define RECLAIM_WORK_DELAY_MS	MSEC_PER_SEC
+
+/*
+ * Fencing is performed by userspace and can happen as we're elected
+ * leader before the server is running.  Once we're running we want to
+ * reclaim resources from any mounts that may have been fenced.
+ *
+ * The reclaim worker runs regularly in the background and reclaims the
+ * resources for mounts that have been fenced.  Once the fenced rid has
+ * been reclaimed the fence request can be removed.
+ *
+ * This is queued by the server work as it starts up, requeues itself
+ * until shutdown, and is then canceled by the server work as it shuts
+ * down.
+ */
+static void reclaim_worker(struct work_struct *work)
+{
+	struct server_info *server = container_of(work, struct server_info, reclaim_dwork.work);
+	struct super_block *sb = server->sb;
+	bool error;
+	int reason;
+	u64 rid;
+	int ret;
+
+	ret = scoutfs_fence_next(sb, &rid, &reason, &error);
+	if (ret < 0)
+		goto out;
+
+	if (error == true) {
+		scoutfs_err(sb, "saw error indicator on fence request for rid %016llx, shutting down server",
+			    rid);
+		scoutfs_server_abort(sb);
+		ret = -ESHUTDOWN;
+		goto out;
+	}
+
+	ret = reclaim_rid(sb, rid);
+	if (ret < 0) {
+		scoutfs_err(sb, "failure to reclaim fenced rid %016llx: err %d, shutting down server",
+			    rid, ret);
+		scoutfs_server_abort(sb);
+		goto out;
+	}
+
+	scoutfs_info(sb, "successfully reclaimed resources for fenced rid %016llx", rid);
+	scoutfs_fence_free(sb, rid);
+	scoutfs_server_recov_finish(sb, rid, SCOUTFS_RECOV_ALL);
+
+	/* tell quorum we've finished fencing all previous leaders */
+	if (reason == SCOUTFS_FENCE_QUORUM_BLOCK_LEADER &&
+	    !scoutfs_fence_reason_pending(sb, reason)) {
+		ret = scoutfs_quorum_fence_complete(sb, server->term);
+		if (ret < 0)
+			goto out;
+	}
+
+	ret = 0;
+
+out:
+	/* queue next reclaim immediately if we're making progress */
+	if (ret == 0)
+		queue_reclaim_work(server, 0);
+	else
+		queue_reclaim_work(server, msecs_to_jiffies(RECLAIM_WORK_DELAY_MS));
+}
+
 static void scoutfs_server_worker(struct work_struct *work)
 {
 	struct server_info *server = container_of(work, struct server_info,
@@ -2113,6 +2249,11 @@ static void scoutfs_server_worker(struct work_struct *work)
 
 	trace_scoutfs_server_work_enter(sb, 0, 0);
 
+	/* first make sure no other servers are still running */
+	ret = scoutfs_quorum_fence_leaders(sb, server->term);
+	if (ret < 0)
+		goto out;
+
 	scoutfs_quorum_slot_sin(super, opts->quorum_slot_nr, &sin);
 	scoutfs_info(sb, "server setting up at "SIN_FMT, SIN_ARG(&sin));
 
@@ -2184,14 +2325,17 @@ static void scoutfs_server_worker(struct work_struct *work)
 	scoutfs_info(sb, "server ready at "SIN_FMT, SIN_ARG(&sin));
 	complete(&server->start_comp);
 
+	queue_reclaim_work(server, 0);
+
 	/* wait_event/wake_up provide barriers */
-	wait_event_interruptible(server->waitq, server->shutting_down);
+	wait_event_interruptible(server->waitq, test_shutting_down(server));
 
 shutdown:
 	scoutfs_info(sb, "server shutting down at "SIN_FMT, SIN_ARG(&sin));
 
 	/* wait for farewell to finish sending messages */
 	flush_work(&server->farewell_work);
+	cancel_delayed_work_sync(&server->reclaim_dwork);
 
 	/* wait for requests to finish, no more requests */
 	scoutfs_net_shutdown(sb, conn);
@@ -2204,6 +2348,7 @@ shutdown:
 	/* wait for extra queues by requests, won't find waiters */
 	flush_work(&server->commit_work);
 
+	scoutfs_fence_stop(sb);
 	scoutfs_lock_server_destroy(sb);
 	scoutfs_omap_server_shutdown(sb);
 
@@ -2211,7 +2356,7 @@ out:
 	scoutfs_net_free_conn(sb, conn);
 
 	/* let quorum know that we've shutdown */
-	scoutfs_quorum_server_shutdown(sb);
+	scoutfs_quorum_server_shutdown(sb, server->term);
 
 	scoutfs_info(sb, "server stopped at "SIN_FMT, SIN_ARG(&sin));
 	trace_scoutfs_server_work_exit(sb, 0, ret);
@@ -2230,7 +2375,7 @@ int scoutfs_server_start(struct super_block *sb, u64 term)
 	DECLARE_SERVER_INFO(sb, server);
 
 	server->err = 0;
-	server->shutting_down = false;
+	set_shutting_down(server, false);
 	server->term = term;
 	init_completion(&server->start_comp);
 
@@ -2294,6 +2439,7 @@ int scoutfs_server_setup(struct super_block *sb)
 	seqcount_init(&server->volopt_seqcount);
 	mutex_init(&server->volopt_mutex);
 	INIT_WORK(&server->fence_pending_recov_work, fence_pending_recov_worker);
+	INIT_DELAYED_WORK(&server->reclaim_dwork, reclaim_worker);
 
 	server->wq = alloc_workqueue("scoutfs_server",
 				     WQ_UNBOUND | WQ_NON_REENTRANT, 0);

kmod/src/super.c

@@ -47,6 +47,7 @@
 #include "recov.h"
 #include "omap.h"
 #include "volopt.h"
+#include "fence.h"
 #include "scoutfs_trace.h"
 
 static struct dentry *scoutfs_debugfs_root;
@@ -270,6 +271,7 @@ static void scoutfs_put_super(struct super_block *sb)
 
 	scoutfs_block_destroy(sb);
 	scoutfs_destroy_triggers(sb);
+	scoutfs_fence_destroy(sb);
 	scoutfs_options_destroy(sb);
 	scoutfs_sysfs_destroy_attrs(sb, &sbi->mopts_ssa);
 	debugfs_remove(sbi->debug_root);
@@ -283,6 +285,21 @@ static void scoutfs_put_super(struct super_block *sb)
 	sb->s_fs_info = NULL;
 }
 
+/*
+ * Record that we're performing a forced unmount.  As put_super drives
+ * destruction of the filesystem we won't issue more network or storage
+ * operations because we assume that they'll hang.  Pending operations
+ * can return errors when it's possible to do so.  We may be racing with
+ * pending operations which can't be canceled.
+ */
+static void scoutfs_umount_begin(struct super_block *sb)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+
+	scoutfs_warn(sb, "forcing unmount, can return errors and lose unsynced data");
+	sbi->forced_unmount = true;
+}
+
 static const struct super_operations scoutfs_super_ops = {
 	.alloc_inode = scoutfs_alloc_inode,
 	.drop_inode = scoutfs_drop_inode,
@@ -292,6 +309,7 @@ static const struct super_operations scoutfs_super_ops = {
 	.statfs = scoutfs_statfs,
 	.show_options = scoutfs_show_options,
 	.put_super = scoutfs_put_super,
+	.umount_begin = scoutfs_umount_begin,
 };
 
 /*
@@ -590,6 +608,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 	      scoutfs_sysfs_create_attrs(sb, &sbi->mopts_ssa,
 				mount_options_attrs, "mount_options") ?:
 	      scoutfs_setup_triggers(sb) ?:
+	      scoutfs_fence_setup(sb) ?:
 	      scoutfs_block_setup(sb) ?:
 	      scoutfs_forest_setup(sb) ?:
 	      scoutfs_item_setup(sb) ?:

kmod/src/super.h

@@ -29,6 +29,7 @@ struct srch_info;
 struct recov_info;
 struct omap_info;
 struct volopt_info;
+struct fence_info;
 
 struct scoutfs_sb_info {
 	struct super_block *sb;
@@ -54,6 +55,7 @@ struct scoutfs_sb_info {
 	struct omap_info *omap_info;
 	struct volopt_info *volopt_info;
 	struct item_cache_info *item_cache_info;
+	struct fence_info *fence_info;
 
 	wait_queue_head_t trans_hold_wq;
 	struct task_struct *trans_task;
@@ -87,6 +89,8 @@ struct scoutfs_sb_info {
 
 	struct dentry *debug_root;
 
+	bool forced_unmount;
+
 	unsigned long corruption_messages_once[SC_NR_LONGS];
 };
 
@@ -107,6 +111,13 @@ static inline bool SCOUTFS_IS_META_BDEV(struct scoutfs_super_block *super_block)
 
 #define SCOUTFS_META_BDEV_MODE (FMODE_READ | FMODE_WRITE | FMODE_EXCL)
 
+static inline bool scoutfs_forcing_unmount(struct super_block *sb)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+
+	return sbi->forced_unmount;
+}
+
 /*
  * A small string embedded in messages that's used to identify a
  * specific mount.  It's the three most significant bytes of the fsid

kmod/src/sysfs.c

@@ -131,9 +131,10 @@ void scoutfs_sysfs_init_attrs(struct super_block *sb,
  *  If this returns success then the file will be visible and show can
  *  be called until unmount.
  */
-int scoutfs_sysfs_create_attrs(struct super_block *sb,
-			       struct scoutfs_sysfs_attrs *ssa,
-			       struct attribute **attrs, char *fmt, ...)
+int scoutfs_sysfs_create_attrs_parent(struct super_block *sb,
+				      struct kobject *parent,
+				      struct scoutfs_sysfs_attrs *ssa,
+				      struct attribute **attrs, char *fmt, ...)
 {
 	va_list args;
 	size_t name_len;
@@ -174,8 +175,8 @@ int scoutfs_sysfs_create_attrs(struct super_block *sb,
 		goto out;
 	}
 
-	ret = kobject_init_and_add(&ssa->kobj, &ssa->ktype,
-				   scoutfs_sysfs_sb_dir(sb), "%s", ssa->name);
+	ret = kobject_init_and_add(&ssa->kobj, &ssa->ktype, parent,
+				   "%s", ssa->name);
 out:
 	if (ret) {
 		kfree(ssa->name);

kmod/src/sysfs.h

@@ -10,6 +10,8 @@
 
 #define SCOUTFS_ATTR_RO(_name)						\
         static struct kobj_attribute scoutfs_attr_##_name = __ATTR_RO(_name)
+#define SCOUTFS_ATTR_RW(_name)						\
+        static struct kobj_attribute scoutfs_attr_##_name = __ATTR_RW(_name)
 
 #define SCOUTFS_ATTR_PTR(_name)						\
         &scoutfs_attr_##_name.attr
@@ -34,9 +36,14 @@ struct scoutfs_sysfs_attrs {
 
 void scoutfs_sysfs_init_attrs(struct super_block *sb,
 			      struct scoutfs_sysfs_attrs *ssa);
-int scoutfs_sysfs_create_attrs(struct super_block *sb,
-			       struct scoutfs_sysfs_attrs *ssa,
-			       struct attribute **attrs, char *fmt, ...);
+int scoutfs_sysfs_create_attrs_parent(struct super_block *sb,
+				      struct kobject *parent,
+				      struct scoutfs_sysfs_attrs *ssa,
+				      struct attribute **attrs, char *fmt, ...);
+#define scoutfs_sysfs_create_attrs(sb, ssa, attrs, fmt, args...)	\
+	scoutfs_sysfs_create_attrs_parent(sb, scoutfs_sysfs_sb_dir(sb),	\
+					  ssa, attrs, fmt, ##args)
+
 void scoutfs_sysfs_destroy_attrs(struct super_block *sb,
 				 struct scoutfs_sysfs_attrs *ssa);
 

kmod/src/trans.c

@@ -185,6 +185,11 @@ void scoutfs_trans_write_func(struct work_struct *work)
 
 	wait_event(sbi->trans_hold_wq, drained_holders(tri));
 
+	if (scoutfs_forcing_unmount(sb)) {
+		ret = -EIO;
+		goto out;
+	}
+
 	trace_scoutfs_trans_write_func(sb,
 			scoutfs_block_writer_dirty_bytes(sb, &tri->wri));
 
@@ -202,7 +207,7 @@ void scoutfs_trans_write_func(struct work_struct *work)
 			if (ret < 0)
 			      s = "clean advance seq";
 		}
-		goto out;
+		goto err;
 	}
 
 	if (sbi->trans_deadline_expired)
@@ -222,11 +227,12 @@ void scoutfs_trans_write_func(struct work_struct *work)
 	      scoutfs_item_write_done(sb) ?:
 	      (s = "advance seq", scoutfs_client_advance_seq(sb, &trans_seq)) ?:
 	      (s = "get log trees", scoutfs_trans_get_log_trees(sb));
-out:
+err:
 	if (ret < 0)
 		scoutfs_err(sb, "critical transaction commit failure: %s, %d",
 			    s, ret);
 
+out:
 	spin_lock(&sbi->trans_write_lock);
 	sbi->trans_write_count++;
 	sbi->trans_write_ret = ret;

tests/.gitignore

@@ -4,3 +4,4 @@ src/dumb_setxattr
 src/handle_cat
 src/bulk_create_paths
 src/find_xattrs
+src/stage_tmpfile

tests/funcs/filter.sh

@@ -62,5 +62,15 @@ t_filter_dmesg()
 	# in debugging kernels we can slow things down a bit
 	re="$re|hrtimer: interrupt took .*"
 
+	# fencing tests force unmounts and trigger timeouts
+	re="$re|scoutfs .* forcing unmount"
+	re="$re|scoutfs .* reconnect timed out"
+	re="$re|scoutfs .* recovery timeout expired"
+	re="$re|scoutfs .* fencing previous leader"
+	re="$re|scoutfs .* reclaimed resources"
+	re="$re|scoutfs .* quorum .* error"
+	re="$re|scoutfs .* error reading quorum block"
+	re="$re|scoutfs .* error .* writing quorum block"
+
 	egrep -v "($re)" 
 }

tests/funcs/fs.sh

@@ -17,6 +17,17 @@ t_sync_seq_index()
 	t_quiet sync
 }
 
+t_mount_rid()
+{
+	local nr="${1:-0}"
+	local mnt="$(eval echo \$T_M$nr)"
+	local rid
+
+	rid=$(scoutfs statfs -s rid -p "$mnt")
+
+	echo "$rid"
+}
+
 #
 # Output the "f.$fsid.r.$rid" identifier string for the given mount
 # number, 0 is used by default if none is specified. 
@@ -132,6 +143,16 @@ t_umount()
 	eval t_quiet umount \$T_M$nr
 }
 
+t_force_umount()
+{
+	local nr="$1"
+
+	test "$nr" -lt "$T_NR_MOUNTS" || \
+		t_fail "fs nr $nr invalid"
+
+	eval t_quiet umount -f \$T_M$nr
+}
+
 #
 # Attempt to mount all the configured mounts, assuming that they're
 # not already mounted.
@@ -277,3 +298,67 @@ t_counter_diff_changed() {
 		echo "counter $which didn't change" ||
 		echo "counter $which changed"
 }
+
+#
+# See if we can find a local mount with the caller's rid.
+#
+t_rid_is_mounted() {
+	local rid="$1"
+	local fr="$1"
+
+	for fr in /sys/fs/scoutfs/*; do
+		if [ "$(cat $fr/rid)" == "$rid" ]; then
+			return 0
+		fi
+	done
+
+	return 1
+}
+
+#
+# A given mount is being fenced if any mount has a fence request pending
+# for it which hasn't finished and been removed.
+#
+t_rid_is_fencing() {
+	local rid="$1"
+	local fr
+
+	for fr in /sys/fs/scoutfs/*; do
+		if [ -d "$fr/fence/$rid" ]; then
+			return 0
+		fi
+	done
+
+	return 1
+}
+
+#
+# Wait until the mount identified by the first rid arg is not in any
+# states specified by the remaining state description word args.
+#
+t_wait_if_rid_is() {
+	local rid="$1"
+
+	while ( [[ $* =~ mounted ]] && t_rid_is_mounted $rid ) ||
+	      ( [[ $* =~ fencing ]] && t_rid_is_fencing $rid ) ; do
+		sleep .5
+	done
+}
+
+#
+# Wait until any mount identifies itself as the elected leader.  We can
+# be waiting while tests mount and unmount so mounts may not be mounted
+# at the test's expected mount points.
+#
+t_wait_for_leader() {
+	local i
+
+	while sleep .25; do
+		for i in $(t_fs_nrs); do
+			local ldr="$(t_sysfs_path $i 2>/dev/null)/quorum/is_leader"
+			if [ "$(cat $ldr 2>/dev/null)" == "1" ]; then
+				return
+			fi
+		done
+	done
+}

tests/golden/fence-and-reclaim

@@ -0,0 +1,5 @@
+== make sure all mounts can see each other
+== force unmount one client, connection timeout, fence nop, mount
+== force unmount all non-server, connection timeout, fence nop, mount
+== force unmount server, quorum elects new leader, fence nop, mount
+== force unmount everything, new server fences all previous

tests/run-tests.sh

@@ -18,10 +18,15 @@ die() {
 	exit 1
 }
 
+timestamp()
+{
+	date '+%F %T.%N'
+}
+
 # output a message with a timestamp to the run.log
 log()
 {
-	echo "[$(date '+%F %T.%N')] $*" >> "$T_RESULTS/run.log"
+	echo "[$(timestamp)] $*" >> "$T_RESULTS/run.log"
 }
 
 # run a logged command, exiting if it fails
@@ -367,6 +372,39 @@ cmd cat /sys/kernel/debug/tracing/set_event
 cmd grep .  /sys/kernel/debug/tracing/options/trace_printk \
 	    /proc/sys/kernel/ftrace_dump_on_oops
 
+#
+# Build a fenced config that runs scripts out of the repository rather
+# than the default system directory
+#
+conf="$T_RESULTS/scoutfs-fencd.conf"
+cat > $conf << EOF
+SCOUTFS_FENCED_DELAY=1
+SCOUTFS_FENCED_RUN=$T_UTILS/fenced/local-force-unmount
+SCOUTFS_FENCED_RUN_ARGS=""
+EOF
+export SCOUTFS_FENCED_CONFIG_FILE="$conf"
+
+#
+# Run the agent in the background, log its output, an kill it if we
+# exit
+#
+fenced_log()
+{
+	echo "[$(timestamp)] $*" >> "$T_RESULTS/fenced.stdout.log"
+}
+fenced_pid=""
+kill_fenced()
+{
+	if test -n "$fenced_pid" -a -d "/proc/$fenced_pid" ; then
+		fenced_log "killing fenced pid $fenced_pid"
+		kill "$fenced_pid"
+	fi
+}
+trap kill_fenced EXIT
+$T_UTILS/fenced/scoutfs-fenced > "$T_RESULTS/fenced.stdout.log" 2> "$T_RESULTS/fenced.stderr.log" &
+fenced_pid=$!
+fenced_log "started fenced pid $fenced_pid in the background"
+
 #
 # mount concurrently so that a quorum is present to elect the leader and
 # start a server.

tests/sequence

@@ -28,6 +28,7 @@ lock-conflicting-batch-commit.sh
 cross-mount-data-free.sh
 persistent-item-vers.sh
 setup-error-teardown.sh
+fence-and-reclaim.sh
 mount-unmount-race.sh
 createmany-parallel-mounts.sh
 archive-light-cycle.sh

tests/tests/fence-and-reclaim.sh

@@ -0,0 +1,127 @@
+#
+# Fence nodes and reclaim their resources.
+#
+
+t_require_commands sleep touch grep sync scoutfs
+t_require_mounts 2
+
+#
+# Make sure that all mounts can read the results of a write from each
+# mount.  And make sure that the greatest of all the written seqs is
+# visible after the writes were commited by remote reads.
+#
+check_read_write()
+{
+	local expected
+	local greatest=0
+	local seq
+	local path
+	local saw
+	local w
+	local r
+
+	for w in $(t_fs_nrs); do
+		expected="$w wrote at $(date --rfc-3339=ns)"
+		eval path="\$T_D${w}/written"
+		echo "$expected" > "$path"
+
+		seq=$(scoutfs stat -s meta_seq $path)
+		if [ "$seq" -gt "$greatest" ]; then
+			greatest=$seq
+		fi
+
+		for r in $(t_fs_nrs); do
+			eval path="\$T_D${r}/written"
+			saw=$(cat "$path")
+			if [ "$saw" != "$expected" ]; then
+				echo "mount $r read '$saw' after mount $w wrote '$expected'"
+			fi
+		done
+	done
+
+	seq=$(scoutfs statfs -s committed_seq -p $T_D0)
+	if [ "$seq" -lt "$greatest" ]; then
+		echo "committed_seq $seq less than greatest $greatest"
+	fi
+}
+
+echo "== make sure all mounts can see each other"
+check_read_write
+
+echo "== force unmount one client, connection timeout, fence nop, mount"
+cl=$(t_first_client_nr)
+sv=$(t_server_nr)
+rid=$(t_mount_rid $cl)
+echo "cl $cl sv $sv rid $rid" >> "$T_TMP.log"
+sync
+t_force_umount $cl
+# wait for client reconnection to timeout
+while grep -q $rid $(t_debugfs_path $sv)/connections; do
+	sleep .5
+done
+while t_rid_is_fencing $rid; do
+	sleep .5
+done
+t_mount $cl
+check_read_write
+
+echo "== force unmount all non-server, connection timeout, fence nop, mount"
+sv=$(t_server_nr)
+pattern="nonsense"
+sync
+for cl in $(t_fs_nrs); do
+	if [ $cl == $sv ]; then
+		continue;
+	fi
+
+	rid=$(t_mount_rid $cl)
+	pattern="$pattern|$rid"
+	echo "cl $cl sv $sv rid $rid" >> "$T_TMP.log"
+
+	t_force_umount $cl
+done
+
+# wait for all client reconnections to timeout
+while egrep -q "($pattern)" $(t_debugfs_path $sv)/connections; do
+	sleep .5
+done
+# wait for all fence requests to complete
+while test -d $(echo /sys/fs/scoutfs/*/fence/* | cut -d " " -f 1); do
+	sleep .5
+done
+# remount all the clients
+for cl in $(t_fs_nrs); do
+	if [ $cl == $sv ]; then
+		continue;
+	fi
+	t_mount $cl
+done
+check_read_write
+
+echo "== force unmount server, quorum elects new leader, fence nop, mount"
+sv=$(t_server_nr)
+rid=$(t_mount_rid $sv)
+echo "sv $sv rid $rid" >> "$T_TMP.log"
+sync
+t_force_umount $sv
+t_wait_for_leader
+# wait until new server is done fencing unmounted leader rid
+while t_rid_is_fencing $rid; do
+	sleep .5
+done
+t_mount $sv
+check_read_write
+
+echo "== force unmount everything, new server fences all previous"
+sync
+for nr in $(t_fs_nrs); do
+	t_force_umount $nr
+done
+t_mount_all
+# wait for all fence requests to complete
+while test -d $(echo /sys/fs/scoutfs/*/fence/* | cut -d " " -f 1); do
+	sleep .5
+done
+check_read_write
+
+t_pass

utils/fenced/local-force-unmount

@@ -0,0 +1,35 @@
+#!/usr/bin/bash
+
+echo_fail() {
+	echo "$@" > /dev/stderr
+	exit 1
+}
+
+rid="$SCOUTFS_FENCED_REQ_RID"
+
+#
+# Look for a local mount with the rid to fence.  Typically we'll at
+# least find the mount with the server that requested the fence that
+# we're processing.   But it's possible that mounts are unmounted
+# before, or while, we're running.
+#
+mnts=$(findmnt -l -n -t scoutfs -o TARGET) || \
+	echo_fail "findmnt -t scoutfs failed" > /dev/stderr
+
+for mnt in $mnts; do
+	mnt_rid=$(scoutfs statfs -p "$mnt" -s rid) || \
+		echo_fail "scoutfs statfs $mnt failed"
+
+	if [ "$mnt_rid" == "$rid" ]; then
+		umount -f "$mnt" || \
+			echo_fail "umout -f $mnt"
+
+		exit 0
+	fi
+done
+
+#
+# If the mount doesn't exist on this host then it can't access the
+# devices by definition and can be considered fenced.
+#
+exit 0

utils/fenced/scoutfs-fenced

@@ -0,0 +1,94 @@
+#!/usr/bin/bash
+
+message_output()
+{
+	printf "[%s] %s\n" "$(date '+%F %T.%N')" "$@"
+}
+
+error_message()
+{
+	message_output "$@" >> /dev/stderr
+}
+
+error_exit()
+{
+	error_message "$@, exiting"
+	exit 1
+}
+
+log_message()
+{
+	message_output "$@" >> /dev/stdout
+}
+
+# restart if we catch hup to re-read the config
+hup_restart()
+{
+	log_message "caught SIGHUP, restarting"
+	exec "$@"
+}
+trap hup_restart SIGHUP
+
+# defaults
+SCOUTFS_FENCED_CONFIG_FILE=${SCOUTFS_FENCED_CONFIG_FILE:-/etc/scoutfs/scoutfs-fenced.conf}
+SCOUTFS_FENCED_DELAY=2
+#SCOUTFS_FENCED_RUN
+#SCOUTFS_FENCED_RUN_ARGS
+
+test -n "$SCOUTFS_FENCED_CONFIG_FILE" || \
+	error_exit "SCOUTFS_FENCED_CONFIG_FILE isn't set"
+test -r "$SCOUTFS_FENCED_CONFIG_FILE" || \
+	error_exit "SCOUTFS_FENCED_CONFIG_FILE isn't readable file"
+
+log_message "reading config file $SCOUTFS_FENCED_CONFIG_FILE"
+
+. "$SCOUTFS_FENCED_CONFIG_FILE" || \
+	error_exit "error sourcing $SCOUTFS_FENCED_CONFIG_FILE as bash script"
+
+for conf in "${!SCOUTFS_FENCED_@}"; do
+	log_message "    config var $conf=${!conf}"
+done
+
+test -n "$SCOUTFS_FENCED_RUN" || \
+	error_exit "SCOUTFS_FENCED_RUN must be set"
+test -x "$SCOUTFS_FENCED_RUN" || \
+	error_exit "SCOUTFS_FENCED_RUN '$SCOUTFS_FENCED_RUN' isn't executable"
+
+#
+# main loop watching for fence request across all filesystems 
+#
+
+while sleep $SCOUTFS_FENCED_DELAY; do
+	for fence in /sys/fs/scoutfs/*/fence/*; do
+		# catches unmatched regex when no dirs
+		if [ ! -d "$fence" ]; then
+			continue
+		fi
+
+		# skip requests that have been handled
+		if [ $(cat "$fence/fenced") == 1 -o $(cat "$fence/error") == 1 ]; then
+			continue
+		fi
+
+		srv=$(basename $(dirname $(dirname $fence)))
+		rid="$(cat $fence/rid)"
+		ip="$(cat $fence/ipv4_addr)"
+		reason="$(cat $fence/reason)"
+
+		log_message "server $srv fencing rid $rid at IP $ip for $reason"
+
+		# export _REQ_ vars for run to use
+		export SCOUTFS_FENCED_REQ_RID="$rid"
+		export SCOUTFS_FENCED_REQ_IP="$ip"
+
+		$run $SCOUTFS_FENCED_RUN_ARGS
+		rc=$?
+		if [ "$rc" != 0 ]; then
+			log_message "server $srv fencing rid $rid saw error status $rc from $run"
+			echo 1 > "$fence/error"
+			continue
+		fi
+
+		echo 1 > "$fence/fenced"
+	done
+done

utils/man/scoutfs-fenced.8

@@ -0,0 +1,66 @@
+.TH scoutfs-fenced 8
+.SH NAME
+scoutfs-fenced \- scoutfs fence request monitoring and dispatch daemon
+.SH DESCRIPTION
+The
+.B scoutfs-fenced
+daemon runs on hosts with mounts that are configured as quorum members
+and could create fence requests.  It watches sysfs directories of
+mounted scoutfs volumes for the directories store requests
+to fence a mount.
+
+.SH ENVIRONMENT
+scoutfs-fenced reads the
+.I SCOUTFS_FENCED_CONFIG_FILE
+environment variable for the path to the config file that contains its
+configuration.  The file must be readable and is sourced as a bash
+script and is expected to set the following configuration variables.
+
+.SH CONFIGURATION
+
+.TP
+.B SCOUTFS_FENCED_DELAY
+The number of seconds to wait beteween checking for fence request
+directories in the sysfs directories of all mounts on the host.
+
+.TP
+.B SCOUTFS_FENCED_RUN
+The path to the command to execute for each fence request.  The file at
+the path must be executable.
+
+.TP
+.B SCOUTFS_FENCED_RUN_ARGS
+The arguments that are unconditionally passed through to the run
+command.
+
+.SH DAEMONIZING AND LOGGING
+
+scoutfs-fenced runs in the foreground and writes to stderr and stdout.
+Disconnecting it from parents and redirecting its output are the
+responsibility of the host environment.
+
+.SH RUN COMMAND INTERFACE
+
+scoutfs-fenced sets enviroment variables for the run command with
+information about the mount that must be fenced:
+
+.TP
+.B SCOUTFS_FENCED_REQ_RID
+The RID of the mount to be fenced.
+.TP
+.B SCOUTFS_FENCED_REQ_IP
+The dotted quad IPv4 address of the last connection from the mount.
+
+.RE
+The return status of the run command indicates if the mount was
+fenced, or not.  If the mount was successfully fenced then the command
+should return a 0 success status.  If the run command returns a non-zero
+failure status then the request will be set as errored and the server
+will shut down.  The next server that starts will create another fence
+request for the mount.
+
+.SH SEE ALSO
+.BR scoutfs (5),
+
+.SH AUTHORS
+Zach Brown <zab@versity.com>

utils/man/scoutfs.5

@@ -1,6 +1,6 @@
 .TH scoutfs 5
 .SH NAME
-scoutfs \- overview and mount options for the scoutfs filesystem
+scoutfs \- high level overview of the scoutfs filesystem
 .SH DESCRIPTION
 A scoutfs filesystem is stored on two block devices.  Multiple mounts of
 the filesystem are supported between hosts that share access to the
@@ -68,7 +68,82 @@ in that zone.  When the mount is not in a zone, or its zone has no more
 free extents, the server will try and find free extents in a zone that
 no other mount currently occupies.  The result is to try and produce
 write streams where only one mount is writing into each zone.
-.SH FURTHER READING
+.SH FENCING
+.B scoutfs
+mounts coordinate exclusive access to shared resources through
+comminication with the mount that was elected leader.
+A mount can malfunction and stop participating at which point it needs
+to be safely isolated ("fenced off") from shared resources before other mounts can
+have their turn at exclusive access.
+.sp
+Only the elected leader can fence mounts.  As the leader decides that a
+mount must be fenced, typically by timeouts expiring without
+comminication from the mount, it creates a fence request.   Fence
+requests are visible as directories in the leader mount's sysfs
+directory.  The fence request directory is named for the RID of the
+mount being fenced.  The directory contains the following files:
+
+.RS
+.TP
+.B elapsec_secs
+Reading this file gives the number of seconds that have passed since
+this fence request was created.
+.TP
+.B error
+This file contains 0 when the fence request is created.  Userspace
+fencing agents write 1 into this file if they are unable to fence the
+mount.  The volume can not make progress until the mount is fenced so
+this will cause the server to stop and another mount will be elected
+leader.
+.TP
+.B fenced
+This file contains 0 when the fence request is created.  Userspace
+fencing agents write 1 into this file once the mount has been fenced.
+.TP
+.B ipv4_addr
+This file contains the dotted quad IPv4 peer address of the last
+connected socket from the mount.  Userspace fencing agents can use this
+to find the host that contains the mount.
+.TP
+.B reason
+This file contains a text string that indicates the reason that the
+mount is being fenced:
+
+.B client_recovery
+- During startup the server found persistent items recording the presence
+of a mount that didn't reconnect to the server in time.
+.sp
+.B client_reconnect
+- A mount disconnected from the server and didn't reconnect in time.
+.sp
+.B quorum_block_leader
+- As a leader was elected it read persistent blocks that indicated that
+a previous leader had not shut down and cleared their quorum block.
+.TP
+.B rid
+This file contains the hex string of the RID of the mount to be fenced.
+.RE
+
+The request directories enable userspace processes to gather the
+information to find the host with the mount to fence, isolate the mount
+by whatever means are appropriate (f.e. cut off network and storage
+communication, force unmount the mount, isolate storage fabric ports,
+reboot the host) and write to the
+.I fenced
+file.
+.sp
+Once the 
+.I fenced
+file is written to the server reclaims the resources
+associated with the fenced mount and resumes normal operations.
+.sp
+If the 
+.I error
+file is written to then the server cannot make forward progress and
+shuts down.  The request can similarly enter an errored state if enough
+time passes before userspace completes the request.
+ 
+.SH CORRUPTION DETECTION
 A
 .B scoutfs
 filesystem can detect corruption at runtime.  A catalog of kernel log

utils/scoutfs-utils.spec.in

@@ -54,12 +54,15 @@ cp man/*.8.gz $RPM_BUILD_ROOT%{_mandir}/man8/.
 install -m 755 -D src/scoutfs $RPM_BUILD_ROOT%{_sbindir}/scoutfs
 install -m 644 -D src/ioctl.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/ioctl.h
 install -m 644 -D src/format.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/format.h
+install -m 755 -D fenced/scoutfs-fenced $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/scoutfs-fenced
+install -m 755 -D fenced/local-force-unmount $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/local-force-unmount
 
 %files
 %defattr(644,root,root,755)
 %{_mandir}/man*/scoutfs*.gz
 %defattr(755,root,root,755)
 %{_sbindir}/scoutfs
+%{_libexecdir}/scoutfs-fenced
 
 %files -n scoutfs-devel
 %defattr(644,root,root,755)

utils/src/print.c

@@ -372,9 +372,13 @@ static int print_mounted_client_entry(struct scoutfs_key *key, void *val,
 				      unsigned val_len, void *arg)
 {
 	struct scoutfs_mounted_client_btree_val *mcv = val;
+	struct in_addr in;
 
-	printf("    rid %016llx flags 0x%x\n",
-	       le64_to_cpu(key->skmc_rid), mcv->flags);
+	memset(&in, 0, sizeof(in));
+	in.s_addr = htonl(le32_to_cpu(mcv->addr.v4.addr));
+
+	printf("    rid %016llx ipv4_addr %s flags 0x%x\n",
+	       le64_to_cpu(key->skmc_rid), inet_ntoa(in), mcv->flags);
 
 	return 0;
 }
@@ -812,16 +816,16 @@ static char *alloc_addr_str(union scoutfs_inet_addr *ia)
 
 static int print_quorum_blocks(int fd, struct scoutfs_super_block *super)
 {
-	struct print_events {
-		size_t offset;
-		char *name;
-	} events[] = {
-		OFF_NAME(write), OFF_NAME(update_term), OFF_NAME(set_leader),
-		OFF_NAME(clear_leader), OFF_NAME(fenced),
+	const static char *event_names[] = {
+		[SCOUTFS_QUORUM_EVENT_BEGIN] = "begin",
+		[SCOUTFS_QUORUM_EVENT_TERM] = "term",
+		[SCOUTFS_QUORUM_EVENT_ELECT] = "elect",
+		[SCOUTFS_QUORUM_EVENT_FENCE] = "fence",
+		[SCOUTFS_QUORUM_EVENT_STOP] = "stop",
+		[SCOUTFS_QUORUM_EVENT_END] = "end",
 	};
 	struct scoutfs_quorum_block *blk = NULL;
 	struct scoutfs_quorum_block_event *ev;
-	char *log_addr = NULL;
 	u64 blkno;
 	int ret;
 	int i;
@@ -830,6 +834,7 @@ static int print_quorum_blocks(int fd, struct scoutfs_super_block *super)
 	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
 		blkno = SCOUTFS_QUORUM_BLKNO + i;
 		free(blk);
+		blk = NULL;
 		ret = read_block(fd, blkno, SCOUTFS_BLOCK_SM_SHIFT, (void **)&blk);
 		if (ret)
 			goto out;
@@ -837,24 +842,19 @@ static int print_quorum_blocks(int fd, struct scoutfs_super_block *super)
 		printf("quorum blkno %llu (slot %llu)\n",
 		       blkno, blkno - SCOUTFS_QUORUM_BLKNO);
 		print_block_header(&blk->hdr, SCOUTFS_BLOCK_SM_SIZE);
-		printf("  term %llu random_write_mark 0x%llx flags 0x%llx\n",
-		       le64_to_cpu(blk->term),
-		       le64_to_cpu(blk->random_write_mark),
-		       le64_to_cpu(blk->flags));
 
-		for (e = 0; e < array_size(events); e++) {
-			ev = (void *)blk + events[e].offset;
+		for (e = 0; e < array_size(event_names); e++) {
+			ev = &blk->events[e];
 
-			printf("  %12s: rid %016llx ts %llu.%08u\n",
-			       events[e].name, le64_to_cpu(ev->rid),
-			       le64_to_cpu(ev->ts.sec),
-			       le32_to_cpu(ev->ts.nsec));
+			printf("  %12s: rid %016llx term %llu ts %llu.%08u\n",
+			       event_names[e], le64_to_cpu(ev->rid), le64_to_cpu(ev->term),
+			       le64_to_cpu(ev->ts.sec), le32_to_cpu(ev->ts.nsec));
 		}
 	}
 
 	ret = 0;
 out:
-	free(log_addr);
+	free(blk);
 
 	return ret;
 }