add local,ipmi,powerman fenced scripts to utils rpm

This adds the fenced scripts so that we have a place to track these and
get updates out to users. This latest version of scripts has the checks
to validate that power off succeeded and not just assume based on power
command return status.

Signed-off-by: Ben McClelland <ben.mcclelland@versity.com>

ReleaseNotes.md

@@ -1,6 +1,25 @@
 Versity ScoutFS Release Notes
 =============================
 
+---
+v1.5
+\
+*Jun 21, 2022*
+
+* **Fix persistent error during server startup**
+\
+  Fixed a case where the server would always hit a consistent error on
+  seartup, preventing the system from mounting.  This required a rare
+  but valid state across the clients.
+
+* **Fix a client hang that would lead to fencing**
+\
+  The client module's use of in-kernel networking was missing annotation
+  that could lead to communication hanging.  The server would fence the
+  client when it stopped communicating.  This could be identified by the
+  server fencing a client after it disconnected with no attempt by the
+  client to reconnect.
+
 ---
 v1.4
 \

kmod/src/alloc.c

@@ -84,6 +84,21 @@ static u64 smallest_order_length(u64 len)
 	return 1ULL << (free_extent_order(len) * 3);
 }
 
+/*
+ * An extent modification dirties three distinct leaves of an allocator
+ * btree as it adds and removes the blkno and size sorted items for the
+ * old and new lengths of the extent.  Dirtying the paths to these
+ * leaves can grow the tree and grow/shrink neighbours at each level.
+ * We over-estimate the number of blocks allocated and freed (the paths
+ * share a root, growth doesn't free) to err on the simpler and safer
+ * side.  The overhead is minimal given the relatively large list blocks
+ * and relatively short allocator trees.
+ */
+static u32 extent_mod_blocks(u32 height)
+{
+	return ((1 + height) * 2) * 3;
+}
+
 /*
  * Free extents don't have flags and are stored in two indexes sorted by
  * block location and by length order, largest first.  The location key
@@ -877,6 +892,14 @@ static int find_zone_extent(struct super_block *sb, struct scoutfs_alloc_root *r
  * -ENOENT is returned if we run out of extents in the source tree
  * before moving the total.
  *
+ * If meta_reserved is non-zero then -EINPROGRESS can be returned if the
+ * current meta allocator's avail blocks or room for freed blocks would
+ * have fallen under the reserved amount.  The could have been
+ * successfully dirtied in this case but the number of blocks moved is
+ * not returned.  The caller is expected to deal with the partial
+ * progress by commiting the dirty trees and examining the resulting
+ * modified trees to see if they need to continue moving extents.
+ *
  * The caller can specify that extents in the source tree should first
  * be found based on their zone bitmaps.  We'll first try to find
  * extents in the exclusive zones, then vacant zones, and then we'll
@@ -891,7 +914,7 @@ int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc,
 		       struct scoutfs_block_writer *wri,
 		       struct scoutfs_alloc_root *dst,
 		       struct scoutfs_alloc_root *src, u64 total,
-		       __le64 *exclusive, __le64 *vacant, u64 zone_blocks)
+		       __le64 *exclusive, __le64 *vacant, u64 zone_blocks, u64 meta_reserved)
 {
 	struct alloc_ext_args args = {
 		.alloc = alloc,
@@ -941,6 +964,14 @@ int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc,
 		if (ret < 0)
 			break;
 
+		if (meta_reserved != 0 &&
+		    scoutfs_alloc_meta_low(sb, alloc, meta_reserved +
+					   extent_mod_blocks(src->root.height) +
+					   extent_mod_blocks(dst->root.height))) {
+			ret = -EINPROGRESS;
+			break;
+		}
+
 		/* searching set start/len, finish initializing alloced extent */
 		ext.map = found.map ? ext.start - found.start + found.map : 0;
 		ext.flags = found.flags;
@@ -1065,15 +1096,6 @@ out:
  * than completely exhausting the avail list or overflowing the freed
  * list.
  *
- * An extent modification dirties three distinct leaves of an allocator
- * btree as it adds and removes the blkno and size sorted items for the
- * old and new lengths of the extent.  Dirtying the paths to these
- * leaves can grow the tree and grow/shrink neighbours at each level.
- * We over-estimate the number of blocks allocated and freed (the paths
- * share a root, growth doesn't free) to err on the simpler and safer
- * side.  The overhead is minimal given the relatively large list blocks
- * and relatively short allocator trees.
- *
  * The caller tells us how many extents they're about to modify and how
  * many other additional blocks they may cow manually.  And finally, the
  * caller could be the first to dirty the avail and freed blocks in the
@@ -1082,7 +1104,7 @@ out:
 static bool list_has_blocks(struct super_block *sb, struct scoutfs_alloc *alloc,
 			    struct scoutfs_alloc_root *root, u32 extents, u32 addl_blocks)
 {
-	u32 tree_blocks = (((1 + root->root.height) * 2) * 3) * extents;
+	u32 tree_blocks = extent_mod_blocks(root->root.height) * extents;
 	u32 most = 1 + tree_blocks + addl_blocks;
 
 	if (le32_to_cpu(alloc->avail.first_nr) < most) {

kmod/src/alloc.h

@@ -131,7 +131,7 @@ int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc,
 		       struct scoutfs_block_writer *wri,
 		       struct scoutfs_alloc_root *dst,
 		       struct scoutfs_alloc_root *src, u64 total,
-		       __le64 *exclusive, __le64 *vacant, u64 zone_blocks);
+		       __le64 *exclusive, __le64 *vacant, u64 zone_blocks, u64 meta_reserved);
 int scoutfs_alloc_insert(struct super_block *sb, struct scoutfs_alloc *alloc,
 			 struct scoutfs_block_writer *wri, struct scoutfs_alloc_root *root,
 			 u64 start, u64 len);

kmod/src/net.c

@@ -991,6 +991,8 @@ static void scoutfs_net_listen_worker(struct work_struct *work)
 		if (ret < 0)
 			break;
 
+		acc_sock->sk->sk_allocation = GFP_NOFS;
+
 		/* inherit accepted request funcs from listening conn */
 		acc_conn = scoutfs_net_alloc_conn(sb, conn->notify_up,
 						  conn->notify_down,
@@ -1053,6 +1055,8 @@ static void scoutfs_net_connect_worker(struct work_struct *work)
 	if (ret)
 		goto out;
 
+	sock->sk->sk_allocation = GFP_NOFS;
+
 	/* caller specified connect timeout */
 	tv.tv_sec = conn->connect_timeout_ms / MSEC_PER_SEC;
 	tv.tv_usec = (conn->connect_timeout_ms % MSEC_PER_SEC) * USEC_PER_MSEC;
@@ -1450,6 +1454,8 @@ int scoutfs_net_bind(struct super_block *sb,
 	if (ret)
 		goto out;
 
+	sock->sk->sk_allocation = GFP_NOFS;
+
 	optval = 1;
 	ret = kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR,
 				(char *)&optval, sizeof(optval));

kmod/src/server.c

@@ -689,23 +689,18 @@ static int alloc_move_refill_zoned(struct super_block *sb, struct scoutfs_alloc_
 	return scoutfs_alloc_move(sb, &server->alloc, &server->wri, dst, src,
 				  min(target - le64_to_cpu(dst->total_len),
 				      le64_to_cpu(src->total_len)),
-				  exclusive, vacant, zone_blocks);
-}
-
-static inline int alloc_move_refill(struct super_block *sb, struct scoutfs_alloc_root *dst,
-				    struct scoutfs_alloc_root *src, u64 lo, u64 target)
-{
-	return alloc_move_refill_zoned(sb, dst, src, lo, target, NULL, NULL, 0);
+				  exclusive, vacant, zone_blocks, 0);
 }
 
 static int alloc_move_empty(struct super_block *sb,
 			    struct scoutfs_alloc_root *dst,
-			    struct scoutfs_alloc_root *src)
+			    struct scoutfs_alloc_root *src, u64 meta_reserved)
 {
 	DECLARE_SERVER_INFO(sb, server);
 
 	return scoutfs_alloc_move(sb, &server->alloc, &server->wri,
-				  dst, src, le64_to_cpu(src->total_len), NULL, NULL, 0);
+				  dst, src, le64_to_cpu(src->total_len), NULL, NULL, 0,
+				  meta_reserved);
 }
 
 /*
@@ -1272,6 +1267,7 @@ static int server_get_log_trees(struct super_block *sb,
 	char *err_str = NULL;
 	u64 nr;
 	int ret;
+	int err;
 
 	if (arg_len != 0) {
 		ret = -EINVAL;
@@ -1315,16 +1311,27 @@ static int server_get_log_trees(struct super_block *sb,
 		goto unlock;
 	}
 
+	if (ret != -ENOENT) {
+		/* need to sync lt with respect to changes in other structures */
+		scoutfs_key_init_log_trees(&key, le64_to_cpu(lt.rid), le64_to_cpu(lt.nr));
+		ret = scoutfs_btree_dirty(sb, &server->alloc, &server->wri,
+					  &super->logs_root, &key);
+		if (ret < 0) {
+			err_str = "dirtying lt btree key";
+			goto unlock;
+		}
+	}
+
 	/* drops and re-acquires the mutex and commit if it has to wait */
 	ret = finalize_and_start_log_merge(sb, &lt, rid, &hold);
 	if (ret < 0)
-		goto unlock;
+		goto update;
 
 	if (get_volopt_val(server, SCOUTFS_VOLOPT_DATA_ALLOC_ZONE_BLOCKS_NR, &data_zone_blocks)) {
 		ret = get_data_alloc_zone_bits(sb, rid, exclusive, vacant, data_zone_blocks);
 		if (ret < 0) {
 			err_str = "getting alloc zone bits";
-			goto unlock;
+			goto update;
 		}
 	} else {
 		data_zone_blocks = 0;
@@ -1341,13 +1348,13 @@ static int server_get_log_trees(struct super_block *sb,
 					&lt.meta_freed);
 	if (ret < 0) {
 		err_str = "splicing committed meta_freed";
-		goto unlock;
+		goto update;
 	}
 
-	ret = alloc_move_empty(sb, &super->data_alloc, &lt.data_freed);
+	ret = alloc_move_empty(sb, &super->data_alloc, &lt.data_freed, 0);
 	if (ret < 0) {
 		err_str = "emptying committed data_freed";
-		goto unlock;
+		goto update;
 	}
 
 	ret = scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri,
@@ -1356,7 +1363,7 @@ static int server_get_log_trees(struct super_block *sb,
 				      SCOUTFS_SERVER_META_FILL_TARGET);
 	if (ret < 0) {
 		err_str = "filling meta_avail";
-		goto unlock;
+		goto update;
 	}
 
 	if (le64_to_cpu(server->meta_avail->total_len) <= scoutfs_server_reserved_meta_blocks(sb))
@@ -1369,7 +1376,7 @@ static int server_get_log_trees(struct super_block *sb,
 				      exclusive, vacant, data_zone_blocks);
 	if (ret < 0) {
 		err_str = "refilling data_avail";
-		goto unlock;
+		goto update;
 	}
 
 	if (le64_to_cpu(lt.data_avail.total_len) < SCOUTFS_SERVER_DATA_FILL_LO)
@@ -1389,7 +1396,7 @@ static int server_get_log_trees(struct super_block *sb,
 		if (ret < 0) {
 			zero_data_alloc_zone_bits(&lt);
 			err_str = "setting data_avail zone bits";
-			goto unlock;
+			goto update;
 		}
 
 		lt.data_alloc_zone_blocks = cpu_to_le64(data_zone_blocks);
@@ -1398,13 +1405,18 @@ static int server_get_log_trees(struct super_block *sb,
 	/* give the transaction a new seq (must have been ==) */
 	lt.get_trans_seq = cpu_to_le64(scoutfs_server_next_seq(sb));
 
+update:
 	/* update client's log tree's item */
-	scoutfs_key_init_log_trees(&key, le64_to_cpu(lt.rid),
-				   le64_to_cpu(lt.nr));
-	ret = scoutfs_btree_force(sb, &server->alloc, &server->wri,
+	scoutfs_key_init_log_trees(&key, le64_to_cpu(lt.rid), le64_to_cpu(lt.nr));
+	err = scoutfs_btree_force(sb, &server->alloc, &server->wri,
 				  &super->logs_root, &key, &lt, sizeof(lt));
-	if (ret < 0)
-		err_str = "updating log trees";
+	BUG_ON(err < 0); /* can duplicate extents.. move dst in super, still in in lt src */
+	if (err < 0) {
+		if (ret == 0) {
+			ret = err;
+			err_str = "updating log trees";
+		}
+	}
 
 unlock:
 	if (unlock_alloc)
@@ -1544,9 +1556,11 @@ static int server_get_roots(struct super_block *sb,
  * read and we finalize the tree so that it will be merged.  We reclaim
  * all the allocator items.
  *
- * The caller holds the commit rwsem which means we do all this work in
- * one server commit.  We'll need to keep the total amount of blocks in
- * trees in check.
+ * The caller holds the commit rwsem which means we have to do our work
+ * in one commit.  The alocator btrees can be very large and very
+ * fragmented.  We return -EINPROGRESS if we couldn't fully reclaim the
+ * allocators in one commit.   The caller should apply the current
+ * commit and call again in a new commit.
  *
  * By the time we're evicting a client they've either synced their data
  * or have been forcefully removed.  The free blocks in the allocator
@@ -1606,9 +1620,9 @@ static int reclaim_open_log_tree(struct super_block *sb, u64 rid)
 	}
 
 	/*
-	 * All of these can return errors after having modified the
-	 * allocator trees.  We have to try and update the roots in the
-	 * log item.
+	 * All of these can return errors, perhaps indicating successful
+	 * partial progress, after having modified the allocator trees.
+	 * We always have to update the roots in the log item.
 	 */
 	mutex_lock(&server->alloc_mutex);
 	ret = (err_str = "splice meta_freed to other_freed",
@@ -1618,18 +1632,21 @@ static int reclaim_open_log_tree(struct super_block *sb, u64 rid)
 	       scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, server->other_freed,
 					 &lt.meta_avail)) ?:
 	      (err_str = "empty data_avail",
-	       alloc_move_empty(sb, &super->data_alloc, &lt.data_avail)) ?:
+	       alloc_move_empty(sb, &super->data_alloc, &lt.data_avail, 100)) ?:
 	      (err_str = "empty data_freed",
-	       alloc_move_empty(sb, &super->data_alloc, &lt.data_freed));
+	       alloc_move_empty(sb, &super->data_alloc, &lt.data_freed, 100));
 	mutex_unlock(&server->alloc_mutex);
 
-	/* the transaction is no longer open */
-	lt.commit_trans_seq = lt.get_trans_seq;
+	/* only finalize, allowing merging, once the allocators are fully freed */
+	if (ret == 0) {
+		/* the transaction is no longer open */
+		lt.commit_trans_seq = lt.get_trans_seq;
 
-	/* the mount is no longer writing to the zones */
-	zero_data_alloc_zone_bits(&lt);
-	le64_add_cpu(&lt.flags, SCOUTFS_LOG_TREES_FINALIZED);
-	lt.finalize_seq = cpu_to_le64(scoutfs_server_next_seq(sb));
+		/* the mount is no longer writing to the zones */
+		zero_data_alloc_zone_bits(&lt);
+		le64_add_cpu(&lt.flags, SCOUTFS_LOG_TREES_FINALIZED);
+		lt.finalize_seq = cpu_to_le64(scoutfs_server_next_seq(sb));
+	}
 
 	err = scoutfs_btree_update(sb, &server->alloc, &server->wri,
 				  &super->logs_root, &key, &lt, sizeof(lt));
@@ -1638,7 +1655,7 @@ static int reclaim_open_log_tree(struct super_block *sb, u64 rid)
 out:
 	mutex_unlock(&server->logs_mutex);
 
-	if (ret < 0)
+	if (ret < 0 && ret != -EINPROGRESS)
 		scoutfs_err(sb, "server error %d reclaiming log trees for rid %016llx: %s",
 			    ret, rid, err_str);
 
@@ -3536,26 +3553,37 @@ struct farewell_request {
  * Reclaim all the resources for a mount which has gone away.  It's sent
  * us a farewell promising to leave or we actively fenced it.
  *
- * It's safe to call this multiple times for a given rid.  Each
- * individual action knows to recognize that it's already been performed
- * and return success.
+ * This can be called multiple times across different servers for
+ * different reclaim attempts.  The existence of the mounted_client item
+ * triggers reclaim and must be deleted last.  Each step knows that it
+ * can be called multiple times and safely recognizes that its work
+ * might have already been done.
+ *
+ * Some steps (reclaiming large fragmented allocators) may need multiple
+ * calls to complete.  They return -EINPROGRESS which tells us to apply
+ * the server commit and retry.
  */
 static int reclaim_rid(struct super_block *sb, u64 rid)
 {
 	COMMIT_HOLD(hold);
 	int ret;
+	int err;
 
-	server_hold_commit(sb, &hold);
+	do {
+		server_hold_commit(sb, &hold);
 
-	/* delete mounted client last, recovery looks for it */
-	ret = scoutfs_lock_server_farewell(sb, rid) ?:
-	      reclaim_open_log_tree(sb, rid) ?:
-	      cancel_srch_compact(sb, rid) ?:
-	      cancel_log_merge(sb, rid) ?:
-	      scoutfs_omap_remove_rid(sb, rid) ?:
-	      delete_mounted_client(sb, rid);
+		err = scoutfs_lock_server_farewell(sb, rid) ?:
+		      reclaim_open_log_tree(sb, rid) ?:
+		      cancel_srch_compact(sb, rid) ?:
+		      cancel_log_merge(sb, rid) ?:
+		      scoutfs_omap_remove_rid(sb, rid) ?:
+		      delete_mounted_client(sb, rid);
 
-	return server_apply_commit(sb, &hold, ret);
+		ret = server_apply_commit(sb, &hold, err == -EINPROGRESS ? 0 : err);
+
+	} while (err == -EINPROGRESS && ret == 0);
+
+	return ret;
 }
 
 /*

utils/fenced/ipmi-remote-host

@@ -0,0 +1,140 @@
+#!/usr/bin/bash
+# /usr/libexec/scoutfs-fenced/run/ipmi-remote-host
+
+# ipmi configuration
+SCOUTFS_IPMI_CONFIG_FILE=${SCOUTFS_IPMI_CONFIG_FILE:-/etc/scoutfs/scoutfs-ipmi.conf}
+SCOUTFS_IPMI_HOSTS_FILE=${SCOUTFS_IPMI_HOSTS_FILE:-/etc/scoutfs/scoutfs-ipmi-hosts.conf}
+
+## hosts file format
+## SCOUTFS_HOST_IP IPMI_ADDRESS
+## ex:
+#  192.168.1.1     192.168.10.1
+
+# command setup
+IPMI_POWER="/sbin/ipmipower"
+SSH_CMD="ssh -o ConnectTimeout=3 -o BatchMode=yes -o StrictHostKeyChecking=no"
+LOGGER="/bin/logger -p local3.crit -t scoutfs-fenced"
+
+$LOGGER "ipmi fence script invoked: IP: $SCOUTFS_FENCED_REQ_IP RID: $SCOUTFS_FENCED_REQ_RID TEST: $IPMITEST"
+
+echo_fail() {
+    echo "$@" >&2
+    $LOGGER "fence failed: $@"
+    exit 1
+}
+
+echo_log() {
+    echo "$@" >&2
+    $LOGGER "fence info: $@"
+}
+
+echo_test_pass() {
+    echo -e "\xE2\x9C\x94 $@"
+}
+
+echo_test_fail() {
+    echo -e "\xE2\x9D\x8C $@"
+}
+
+test -n "$SCOUTFS_IPMI_CONFIG_FILE" || \
+    echo_fail "SCOUTFS_IPMI_CONFIG_FILE isn't set"
+test -r "$SCOUTFS_IPMI_CONFIG_FILE" || \
+    echo_fail "$SCOUTFS_IPMI_CONFIG_FILE isn't readable file"
+. "$SCOUTFS_IPMI_CONFIG_FILE"
+test -n "$SCOUTFS_IPMI_HOSTS_FILE" || \
+    echo_fail "SCOUTFS_IPMI_HOSTS_FILE isn't set"
+test -r "$SCOUTFS_IPMI_HOSTS_FILE" || \
+    echo_fail "$SCOUTFS_IPMI_HOSTS_FILE isn't readable file"
+test -x "$IPMI_POWER" || \
+    echo_fail "$IPMI_POWER not found, need to install freeimpi?"
+
+export ip="$SCOUTFS_FENCED_REQ_IP"
+export rid="$SCOUTFS_FENCED_REQ_RID"
+
+getIPMIhost () {
+    host=$(awk -v ip="$1" '$1 == ip {print $2}' "$SCOUTFS_IPMI_HOSTS_FILE") || \
+        echo_fail "lookup ipmi host failed"
+    echo "$host"
+}
+
+powerOffHost() {
+    # older versions of ipmipower inverted wait-until-off/wait-until-on, so specify both
+    $IPMI_POWER $IPMI_OPTS -h "$1" --wait-until-off --wait-until-on --off || \
+        echo_fail "ipmi power off $1 failed"
+
+    ipmioutput=$($IPMI_POWER $IPMI_OPTS -h "$1" --stat) || \
+        echo_fail "ipmi power stat $1 failed"
+
+    if [[ ! "$ipmioutput" =~ off ]]; then
+        echo_fail "ipmi stat $1 not off"
+    fi
+
+    $LOGGER "ipmi fence power down $1 success"
+
+    exit 0
+}
+
+if [ -n "$IPMITEST" ]; then
+    for i in $(awk '!/^($|[[:space:]]*#)/ {print $1}' "$SCOUTFS_IPMI_HOSTS_FILE"); do
+        if ! $SSH_CMD "$i" /bin/true; then
+            echo_test_fail "ssh $i"
+        else
+            echo_test_pass "ssh $i"
+        fi
+	host=$(getIPMIhost "$i")
+        if [ -z "$host" ]; then
+            echo_test_fail "ipmi config $i $host"
+        else
+            if ! $IPMI_POWER $IPMI_OPTS -h "$host" --stat; then
+                echo_test_fail "ipmi $i"
+            else
+                echo_test_pass "ipmi $i"
+            fi
+        fi
+    done
+    exit 0
+fi
+
+if [ -z "$ip" ]; then
+    echo_fail "no IP given for fencing"
+fi
+
+host=$(getIPMIhost "$ip")
+if [ -z "$host" ]; then
+    echo_fail "no IPMI host found for fence IP"
+fi
+
+# first check via ssh if the mount still exists
+# if ssh succeeds, we will only power down the node if mounted
+if ! output=$($SSH_CMD "$ip" "echo BEGIN; LC_ALL=C egrep -m 1 '(^0x*|^$rid$)' /sys/kernel/boot_params/version /sys/fs/scoutfs/f*r*/rid; echo END"); then
+    # ssh not working, just power down host
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ BEGIN ]]; then
+    # ssh failure
+    echo_log "no BEGIN"
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ \/boot_params\/ ]]; then
+    # ssh failure
+    echo_log "no boot params"
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ END ]]; then
+    # ssh failure
+    echo_log "no END"
+    powerOffHost "$host"
+fi
+
+if [[ "$output" =~ "rid:$rid" ]]; then
+    # rid still mounted, power down
+    echo_log "rid $rid still mounted"
+    powerOffHost "$host"
+fi
+
+$LOGGER "ipmi fence host $ip/$host success (rid $rid not mounted)"
+exit 0
+

utils/fenced/local-force-unmount

@@ -0,0 +1,36 @@
+#!/usr/bin/bash
+# /usr/libexec/scoutfs-fenced/run/local-force-umount
+
+echo_fail() {
+	echo "$@" > /dev/stderr
+	exit 1
+}
+
+rid="$SCOUTFS_FENCED_REQ_RID"
+
+#
+# Look for a local mount with the rid to fence.  Typically we'll at
+# least find the mount with the server that requested the fence that
+# we're processing.   But it's possible that mounts are unmounted
+# before, or while, we're running.
+#
+mnts=$(findmnt -l -n -t scoutfs -o TARGET) || \
+	echo_fail "findmnt -t scoutfs failed" > /dev/stderr
+
+for mnt in $mnts; do
+	mnt_rid=$(scoutfs statfs -p "$mnt" -s rid) || \
+		echo_fail "scoutfs statfs $mnt failed"
+
+	if [ "$mnt_rid" == "$rid" ]; then
+		umount -f "$mnt" || \
+			echo_fail "umout -f $mnt"
+
+		exit 0
+	fi
+done
+
+#
+# If the mount doesn't exist on this host then it can't access the
+# devices by definition and can be considered fenced.
+#
+exit 0

utils/fenced/powerman-remote-host

@@ -0,0 +1,139 @@
+#!/usr/bin/bash
+# /usr/libexec/scoutfs-fenced/run/powerman-remote-host
+
+# powerman configuration
+SCOUTFS_PM_CONFIG_FILE=${SCOUTFS_PM_CONFIG_FILE:-/etc/scoutfs/scoutfs-pm.conf}
+SCOUTFS_PM_HOSTS_FILE=${SCOUTFS_PM_HOSTS_FILE:-/etc/scoutfs/scoutfs-pm-hosts.conf}
+
+## hosts file format
+## SCOUTFS_HOST_IP POWERMAN_NODE_NAME
+## ex:
+#  192.168.1.1     dm1
+
+# command setup
+PM_CMD="/usr/bin/pm"
+SSH_CMD="ssh -o ConnectTimeout=3 -o BatchMode=yes -o StrictHostKeyChecking=no"
+LOGGER="/bin/logger -p local3.crit -t scoutfs-fenced"
+
+$LOGGER "ipmi fence script invoked: IP: $SCOUTFS_FENCED_REQ_IP RID: $SCOUTFS_FENCED_REQ_RID TEST: $IPMITEST"
+
+echo_fail() {
+    echo "$@" >&2
+    $LOGGER "fence failed: $@"
+    exit 1
+}
+
+echo_log() {
+    echo "$@" >&2
+    $LOGGER "fence info: $@"
+}
+
+echo_test_pass() {
+    echo -e "\xE2\x9C\x94 $@"
+}
+
+echo_test_fail() {
+    echo -e "\xE2\x9D\x8C $@"
+}
+
+test -n "$SCOUTFS_PM_CONFIG_FILE" || \
+    echo_fail "SCOUTFS_PM_CONFIG_FILE isn't set"
+test -r "$SCOUTFS_PM_CONFIG_FILE" || \
+    echo_fail "$SCOUTFS_PM_CONFIG_FILE isn't readable file"
+. "$SCOUTFS_PM_CONFIG_FILE"
+test -n "$SCOUTFS_PM_HOSTS_FILE" || \
+    echo_fail "SCOUTFS_PM_HOSTS_FILE isn't set"
+test -r "$SCOUTFS_PM_HOSTS_FILE" || \
+    echo_fail "$SCOUTFS_PM_HOSTS_FILE isn't readable file"
+test -x "$PM_CMD" || \
+    echo_fail "$PMCMD not found, need to install powerman?"
+
+export ip="$SCOUTFS_FENCED_REQ_IP"
+fence_rid="$SCOUTFS_FENCED_REQ_RID"
+
+getPMhost () {
+    host=$(awk -v ip="$1" '$1 == ip {print $2}' "$SCOUTFS_PM_HOSTS_FILE") || \
+        echo_fail "lookup pm host failed"
+    echo "$host"
+}
+
+powerOffHost() {
+    $PM_CMD $PM_OPTS "$1" -0 || \
+        echo_fail "pm power off $host failed"
+
+    pmoutput=$($PM_CMD $PM_OPTS "$1" -q | grep "$1") || \
+        echo_fail "powerman power stat $1 failed"
+
+    if [[ ! "$pmoutput" =~ off ]]; then
+        echo_fail "powerman stat $1 not off"
+    fi
+
+    $LOGGER "powerman fence power down $1 success"
+
+    exit 0
+}
+
+if [ -n "$PMTEST" ]; then
+    for i in $(awk '!/^($|[[:space:]]*#)/ {print $1}' "$SCOUTFS_PM_HOSTS_FILE"); do
+        if ! $SSH_CMD "$i" /bin/true; then
+            echo_test_fail "ssh $i"
+        else
+            echo_test_pass "ssh $i"
+        fi
+	host=$(getPMhost "$i")
+        if [ -z "$host" ]; then
+            echo_test_fail "pm config $i $host"
+        else
+            if ! $PM_CMD $PM_OPTS "$host" -q; then
+                echo_test_fail "pm $i"
+            else
+                echo_test_pass "pm $i"
+            fi
+        fi
+    done
+    exit 0
+fi
+
+if [ -z "$ip" ]; then
+    echo_fail "no IP given for fencing"
+fi
+
+host=$(getPMhost "$ip")
+if [ -z "$host" ]; then
+    echo_fail "no host found for fence IP"
+fi
+
+# first check via ssh if the mount still exists
+# if ssh succeeds, we will only power down the node if mounted
+if ! output=$($SSH_CMD "$ip" "echo BEGIN; LC_ALL=C egrep -m 1 '(^0x*|^$rid$)' /sys/kernel/boot_params/version /sys/fs/scoutfs/f*r*/rid; echo END"); then
+    # ssh not working, just power down host
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ BEGIN ]]; then
+    # ssh failure
+    echo_log "no BEGIN"
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ \/boot_params\/ ]]; then
+    # ssh failure
+    echo_log "no boot params"
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ END ]]; then
+    # ssh failure
+    echo_log "no END"
+    powerOffHost "$host"
+fi
+
+if [[ "$output" =~ "rid:$rid" ]]; then
+    # rid still mounted, power down
+    echo_log "rid $rid still mounted"
+    powerOffHost "$host"
+fi
+
+$LOGGER "powerman fence host $ip/$host success (rid $rid not mounted)"
+exit 0
+

utils/fenced/scoutfs-ipmi-hosts.conf

@@ -0,0 +1,11 @@
+# /etc/scoutfs/scoutfs-ipmi-hosts.conf
+
+## config file format
+##
+## SCOUTFS_HOST_IP must match the interface used for scoutfs
+## leader/follower communications
+##
+## SCOUTFS_HOST_IP IPMI_ADDRESS
+## ex:
+#192.168.1.1     192.168.10.1
+

utils/fenced/scoutfs-ipmi.conf

@@ -0,0 +1,10 @@
+#!/usr/bin/bash
+# /etc/scoutfs/scoutfs-ipmi.conf
+
+IPMI_USER="admin"
+IPMI_PASSWORD="password"
+IPMI_OPTS="-D LAN_2_0 -u $IPMI_USER -p $IPMI_PASSWORD"
+
+# some Intel BMCs need -I 17
+# IPMI_OPTS="-D LAN_2_0 -u $IPMI_USER -p $IPMI_PASSWORD -I 17"
+

utils/fenced/scoutfs-pm-hosts.conf

@@ -0,0 +1,11 @@
+# /etc/scoutfs/scoutfs-ipmi-hosts.conf
+
+## config file format
+##
+## SCOUTFS_HOST_IP must match the interface used for scoutfs
+## leader/follower communications
+##
+## SCOUTFS_HOST_IP POWERMAN_NODE_NAME
+## ex:
+#192.168.1.1     node1
+

utils/fenced/scoutfs-pm.conf

@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+# /etc/scoutfs/scoutfs-pm.conf
+
+PM_OPTS=""
+
+# optionally specify remote powerman server
+#PM_OPTS="-h pm-server.localdomain"
+

utils/scoutfs-utils.spec.in

@@ -55,14 +55,21 @@ install -m 755 -D src/scoutfs $RPM_BUILD_ROOT%{_sbindir}/scoutfs
 install -m 644 -D src/ioctl.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/ioctl.h
 install -m 644 -D src/format.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/format.h
 install -m 755 -D fenced/scoutfs-fenced $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/scoutfs-fenced
+install -m 755 -D fenced/local-force-unmount $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/local-force-unmount
+install -m 755 -D fenced/ipmi-remote-host $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/ipmi-remote-host
+install -m 755 -D fenced/powerman-remote-host $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/powerman-remote-host
 install -m 644 -D fenced/scoutfs-fenced.service $RPM_BUILD_ROOT%{_unitdir}/scoutfs-fenced.service
 install -m 644 -D fenced/scoutfs-fenced.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-fenced.conf.example
+install -m 644 -D fenced/scoutfs-ipmi.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-ipmi.conf
+install -m 644 -D fenced/scoutfs-ipmi-hosts.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-ipmi-hosts.conf
+install -m 644 -D fenced/scoutfs-pm.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-pm.conf
+install -m 644 -D fenced/scoutfs-pm-hosts.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-pm-hosts.conf
 
 %files
 %defattr(644,root,root,755)
 %{_mandir}/man*/scoutfs*.gz
 %{_unitdir}/scoutfs-fenced.service
-%{_sysconfdir}/scoutfs
+%config(noreplace) %{_sysconfdir}/scoutfs
 %defattr(755,root,root,755)
 %{_sbindir}/scoutfs
 %{_libexecdir}/scoutfs-fenced