add local,ipmi,powerman fenced scripts to utils rpm

This adds the fenced scripts so that we have a place to track these and
get updates out to users. This latest version of scripts has the checks
to validate that power off succeeded and not just assume based on power
command return status.

Signed-off-by: Ben McClelland <ben.mcclelland@versity.com>

ReleaseNotes.md

@@ -1,45 +1,6 @@
 Versity ScoutFS Release Notes
 =============================
 
----
-v1.7
-\
-*Aug 26, 2022*
-
-* **Fixed possible persistent errors moving freed data extents**
-\
-  Fixed a case where the server could hit persistent errors trying to
-  move a client's freed extents in one commit.  The client had to free
-  a large number of extents that occupied distant positions in the
-  global free extent btree.  Very large fragmented files could cause
-  this.  The server now moves the freed extents in multiple commits and
-  can always ensure forward progress.
-
-* **Fixed possible persistent errors from freed duplicate extents**
-\
-  Background orphan deletion wasn't properly synchronizing with
-  foreground tasks deleting very large files.  If a deletion took long
-  enough then background deletion could also attempt to delete inode items
-  while the deletion was making progress.  This could create duplicate
-  deletions of data extent items which causes the server to abort when
-  it later discovers the duplicate extents as it merges free lists.
-
----
-v1.6
-\
-*Jul 7, 2022*
-
-* **Fix memory leaks in rare corner cases**
-\
-  Analysis tools found a few corner cases that leaked small structures,
-  generally around error handling or startup and shutdown.
-
-* **Add --skip-likely-huge scoutfs print command option**
-\
-  Add an option to scoutfs print to reduce the size of the output
-  so that it can be used to see system-wide metadata without being
-  overwhelmed by file-level details.
-
 ---
 v1.5
 \

kmod/src/alloc.c

@@ -892,11 +892,12 @@ static int find_zone_extent(struct super_block *sb, struct scoutfs_alloc_root *r
  * -ENOENT is returned if we run out of extents in the source tree
  * before moving the total.
  *
- * If meta_budget is non-zero then -EINPROGRESS can be returned if the
- * the caller's budget is consumed in the allocator during this call
- * (though not necessarily by us, we don't have per-thread tracking of
- * allocator consumption :/).  The call can still have made progress and
- * caller is expected commit the dirty trees and examining the resulting
+ * If meta_reserved is non-zero then -EINPROGRESS can be returned if the
+ * current meta allocator's avail blocks or room for freed blocks would
+ * have fallen under the reserved amount.  The could have been
+ * successfully dirtied in this case but the number of blocks moved is
+ * not returned.  The caller is expected to deal with the partial
+ * progress by commiting the dirty trees and examining the resulting
  * modified trees to see if they need to continue moving extents.
  *
  * The caller can specify that extents in the source tree should first
@@ -913,7 +914,7 @@ int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc,
 		       struct scoutfs_block_writer *wri,
 		       struct scoutfs_alloc_root *dst,
 		       struct scoutfs_alloc_root *src, u64 total,
-		       __le64 *exclusive, __le64 *vacant, u64 zone_blocks, u64 meta_budget)
+		       __le64 *exclusive, __le64 *vacant, u64 zone_blocks, u64 meta_reserved)
 {
 	struct alloc_ext_args args = {
 		.alloc = alloc,
@@ -921,8 +922,6 @@ int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc,
 	};
 	struct scoutfs_extent found;
 	struct scoutfs_extent ext;
-	u32 avail_start = 0;
-	u32 freed_start = 0;
 	u64 moved = 0;
 	u64 count;
 	int ret = 0;
@@ -933,9 +932,6 @@ int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc,
 		vacant = NULL;
 	}
 
-	if (meta_budget != 0)
-		scoutfs_alloc_meta_remaining(alloc, &avail_start, &freed_start);
-
 	while (moved < total) {
 		count = total - moved;
 
@@ -968,10 +964,10 @@ int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc,
 		if (ret < 0)
 			break;
 
-		if (meta_budget != 0 &&
-		    scoutfs_alloc_meta_low_since(alloc, avail_start, freed_start, meta_budget,
-						 extent_mod_blocks(src->root.height) +
-						 extent_mod_blocks(dst->root.height))) {
+		if (meta_reserved != 0 &&
+		    scoutfs_alloc_meta_low(sb, alloc, meta_reserved +
+					   extent_mod_blocks(src->root.height) +
+					   extent_mod_blocks(dst->root.height))) {
 			ret = -EINPROGRESS;
 			break;
 		}
@@ -1355,27 +1351,6 @@ void scoutfs_alloc_meta_remaining(struct scoutfs_alloc *alloc, u32 *avail_total,
 	} while (read_seqretry(&alloc->seqlock, seq));
 }
 
-/*
- * Returns true if the caller's consumption of nr from either avail or
- * freed would end up exceeding their budget relative to the starting
- * remaining snapshot they took.
- */
-bool scoutfs_alloc_meta_low_since(struct scoutfs_alloc *alloc, u32 avail_start, u32 freed_start,
-				  u32 budget, u32 nr)
-{
-	u32 avail_use;
-	u32 freed_use;
-	u32 avail;
-	u32 freed;
-
-	scoutfs_alloc_meta_remaining(alloc, &avail, &freed);
-
-	avail_use = avail_start - avail;
-	freed_use = freed_start - freed;
-
-	return ((avail_use + nr) > budget) || ((freed_use + nr) > budget);
-}
-
 bool scoutfs_alloc_test_flag(struct super_block *sb,
 			    struct scoutfs_alloc *alloc, u32 flag)
 {

kmod/src/alloc.h

@@ -131,7 +131,7 @@ int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc,
 		       struct scoutfs_block_writer *wri,
 		       struct scoutfs_alloc_root *dst,
 		       struct scoutfs_alloc_root *src, u64 total,
-		       __le64 *exclusive, __le64 *vacant, u64 zone_blocks, u64 meta_budget);
+		       __le64 *exclusive, __le64 *vacant, u64 zone_blocks, u64 meta_reserved);
 int scoutfs_alloc_insert(struct super_block *sb, struct scoutfs_alloc *alloc,
 			 struct scoutfs_block_writer *wri, struct scoutfs_alloc_root *root,
 			 u64 start, u64 len);
@@ -159,8 +159,6 @@ int scoutfs_alloc_splice_list(struct super_block *sb,
 bool scoutfs_alloc_meta_low(struct super_block *sb,
 			    struct scoutfs_alloc *alloc, u32 nr);
 void scoutfs_alloc_meta_remaining(struct scoutfs_alloc *alloc, u32 *avail_total, u32 *freed_space);
-bool scoutfs_alloc_meta_low_since(struct scoutfs_alloc *alloc, u32 avail_start, u32 freed_start,
-				  u32 budget, u32 nr);
 bool scoutfs_alloc_test_flag(struct super_block *sb,
 			    struct scoutfs_alloc *alloc, u32 flag);
 

kmod/src/inode.c

@@ -1685,7 +1685,6 @@ static int try_delete_inode_items(struct super_block *sb, u64 ino)
 	struct scoutfs_lock *lock = NULL;
 	struct scoutfs_inode sinode;
 	struct scoutfs_key key;
-	bool clear_trying = false;
 	u64 group_nr;
 	int bit_nr;
 	int ret;
@@ -1705,7 +1704,6 @@ static int try_delete_inode_items(struct super_block *sb, u64 ino)
 		ret = 0;
 		goto out;
 	}
-	clear_trying = true;
 
 	/* can't delete if it's cached in local or remote mounts */
 	if (scoutfs_omap_test(sb, ino) || test_bit_le(bit_nr, ldata->map.bits)) {
@@ -1732,7 +1730,7 @@ static int try_delete_inode_items(struct super_block *sb, u64 ino)
 
 	ret = delete_inode_items(sb, ino, &sinode, lock, orph_lock);
 out:
-	if (clear_trying)
+	if (ldata)
 		clear_bit(bit_nr, ldata->trying);
 
 	scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE);

kmod/src/net.c

@@ -355,7 +355,6 @@ static int submit_send(struct super_block *sb,
 		}
 		if (rid != 0) {
 			spin_unlock(&conn->lock);
-			kfree(msend);
 			return -ENOTCONN;
 		}
 	}
@@ -1346,12 +1345,10 @@ scoutfs_net_alloc_conn(struct super_block *sb,
 	if (!conn)
 		return NULL;
 
-	if (info_size) {
-		conn->info = kzalloc(info_size, GFP_NOFS);
-		if (!conn->info) {
-			kfree(conn);
-			return NULL;
-		}
+	conn->info = kzalloc(info_size, GFP_NOFS);
+	if (!conn->info) {
+		kfree(conn);
+		return NULL;
 	}
 
 	conn->workq = alloc_workqueue("scoutfs_net_%s",

kmod/src/omap.c

@@ -157,15 +157,6 @@ static int free_rid(struct omap_rid_list *list, struct omap_rid_entry *entry)
 	return nr;
 }
 
-static void free_rid_list(struct omap_rid_list *list)
-{
-	struct omap_rid_entry *entry;
-	struct omap_rid_entry *tmp;
-
-	list_for_each_entry_safe(entry, tmp, &list->head, head)
-		free_rid(list, entry);
-}
-
 static int copy_rids(struct omap_rid_list *to, struct omap_rid_list *from, spinlock_t *from_lock)
 {
 	struct omap_rid_entry *entry;
@@ -813,10 +804,6 @@ void scoutfs_omap_server_shutdown(struct super_block *sb)
 	llist_for_each_entry_safe(req, tmp, requests, llnode)
 		kfree(req);
 
-	spin_lock(&ominf->lock);
-	free_rid_list(&ominf->rids);
-	spin_unlock(&ominf->lock);
-
 	synchronize_rcu();
 }
 
@@ -877,10 +864,6 @@ void scoutfs_omap_destroy(struct super_block *sb)
 		rhashtable_walk_stop(&iter);
 		rhashtable_walk_exit(&iter);
 
-		spin_lock(&ominf->lock);
-		free_rid_list(&ominf->rids);
-		spin_unlock(&ominf->lock);
-
 		rhashtable_destroy(&ominf->group_ht);
 		rhashtable_destroy(&ominf->req_ht);
 		kfree(ominf);

kmod/src/server.c

@@ -694,13 +694,13 @@ static int alloc_move_refill_zoned(struct super_block *sb, struct scoutfs_alloc_
 
 static int alloc_move_empty(struct super_block *sb,
 			    struct scoutfs_alloc_root *dst,
-			    struct scoutfs_alloc_root *src, u64 meta_budget)
+			    struct scoutfs_alloc_root *src, u64 meta_reserved)
 {
 	DECLARE_SERVER_INFO(sb, server);
 
 	return scoutfs_alloc_move(sb, &server->alloc, &server->wri,
 				  dst, src, le64_to_cpu(src->total_len), NULL, NULL, 0,
-				  meta_budget);
+				  meta_reserved);
 }
 
 /*
@@ -1226,82 +1226,6 @@ static int finalize_and_start_log_merge(struct super_block *sb, struct scoutfs_l
 	return ret;
 }
 
-/*
- * The calling get_log_trees ran out of available blocks in its commit's
- * metadata allocator while moving extents from the log tree's
- * data_freed into the core data_avail.  This finishes moving the
- * extents in as many additional commits as it takes.   The logs mutex
- * is nested inside holding commits so we recheck the persistent item
- * each time we commit to make sure it's still what we think.   The
- * caller is still going to send the item to the client so we update the
- * caller's each time we make progress.  This is a best-effort attempt
- * to clean up and it's valid to leave extents in data_freed we don't
- * return errors to the caller.  The client will continue the work later
- * in get_log_trees or as the rid is reclaimed.
- */
-static void try_drain_data_freed(struct super_block *sb, struct scoutfs_log_trees *lt)
-{
-	DECLARE_SERVER_INFO(sb, server);
-	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
-	const u64 rid = le64_to_cpu(lt->rid);
-	const u64 nr = le64_to_cpu(lt->nr);
-	struct scoutfs_log_trees drain;
-	struct scoutfs_key key;
-	COMMIT_HOLD(hold);
-	int ret = 0;
-	int err;
-
-	scoutfs_key_init_log_trees(&key, rid, nr);
-
-	while (lt->data_freed.total_len != 0) {
-		server_hold_commit(sb, &hold);
-		mutex_lock(&server->logs_mutex);
-
-		ret = find_log_trees_item(sb, &super->logs_root, false, rid, U64_MAX, &drain);
-		if (ret < 0)
-			break;
-
-		/* careful to only keep draining the caller's specific open trans */
-		if (drain.nr != lt->nr || drain.get_trans_seq != lt->get_trans_seq ||
-		    drain.commit_trans_seq != lt->commit_trans_seq || drain.flags != lt->flags) {
-			ret = -ENOENT;
-			break;
-		}
-
-		ret = scoutfs_btree_dirty(sb, &server->alloc, &server->wri,
-					  &super->logs_root, &key);
-		if (ret < 0)
-			break;
-
-		/* moving can modify and return errors, always update caller and item */
-		mutex_lock(&server->alloc_mutex);
-		ret = alloc_move_empty(sb, &super->data_alloc, &drain.data_freed,
-				       COMMIT_HOLD_ALLOC_BUDGET / 2);
-		mutex_unlock(&server->alloc_mutex);
-		if (ret == -EINPROGRESS)
-			ret = 0;
-
-		*lt = drain;
-		err = scoutfs_btree_force(sb, &server->alloc, &server->wri,
-					  &super->logs_root, &key, &drain, sizeof(drain));
-		BUG_ON(err < 0); /* dirtying must guarantee success */
-
-		mutex_unlock(&server->logs_mutex);
-
-		ret = server_apply_commit(sb, &hold, ret);
-		if (ret < 0) {
-			ret = 0; /* don't try to abort, ignoring ret */
-			break;
-		}
-	}
-
-	/* try to cleanly abort and write any partial dirty btree blocks, but ignore result */
-	if (ret < 0) {
-		mutex_unlock(&server->logs_mutex);
-		server_apply_commit(sb, &hold, 0);
-	}
-}
-
 /*
  * Give the client roots to all the trees that they'll use to build
  * their transaction.
@@ -1427,9 +1351,7 @@ static int server_get_log_trees(struct super_block *sb,
 		goto update;
 	}
 
-	ret = alloc_move_empty(sb, &super->data_alloc, &lt.data_freed, 100);
-	if (ret == -EINPROGRESS)
-		ret = 0;
+	ret = alloc_move_empty(sb, &super->data_alloc, &lt.data_freed, 0);
 	if (ret < 0) {
 		err_str = "emptying committed data_freed";
 		goto update;
@@ -1507,10 +1429,6 @@ out:
 		scoutfs_err(sb, "error %d getting log trees for rid %016llx: %s",
 			    ret, rid, err_str);
 
-	/* try to drain excessive data_freed with additional commits, if needed, ignoring err */
-	if (ret == 0)
-		try_drain_data_freed(sb, &lt);
-
 	return scoutfs_net_response(sb, conn, cmd, id, ret, &lt, sizeof(lt));
 }
 

kmod/src/super.c

@@ -496,7 +496,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 
 	ret = assign_random_id(sbi);
 	if (ret < 0)
-		goto out;
+		return ret;
 
 	spin_lock_init(&sbi->next_ino_lock);
 	spin_lock_init(&sbi->data_wait_root.lock);
@@ -505,7 +505,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 	/* parse options early for use during setup */
 	ret = scoutfs_options_early_setup(sb, data);
 	if (ret < 0)
-		goto out;
+		return ret;
 	scoutfs_options_read(sb, &opts);
 
 	ret = sb_set_blocksize(sb, SCOUTFS_BLOCK_SM_SIZE);

tests/Makefile

@@ -10,8 +10,7 @@ BIN := src/createmany			\
 	src/bulk_create_paths		\
 	src/stage_tmpfile		\
 	src/find_xattrs			\
-	src/create_xattr_loop		\
-	src/fragmented_data_extents
+	src/create_xattr_loop
 
 DEPS := $(wildcard src/*.d)
 

tests/golden/large-fragmented-free

@@ -1,3 +0,0 @@
-== creating fragmented extents
-== unlink file with moved extents to free extents per block
-== cleanup

tests/sequence

@@ -9,7 +9,6 @@ fallocate.sh
 setattr_more.sh
 offline-extent-waiting.sh
 move-blocks.sh
-large-fragmented-free.sh
 enospc.sh
 srch-basic-functionality.sh
 simple-xattr-unit.sh

tests/src/fragmented_data_extents.c

@@ -1,113 +0,0 @@
-/*
- * Copyright (C) 2021 Versity Software, Inc.  All rights reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public
- * License v2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- */
-
-/*
- * This creates fragmented data extents.
- *
- * A file is created that has alternating free and allocated extents.
- * This also results in the global allocator having the matching
- * fragmented free extent pattern.  While that file is being created,
- * occasionally an allocated extent is moved to another file.   This
- * results in a file that has fragmented extents at a given stride that
- * can be deleted to create free data extents with a given stride.
- *
- * We don't have hole punching so to do this quickly we use a goofy
- * combination of fallocate, truncate, and our move_blocks ioctl.
- */
-
-#ifndef _GNU_SOURCE
-#define _GNU_SOURCE
-#endif
-#include <unistd.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/ioctl.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <linux/types.h>
-#include <assert.h>
-
-#include "ioctl.h"
-
-#define BLOCK_SIZE 4096
-
-int main(int argc, char **argv)
-{
-	struct scoutfs_ioctl_move_blocks mb = {0,};
-	unsigned long long freed_extents;
-	unsigned long long move_stride;
-	unsigned long long i;
-	int alloc_fd;
-	int trunc_fd;
-	off_t off;
-	int ret;
-
-	if (argc != 5) {
-		printf("%s <freed_extents> <move_stride> <alloc_file> <trunc_file>\n", argv[0]);
-		return 1;
-	}
-
-	freed_extents = strtoull(argv[1], NULL, 0);
-	move_stride = strtoull(argv[2], NULL, 0);
-
-	alloc_fd = open(argv[3], O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
-	if (alloc_fd == -1) {
-		fprintf(stderr, "error opening %s: %d (%s)\n", argv[3], errno, strerror(errno));
-		exit(1);
-	}
-
-	trunc_fd = open(argv[4], O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
-	if (trunc_fd == -1) {
-		fprintf(stderr, "error opening %s: %d (%s)\n", argv[4], errno, strerror(errno));
-		exit(1);
-	}
-
-	for (i = 0, off = 0; i < freed_extents; i++, off += BLOCK_SIZE * 2) {
-
-		ret = fallocate(alloc_fd, 0, off, BLOCK_SIZE * 2);
-		if (ret < 0) {
-			fprintf(stderr, "fallocate at off %llu error: %d (%s)\n",
-				(unsigned long long)off, errno, strerror(errno));
-			exit(1);
-		}
-
-		ret = ftruncate(alloc_fd, off + BLOCK_SIZE);
-		if (ret < 0) {
-			fprintf(stderr, "truncate to off %llu error: %d (%s)\n",
-				(unsigned long long)off + BLOCK_SIZE, errno, strerror(errno));
-			exit(1);
-		}
-
-		if ((i % move_stride) == 0) {
-			mb.from_fd = alloc_fd;
-			mb.from_off = off;
-			mb.len = BLOCK_SIZE;
-			mb.to_off = i * BLOCK_SIZE;
-
-			ret = ioctl(trunc_fd, SCOUTFS_IOC_MOVE_BLOCKS, &mb);
-			if (ret < 0) {
-				fprintf(stderr, "move from off %llu error: %d (%s)\n",
-					(unsigned long long)off,
-					errno, strerror(errno));
-			}
-		}
-	}
-
-	if (alloc_fd > -1)
-		close(alloc_fd);
-	if (trunc_fd > -1)
-		close(trunc_fd);
-
-	return 0;
-}

tests/tests/large-fragmented-free.sh

@@ -1,22 +0,0 @@
-#
-# Make sure the server can handle a transaction with a data_freed whose
-# blocks all hit different btree blocks in the main free list.  It
-# probably has to be merged in multiple commits.
-#
-
-t_require_commands fragmented_data_extents
-
-EXTENTS_PER_BTREE_BLOCK=600
-EXTENTS_PER_LIST_BLOCK=8192
-FREED_EXTENTS=$((EXTENTS_PER_BTREE_BLOCK * EXTENTS_PER_LIST_BLOCK))
-
-echo "== creating fragmented extents"
-fragmented_data_extents $FREED_EXTENTS $EXTENTS_PER_BTREE_BLOCK "$T_D0/alloc" "$T_D0/move"
-
-echo "== unlink file with moved extents to free extents per block"
-rm -f "$T_D0/move"
-
-echo "== cleanup"
-rm -f "$T_D0/alloc"
-
-t_pass

utils/fenced/ipmi-remote-host

@@ -0,0 +1,140 @@
+#!/usr/bin/bash
+# /usr/libexec/scoutfs-fenced/run/ipmi-remote-host
+
+# ipmi configuration
+SCOUTFS_IPMI_CONFIG_FILE=${SCOUTFS_IPMI_CONFIG_FILE:-/etc/scoutfs/scoutfs-ipmi.conf}
+SCOUTFS_IPMI_HOSTS_FILE=${SCOUTFS_IPMI_HOSTS_FILE:-/etc/scoutfs/scoutfs-ipmi-hosts.conf}
+
+## hosts file format
+## SCOUTFS_HOST_IP IPMI_ADDRESS
+## ex:
+#  192.168.1.1     192.168.10.1
+
+# command setup
+IPMI_POWER="/sbin/ipmipower"
+SSH_CMD="ssh -o ConnectTimeout=3 -o BatchMode=yes -o StrictHostKeyChecking=no"
+LOGGER="/bin/logger -p local3.crit -t scoutfs-fenced"
+
+$LOGGER "ipmi fence script invoked: IP: $SCOUTFS_FENCED_REQ_IP RID: $SCOUTFS_FENCED_REQ_RID TEST: $IPMITEST"
+
+echo_fail() {
+    echo "$@" >&2
+    $LOGGER "fence failed: $@"
+    exit 1
+}
+
+echo_log() {
+    echo "$@" >&2
+    $LOGGER "fence info: $@"
+}
+
+echo_test_pass() {
+    echo -e "\xE2\x9C\x94 $@"
+}
+
+echo_test_fail() {
+    echo -e "\xE2\x9D\x8C $@"
+}
+
+test -n "$SCOUTFS_IPMI_CONFIG_FILE" || \
+    echo_fail "SCOUTFS_IPMI_CONFIG_FILE isn't set"
+test -r "$SCOUTFS_IPMI_CONFIG_FILE" || \
+    echo_fail "$SCOUTFS_IPMI_CONFIG_FILE isn't readable file"
+. "$SCOUTFS_IPMI_CONFIG_FILE"
+test -n "$SCOUTFS_IPMI_HOSTS_FILE" || \
+    echo_fail "SCOUTFS_IPMI_HOSTS_FILE isn't set"
+test -r "$SCOUTFS_IPMI_HOSTS_FILE" || \
+    echo_fail "$SCOUTFS_IPMI_HOSTS_FILE isn't readable file"
+test -x "$IPMI_POWER" || \
+    echo_fail "$IPMI_POWER not found, need to install freeimpi?"
+
+export ip="$SCOUTFS_FENCED_REQ_IP"
+export rid="$SCOUTFS_FENCED_REQ_RID"
+
+getIPMIhost () {
+    host=$(awk -v ip="$1" '$1 == ip {print $2}' "$SCOUTFS_IPMI_HOSTS_FILE") || \
+        echo_fail "lookup ipmi host failed"
+    echo "$host"
+}
+
+powerOffHost() {
+    # older versions of ipmipower inverted wait-until-off/wait-until-on, so specify both
+    $IPMI_POWER $IPMI_OPTS -h "$1" --wait-until-off --wait-until-on --off || \
+        echo_fail "ipmi power off $1 failed"
+
+    ipmioutput=$($IPMI_POWER $IPMI_OPTS -h "$1" --stat) || \
+        echo_fail "ipmi power stat $1 failed"
+
+    if [[ ! "$ipmioutput" =~ off ]]; then
+        echo_fail "ipmi stat $1 not off"
+    fi
+
+    $LOGGER "ipmi fence power down $1 success"
+
+    exit 0
+}
+
+if [ -n "$IPMITEST" ]; then
+    for i in $(awk '!/^($|[[:space:]]*#)/ {print $1}' "$SCOUTFS_IPMI_HOSTS_FILE"); do
+        if ! $SSH_CMD "$i" /bin/true; then
+            echo_test_fail "ssh $i"
+        else
+            echo_test_pass "ssh $i"
+        fi
+	host=$(getIPMIhost "$i")
+        if [ -z "$host" ]; then
+            echo_test_fail "ipmi config $i $host"
+        else
+            if ! $IPMI_POWER $IPMI_OPTS -h "$host" --stat; then
+                echo_test_fail "ipmi $i"
+            else
+                echo_test_pass "ipmi $i"
+            fi
+        fi
+    done
+    exit 0
+fi
+
+if [ -z "$ip" ]; then
+    echo_fail "no IP given for fencing"
+fi
+
+host=$(getIPMIhost "$ip")
+if [ -z "$host" ]; then
+    echo_fail "no IPMI host found for fence IP"
+fi
+
+# first check via ssh if the mount still exists
+# if ssh succeeds, we will only power down the node if mounted
+if ! output=$($SSH_CMD "$ip" "echo BEGIN; LC_ALL=C egrep -m 1 '(^0x*|^$rid$)' /sys/kernel/boot_params/version /sys/fs/scoutfs/f*r*/rid; echo END"); then
+    # ssh not working, just power down host
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ BEGIN ]]; then
+    # ssh failure
+    echo_log "no BEGIN"
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ \/boot_params\/ ]]; then
+    # ssh failure
+    echo_log "no boot params"
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ END ]]; then
+    # ssh failure
+    echo_log "no END"
+    powerOffHost "$host"
+fi
+
+if [[ "$output" =~ "rid:$rid" ]]; then
+    # rid still mounted, power down
+    echo_log "rid $rid still mounted"
+    powerOffHost "$host"
+fi
+
+$LOGGER "ipmi fence host $ip/$host success (rid $rid not mounted)"
+exit 0
+

utils/fenced/local-force-unmount

@@ -0,0 +1,36 @@
+#!/usr/bin/bash
+# /usr/libexec/scoutfs-fenced/run/local-force-umount
+
+echo_fail() {
+	echo "$@" > /dev/stderr
+	exit 1
+}
+
+rid="$SCOUTFS_FENCED_REQ_RID"
+
+#
+# Look for a local mount with the rid to fence.  Typically we'll at
+# least find the mount with the server that requested the fence that
+# we're processing.   But it's possible that mounts are unmounted
+# before, or while, we're running.
+#
+mnts=$(findmnt -l -n -t scoutfs -o TARGET) || \
+	echo_fail "findmnt -t scoutfs failed" > /dev/stderr
+
+for mnt in $mnts; do
+	mnt_rid=$(scoutfs statfs -p "$mnt" -s rid) || \
+		echo_fail "scoutfs statfs $mnt failed"
+
+	if [ "$mnt_rid" == "$rid" ]; then
+		umount -f "$mnt" || \
+			echo_fail "umout -f $mnt"
+
+		exit 0
+	fi
+done
+
+#
+# If the mount doesn't exist on this host then it can't access the
+# devices by definition and can be considered fenced.
+#
+exit 0

utils/fenced/powerman-remote-host

@@ -0,0 +1,139 @@
+#!/usr/bin/bash
+# /usr/libexec/scoutfs-fenced/run/powerman-remote-host
+
+# powerman configuration
+SCOUTFS_PM_CONFIG_FILE=${SCOUTFS_PM_CONFIG_FILE:-/etc/scoutfs/scoutfs-pm.conf}
+SCOUTFS_PM_HOSTS_FILE=${SCOUTFS_PM_HOSTS_FILE:-/etc/scoutfs/scoutfs-pm-hosts.conf}
+
+## hosts file format
+## SCOUTFS_HOST_IP POWERMAN_NODE_NAME
+## ex:
+#  192.168.1.1     dm1
+
+# command setup
+PM_CMD="/usr/bin/pm"
+SSH_CMD="ssh -o ConnectTimeout=3 -o BatchMode=yes -o StrictHostKeyChecking=no"
+LOGGER="/bin/logger -p local3.crit -t scoutfs-fenced"
+
+$LOGGER "ipmi fence script invoked: IP: $SCOUTFS_FENCED_REQ_IP RID: $SCOUTFS_FENCED_REQ_RID TEST: $IPMITEST"
+
+echo_fail() {
+    echo "$@" >&2
+    $LOGGER "fence failed: $@"
+    exit 1
+}
+
+echo_log() {
+    echo "$@" >&2
+    $LOGGER "fence info: $@"
+}
+
+echo_test_pass() {
+    echo -e "\xE2\x9C\x94 $@"
+}
+
+echo_test_fail() {
+    echo -e "\xE2\x9D\x8C $@"
+}
+
+test -n "$SCOUTFS_PM_CONFIG_FILE" || \
+    echo_fail "SCOUTFS_PM_CONFIG_FILE isn't set"
+test -r "$SCOUTFS_PM_CONFIG_FILE" || \
+    echo_fail "$SCOUTFS_PM_CONFIG_FILE isn't readable file"
+. "$SCOUTFS_PM_CONFIG_FILE"
+test -n "$SCOUTFS_PM_HOSTS_FILE" || \
+    echo_fail "SCOUTFS_PM_HOSTS_FILE isn't set"
+test -r "$SCOUTFS_PM_HOSTS_FILE" || \
+    echo_fail "$SCOUTFS_PM_HOSTS_FILE isn't readable file"
+test -x "$PM_CMD" || \
+    echo_fail "$PMCMD not found, need to install powerman?"
+
+export ip="$SCOUTFS_FENCED_REQ_IP"
+fence_rid="$SCOUTFS_FENCED_REQ_RID"
+
+getPMhost () {
+    host=$(awk -v ip="$1" '$1 == ip {print $2}' "$SCOUTFS_PM_HOSTS_FILE") || \
+        echo_fail "lookup pm host failed"
+    echo "$host"
+}
+
+powerOffHost() {
+    $PM_CMD $PM_OPTS "$1" -0 || \
+        echo_fail "pm power off $host failed"
+
+    pmoutput=$($PM_CMD $PM_OPTS "$1" -q | grep "$1") || \
+        echo_fail "powerman power stat $1 failed"
+
+    if [[ ! "$pmoutput" =~ off ]]; then
+        echo_fail "powerman stat $1 not off"
+    fi
+
+    $LOGGER "powerman fence power down $1 success"
+
+    exit 0
+}
+
+if [ -n "$PMTEST" ]; then
+    for i in $(awk '!/^($|[[:space:]]*#)/ {print $1}' "$SCOUTFS_PM_HOSTS_FILE"); do
+        if ! $SSH_CMD "$i" /bin/true; then
+            echo_test_fail "ssh $i"
+        else
+            echo_test_pass "ssh $i"
+        fi
+	host=$(getPMhost "$i")
+        if [ -z "$host" ]; then
+            echo_test_fail "pm config $i $host"
+        else
+            if ! $PM_CMD $PM_OPTS "$host" -q; then
+                echo_test_fail "pm $i"
+            else
+                echo_test_pass "pm $i"
+            fi
+        fi
+    done
+    exit 0
+fi
+
+if [ -z "$ip" ]; then
+    echo_fail "no IP given for fencing"
+fi
+
+host=$(getPMhost "$ip")
+if [ -z "$host" ]; then
+    echo_fail "no host found for fence IP"
+fi
+
+# first check via ssh if the mount still exists
+# if ssh succeeds, we will only power down the node if mounted
+if ! output=$($SSH_CMD "$ip" "echo BEGIN; LC_ALL=C egrep -m 1 '(^0x*|^$rid$)' /sys/kernel/boot_params/version /sys/fs/scoutfs/f*r*/rid; echo END"); then
+    # ssh not working, just power down host
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ BEGIN ]]; then
+    # ssh failure
+    echo_log "no BEGIN"
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ \/boot_params\/ ]]; then
+    # ssh failure
+    echo_log "no boot params"
+    powerOffHost "$host"
+fi
+
+if [[ ! "$output" =~ END ]]; then
+    # ssh failure
+    echo_log "no END"
+    powerOffHost "$host"
+fi
+
+if [[ "$output" =~ "rid:$rid" ]]; then
+    # rid still mounted, power down
+    echo_log "rid $rid still mounted"
+    powerOffHost "$host"
+fi
+
+$LOGGER "powerman fence host $ip/$host success (rid $rid not mounted)"
+exit 0
+

utils/fenced/scoutfs-ipmi-hosts.conf

@@ -0,0 +1,11 @@
+# /etc/scoutfs/scoutfs-ipmi-hosts.conf
+
+## config file format
+##
+## SCOUTFS_HOST_IP must match the interface used for scoutfs
+## leader/follower communications
+##
+## SCOUTFS_HOST_IP IPMI_ADDRESS
+## ex:
+#192.168.1.1     192.168.10.1
+

utils/fenced/scoutfs-ipmi.conf

@@ -0,0 +1,10 @@
+#!/usr/bin/bash
+# /etc/scoutfs/scoutfs-ipmi.conf
+
+IPMI_USER="admin"
+IPMI_PASSWORD="password"
+IPMI_OPTS="-D LAN_2_0 -u $IPMI_USER -p $IPMI_PASSWORD"
+
+# some Intel BMCs need -I 17
+# IPMI_OPTS="-D LAN_2_0 -u $IPMI_USER -p $IPMI_PASSWORD -I 17"
+

utils/fenced/scoutfs-pm-hosts.conf

@@ -0,0 +1,11 @@
+# /etc/scoutfs/scoutfs-ipmi-hosts.conf
+
+## config file format
+##
+## SCOUTFS_HOST_IP must match the interface used for scoutfs
+## leader/follower communications
+##
+## SCOUTFS_HOST_IP POWERMAN_NODE_NAME
+## ex:
+#192.168.1.1     node1
+

utils/fenced/scoutfs-pm.conf

@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+# /etc/scoutfs/scoutfs-pm.conf
+
+PM_OPTS=""
+
+# optionally specify remote powerman server
+#PM_OPTS="-h pm-server.localdomain"
+

utils/man/scoutfs.8

@@ -597,7 +597,7 @@ format.
 .PD
 
 .TP
-.BI "print {-S|--skip-likely-huge} META-DEVICE"
+.BI "print META-DEVICE"
 .sp
 Prints out all of the metadata in the file system.  This makes no effort
 to ensure that the structures are consistent as they're traversed and
@@ -607,20 +607,6 @@ output.
 .PD 0
 .TP
 .sp
-.B "-S, --skip-likely-huge"
-Skip printing structures that are likely to be very large.  The
-structures that are skipped tend to be global and whose size tends to be
-related to the size of the volume.   Examples of skipped structures include
-the global fs items, srch files, and metadata and data
-allocators.  Similar structures that are not skipped are related to the
-number of mounts and are maintained at a relatively reasonable size.
-These include per-mount log trees, srch files, allocators, and the
-metadata allocators used by server commits.
-.sp
-Skipping the larger structures limits the print output to a relatively
-constant size rather than being a large multiple of the used metadata
-space of the volume making the output much more useful for inspection.
-.TP
 .B "META-DEVICE"
 The path to the metadata device for the filesystem whose metadata will be
 printed.  Since this command reads via the host's buffer cache, it may not

utils/scoutfs-utils.spec.in

@@ -55,14 +55,21 @@ install -m 755 -D src/scoutfs $RPM_BUILD_ROOT%{_sbindir}/scoutfs
 install -m 644 -D src/ioctl.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/ioctl.h
 install -m 644 -D src/format.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/format.h
 install -m 755 -D fenced/scoutfs-fenced $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/scoutfs-fenced
+install -m 755 -D fenced/local-force-unmount $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/local-force-unmount
+install -m 755 -D fenced/ipmi-remote-host $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/ipmi-remote-host
+install -m 755 -D fenced/powerman-remote-host $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/powerman-remote-host
 install -m 644 -D fenced/scoutfs-fenced.service $RPM_BUILD_ROOT%{_unitdir}/scoutfs-fenced.service
 install -m 644 -D fenced/scoutfs-fenced.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-fenced.conf.example
+install -m 644 -D fenced/scoutfs-ipmi.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-ipmi.conf
+install -m 644 -D fenced/scoutfs-ipmi-hosts.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-ipmi-hosts.conf
+install -m 644 -D fenced/scoutfs-pm.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-pm.conf
+install -m 644 -D fenced/scoutfs-pm-hosts.conf $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-pm-hosts.conf
 
 %files
 %defattr(644,root,root,755)
 %{_mandir}/man*/scoutfs*.gz
 %{_unitdir}/scoutfs-fenced.service
-%{_sysconfdir}/scoutfs
+%config(noreplace) %{_sysconfdir}/scoutfs
 %defattr(755,root,root,755)
 %{_sbindir}/scoutfs
 %{_libexecdir}/scoutfs-fenced

utils/src/print.c

@@ -8,7 +8,6 @@
 #include <errno.h>
 #include <string.h>
 #include <stdarg.h>
-#include <stdbool.h>
 #include <ctype.h>
 #include <uuid/uuid.h>
 #include <sys/socket.h>
@@ -990,10 +989,9 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
 
 struct print_args {
 	char *meta_device;
-	bool skip_likely_huge;
 };
 
-static int print_volume(int fd, struct print_args *args)
+static int print_volume(int fd)
 {
 	struct scoutfs_super_block *super = NULL;
 	struct print_recursion_args pa;
@@ -1043,26 +1041,23 @@ static int print_volume(int fd, struct print_args *args)
 			ret = err;
 	}
 
-	if (!args->skip_likely_huge) {
-		for (i = 0; i < array_size(super->meta_alloc); i++) {
-			snprintf(str, sizeof(str), "meta_alloc[%u]", i);
-			err = print_btree(fd, super, str, &super->meta_alloc[i].root,
-					  print_alloc_item, NULL);
-			if (err && !ret)
-				ret = err;
-		}
-
-		err = print_btree(fd, super, "data_alloc", &super->data_alloc.root,
+	for (i = 0; i < array_size(super->meta_alloc); i++) {
+		snprintf(str, sizeof(str), "meta_alloc[%u]", i);
+		err = print_btree(fd, super, str, &super->meta_alloc[i].root,
 				  print_alloc_item, NULL);
 		if (err && !ret)
 			ret = err;
 	}
 
+	err = print_btree(fd, super, "data_alloc", &super->data_alloc.root,
+			  print_alloc_item, NULL);
+	if (err && !ret)
+		ret = err;
+
 	err = print_btree(fd, super, "srch_root", &super->srch_root,
 			  print_srch_root_item, NULL);
 	if (err && !ret)
 		ret = err;
-
 	err = print_btree(fd, super, "logs_root", &super->logs_root,
 			  print_log_trees_item, NULL);
 	if (err && !ret)
@@ -1070,23 +1065,19 @@ static int print_volume(int fd, struct print_args *args)
 
 	pa.super = super;
 	pa.fd = fd;
-	if (!args->skip_likely_huge) {
-		err = print_btree_leaf_items(fd, super, &super->srch_root.ref,
-					     print_srch_root_files, &pa);
-		if (err && !ret)
-			ret = err;
-	}
+	err = print_btree_leaf_items(fd, super, &super->srch_root.ref,
+				     print_srch_root_files, &pa);
+	if (err && !ret)
+		ret = err;
 	err = print_btree_leaf_items(fd, super, &super->logs_root.ref,
 				     print_log_trees_roots, &pa);
 	if (err && !ret)
 		ret = err;
 
-	if (!args->skip_likely_huge) {
-		err = print_btree(fd, super, "fs_root", &super->fs_root,
-				  print_fs_item, NULL);
-		if (err && !ret)
-			ret = err;
-	}
+	err = print_btree(fd, super, "fs_root", &super->fs_root,
+			  print_fs_item, NULL);
+	if (err && !ret)
+		ret = err;
 
 out:
 	free(super);
@@ -1107,7 +1098,7 @@ static int do_print(struct print_args *args)
 		return ret;
 	}
 
-	ret = print_volume(fd, args);
+	ret = print_volume(fd);
 	close(fd);
 	return ret;
 };
@@ -1117,9 +1108,6 @@ static int parse_opt(int key, char *arg, struct argp_state *state)
 	struct print_args *args = state->input;
 
 	switch (key) {
-	case 'S':
-		args->skip_likely_huge = true;
-		break;
 	case ARGP_KEY_ARG:
 		if (!args->meta_device)
 			args->meta_device = strdup_or_error(state, arg);
@@ -1137,13 +1125,8 @@ static int parse_opt(int key, char *arg, struct argp_state *state)
 	return 0;
 }
 
-static struct argp_option options[] = {
-	{ "skip-likely-huge", 'S', NULL, 0, "Skip large structures to minimize output size"},
-	{ NULL }
-};
-
 static struct argp argp = {
-	options,
+	NULL,
 	parse_opt,
 	"META-DEV",
 	"Print metadata structures"