Remove update_finalized_and_inc_nr() missed during rebase

Signed-off-by: Chris Kirby <ckirby@versity.com>
Finalize open log tree in one commit
2026-01-10 05:37:25 +00:00 · 2025-11-03 14:42:27 -06:00 · 2025-11-03 14:17:32 -06:00 · 2025-11-03 14:16:52 -06:00
26 changed files with 1545 additions and 1399 deletions
--- a/ReleaseNotes.md
+++ b/ReleaseNotes.md
@@ -1,41 +1,6 @@
 Versity ScoutFS Release Notes
 =============================

---
-v1.26
-\
-*Nov 17, 2025*
-
-Add the ino\_alloc\_per\_lock mount option.  This changes the number of
-inode numbers allocated under each cluster lock and can alleviate lock
-contention for some patterns of larger file creation.
-
-Add the tcp\_keepalive\_timeout\_ms mount option.  This can enable the
-system to survive longer periods of networking outages.
-
-Fix a rare double free of internal btree metadata blocks when merging
-log trees.  The duplicated freed metadata block numbers would cause
-persistent errors in the server, preventing the server from starting and
-hanging the system.
-
-Fix the data\_wait interface to not require the correct data\_version of
-the inode when raising an error.  This lets callers raise errors when
-they're unable to recall the details of the inode to discover its
-data\_version.
-
-Change scoutfs to more aggressively reclaim cached memory when under
-memory pressure.  This makes scoutfs behave more like other kernel
-components and it integrates better with the reclaim policy heuristics
-in the VM core of the kernel.
-
-Change scoutfs to more efficiently transmit and receive socket messages.
-Under heavy load this can process messages sufficiently more quickly to
-avoid hung task messages for tasks that were waiting for cluster lock
-messages to be processed.
-
-Fix faulty server block commit budget calculations that were generating
-spurious "holders exceeded alloc budget" console messages.
-
 ---
 v1.25
 \
--- a/kmod/src/Makefile
+++ b/kmod/src/Makefile
@@ -14,6 +14,7 @@ scoutfs-y +=			\
 	alloc.o			\
 	block.o			\
 	btree.o			\
+	check.o			\
 	client.o		\
 	counters.o		\
 	data.o			\
--- a/kmod/src/check.c
+++ b/kmod/src/check.c
@@ -0,0 +1,356 @@
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/vmalloc.h>
+#include <linux/bitmap.h>
+
+#include "super.h"
+#include "format.h"
+#include "block.h"
+#include "msg.h"
+#include "avl.h"
+#include "check.h"
+
+struct bit_map {
+	unsigned long *addr;
+	long size;
+	long bytes;
+};
+
+static bool enabled = true;
+
+#define warn_once_disable(sb, cond, fmt, args...)	\
+({							\
+	bool cond_ = (cond);				\
+	static bool warned_ = false;			\
+							\
+	if (cond_ && !warned_) {			\
+		scoutfs_err(sb, "check: " fmt, ##args);	\
+		warned_ = true;				\
+		enabled = false;			\
+	}						\
+							\
+	cond_;						\
+})
+
+static void check_blkno(struct super_block *sb, struct bit_map *map, long nr)
+{
+	if (nr != 0 && !warn_once_disable(sb, nr < 0 || nr >= map->size,
+					  "nr %ld outside map->size %ld", nr, map->size))
+		warn_once_disable(sb, test_and_set_bit(nr, map->addr),
+			           "nr %ld already set", nr);
+}
+
+static void check_extent(struct super_block *sb, struct bit_map *map, u64 start, u64 len)
+{
+	unsigned long nr;
+
+	if (!warn_once_disable(sb, start >= map->size || len > map->size ||
+				   (start + len) > map->size,
+				   "start %llu len %llu oustdie map->size %ld",
+				   start, len, map->size)) {
+
+		nr = find_next_bit(map->addr, map->size, start);
+		warn_once_disable(sb, nr < start + len,
+				  "start %llu len %llu has bits already set, first %lu",
+				  start, len, nr);
+
+		bitmap_set(map->addr, start, len);
+	}
+}
+
+static void check_block_ref(struct super_block *sb, struct bit_map *map,
+			    struct scoutfs_block_ref *ref)
+{
+	check_blkno(sb, map, le64_to_cpu(ref->blkno));
+}
+
+/*
+ * As long as we're not handling errors, we can have this return the
+ * pointer to the block data if it was read successfully.  Everything
+ * else returns null and the caller backs off.
+ */
+static void *read_block_ref(struct super_block *sb, struct bit_map *map,
+			    struct scoutfs_block_ref *ref, u32 magic,
+			    struct scoutfs_block **bl_ret)
+{
+	check_block_ref(sb, map, ref);
+
+	if (ref->blkno != 0 && scoutfs_block_read_ref(sb, ref, magic, bl_ret) == 0)
+		return (*bl_ret)->data;
+
+	return NULL;
+}
+
+/* returns false if caller should stop iterating */
+typedef bool (*check_btree_item_cb)(struct super_block *sb, struct bit_map *map,
+				    struct scoutfs_key *key, void *val, u16 val_len);
+
+/*
+ * We walk the items in key order via the avl so that the item callbacks
+ * can have us stop iterating based on their knowledge of key ordering.
+ */
+static void check_btree_block_ref(struct super_block *sb, struct bit_map *map,
+				  u8 level, struct scoutfs_block_ref *ref,
+				  check_btree_item_cb item_cb)
+{
+	struct scoutfs_block *bl = NULL;
+	struct scoutfs_btree_block *bt;
+	struct scoutfs_btree_item *item;
+	struct scoutfs_avl_node *node;
+	void *val;
+	u16 val_off;
+	u16 val_len;
+
+	if (!(bt = read_block_ref(sb, map, ref, SCOUTFS_BLOCK_MAGIC_BTREE, &bl)))
+		return;
+
+	if (bt->level != level)
+		goto out;
+
+	for (node = scoutfs_avl_first(&bt->item_root);
+	     node != NULL;
+	     node = scoutfs_avl_next(&bt->item_root, node)) {
+		item = container_of(node, struct scoutfs_btree_item, node);
+
+		val_off = le16_to_cpu(item->val_off);
+		val_len = le16_to_cpu(item->val_len);
+		val = (void *)bt + val_off;
+
+		if (bt->level > 0)
+			check_btree_block_ref(sb, map, bt->level - 1, val, item_cb);
+		else if (item_cb && !item_cb(sb, map, &item->key, val, val_len))
+			break;
+	}
+out:
+	scoutfs_block_put(sb, bl);
+}
+
+static void check_btree_root(struct super_block *sb, struct bit_map *map,
+			     struct scoutfs_btree_root *root, check_btree_item_cb item_cb)
+{
+	if (root->height > 0)
+		check_btree_block_ref(sb, map, root->height - 1, &root->ref, item_cb);
+}
+
+static bool check_alloc_extent_item(struct super_block *sb, struct bit_map *map,
+				    struct scoutfs_key *key, void *val, u16 val_len)
+{
+	/* XXX only checking primary blkno items */
+	if (key->sk_zone == SCOUTFS_FREE_EXTENT_BLKNO_ZONE) {
+		check_extent(sb, map, le64_to_cpu(key->skfb_end) - le64_to_cpu(key->skfb_len) + 1,
+				      le64_to_cpu(key->skfb_len));
+		return true;
+	}
+
+	/* otherwise stop iterating over items */
+	return false;
+}
+
+static void check_alloc_root(struct super_block *sb, struct bit_map *map,
+			     struct scoutfs_alloc_root *root)
+{
+	check_btree_root(sb, map, &root->root, check_alloc_extent_item);
+}
+
+static void check_alloc_list_block_ref(struct super_block *sb, struct bit_map *map,
+				       struct scoutfs_block_ref *caller_ref)
+{
+	struct scoutfs_alloc_list_block *lblk;
+	struct scoutfs_block_ref ref;
+	struct scoutfs_block *bl;
+	u32 start;
+	u32 nr;
+	u32 i;
+
+	ref = *caller_ref;
+
+	while ((lblk = read_block_ref(sb, map, &ref, SCOUTFS_BLOCK_MAGIC_ALLOC_LIST, &bl))) {
+
+		start = le32_to_cpu(lblk->start);
+		nr = le32_to_cpu(lblk->nr);
+
+		/* could sort and combine into extents */
+		for (i = 0; i < nr; i++)
+			check_blkno(sb, map, le64_to_cpu(lblk->blknos[start + i]));
+
+		ref = lblk->next;
+		scoutfs_block_put(sb, bl);
+	}
+}
+
+static void check_alloc_list_head(struct super_block *sb, struct bit_map *map,
+				  struct scoutfs_alloc_list_head *lhead)
+{
+	check_alloc_list_block_ref(sb, map, &lhead->ref);
+}
+
+static bool check_log_merge_item(struct super_block *sb, struct bit_map *map,
+				 struct scoutfs_key *key, void *val, u16 val_len)
+{
+	struct scoutfs_log_merge_request *req;
+	struct scoutfs_log_merge_complete *comp;
+	struct scoutfs_log_merge_freeing *fr;
+
+	switch(key->sk_zone) {
+	case SCOUTFS_LOG_MERGE_REQUEST_ZONE:
+		req = val;
+		check_alloc_list_head(sb, map, &req->meta_avail);
+		check_alloc_list_head(sb, map, &req->meta_freed);
+		/* logs_root and root are shared refs */
+		break;
+
+	case SCOUTFS_LOG_MERGE_COMPLETE_ZONE:
+		comp = val;
+		check_alloc_list_head(sb, map, &comp->meta_avail);
+		check_alloc_list_head(sb, map, &comp->meta_freed);
+		/* XXX merged subtree?   hmm. */
+		break;
+
+	case SCOUTFS_LOG_MERGE_FREEING_ZONE:
+		fr = val;
+		check_btree_root(sb, map, &fr->root, NULL);
+		break;
+	}
+
+	return true;
+}
+
+static void check_srch_file_block_ref(struct super_block *sb, struct bit_map *map,
+				      u8 level, struct scoutfs_block_ref *ref)
+{
+	struct scoutfs_block *bl = NULL;
+	struct scoutfs_srch_parent *srp;
+	int i;
+
+	if (level == 0) {
+		check_block_ref(sb, map, ref);
+		return;
+	}
+
+	if (!(srp = read_block_ref(sb, map, ref, SCOUTFS_BLOCK_MAGIC_SRCH_PARENT, &bl)))
+		return;
+
+	for (i = 0; i < SCOUTFS_SRCH_PARENT_REFS; i++)
+		check_srch_file_block_ref(sb, map, level - 1, &srp->refs[i]);
+
+	scoutfs_block_put(sb, bl);
+}
+
+static void check_srch_file(struct super_block *sb, struct bit_map *map,
+			    struct scoutfs_srch_file *sfl)
+{
+	if (sfl->height > 0)
+		check_srch_file_block_ref(sb, map, sfl->height - 1, &sfl->ref);
+}
+
+static bool check_srch_item(struct super_block *sb, struct bit_map *map,
+			    struct scoutfs_key *key, void *val, u16 val_len)
+{
+	struct scoutfs_srch_file *sfl;
+	struct scoutfs_srch_compact *sc;
+
+	switch(key->sk_type) {
+	case SCOUTFS_SRCH_BLOCKS_TYPE:
+	case SCOUTFS_SRCH_LOG_TYPE:
+		sfl = val;
+		check_srch_file(sb, map, sfl);
+		break;
+	case SCOUTFS_SRCH_PENDING_TYPE:
+	case SCOUTFS_SRCH_BUSY_TYPE:
+		sc = val;
+		check_alloc_list_head(sb, map, &sc->meta_avail);
+		check_alloc_list_head(sb, map, &sc->meta_freed);
+		check_srch_file(sb, map, &sc->out);
+		break;
+	}
+
+	return true;
+}
+
+static bool check_log_trees_item(struct super_block *sb, struct bit_map *map,
+				 struct scoutfs_key *key, void *val, u16 val_len)
+{
+	struct scoutfs_log_trees *lt = val;
+
+	check_alloc_list_head(sb, map, &lt->meta_avail);
+	check_alloc_list_head(sb, map, &lt->meta_freed);
+	check_btree_root(sb, map, &lt->item_root, NULL);
+	check_block_ref(sb, map, &lt->bloom_ref);
+	check_btree_root(sb, map, &lt->data_avail.root, NULL);
+	check_btree_root(sb, map, &lt->data_freed.root, NULL);
+	check_srch_file(sb, map, &lt->srch_file);
+
+	return true;
+}
+
+static void check_super(struct super_block *sb, struct bit_map *map,
+			struct scoutfs_super_block *super)
+{
+	check_alloc_root(sb, map, &super->meta_alloc[0]);
+	check_alloc_root(sb, map, &super->meta_alloc[1]);
+	check_btree_root(sb, map, &super->data_alloc.root, NULL);
+	check_alloc_list_head(sb, map, &super->server_meta_avail[0]);
+	check_alloc_list_head(sb, map, &super->server_meta_avail[1]);
+	check_alloc_list_head(sb, map, &super->server_meta_freed[0]);
+	check_alloc_list_head(sb, map, &super->server_meta_freed[1]);
+	check_btree_root(sb, map, &super->fs_root, NULL);
+	check_btree_root(sb, map, &super->logs_root, check_log_trees_item);
+	check_btree_root(sb, map, &super->log_merge, check_log_merge_item);
+	check_btree_root(sb, map, &super->mounted_clients, NULL);
+	check_btree_root(sb, map, &super->srch_root, check_srch_item);
+}
+
+static void check_map(struct super_block *sb, struct bit_map *map)
+{
+	unsigned long nr = find_next_zero_bit(map->addr, map->size, 0);
+
+	warn_once_disable(sb, nr < map->size,
+			  "final map has missing bits, first %lu", nr);
+}
+
+/*
+ * This is called while the persistent block structures are stable.
+ * While we might have to drop stale cache as we read these blocks, we
+ * should be able to walk stable block references from the super.
+ */
+void scoutfs_check_meta_refs(struct super_block *sb, struct scoutfs_super_block *super)
+{
+	static struct bit_map map = {NULL,};
+	unsigned long bytes;
+	u64 size;
+
+	if (!enabled)
+		return;
+
+	size = le64_to_cpu(super->total_meta_blocks);
+
+	if (warn_once_disable(sb, size <= SCOUTFS_META_DEV_START_BLKNO,
+			       "total_meta %llu too small", size) ||
+	    warn_once_disable(sb, size > LONG_MAX,
+			       "total_meta %llu too large", size))
+		return;
+
+	bytes = DIV_ROUND_UP(size, 8);
+	if (size != map.size) {
+		if (map.addr) {
+			vfree(map.addr);
+			map.addr = NULL;
+		}
+
+		map.addr = vmalloc(bytes);
+		if (warn_once_disable(sb, !map.addr, "couldn't alloc %lu byte vmalloc", bytes))
+			return;
+
+		map.size = size;
+	}
+
+	memset(map.addr, 0, bytes);
+	/* initial large block numbers used by padding and 4k super and quorum blocks */
+	bitmap_set(map.addr, 0, SCOUTFS_META_DEV_START_BLKNO);
+
+	check_super(sb, &map, super);
+	check_map(sb, &map);
+
+	if (!enabled)
+		panic("found inconsistent meta refs");
+}
--- a/kmod/src/check.h
+++ b/kmod/src/check.h
@@ -0,0 +1,6 @@
+#ifndef _SCOUTFS_CHECK_H_
+#define _SCOUTFS_CHECK_H_
+
+void scoutfs_check_meta_refs(struct super_block *sb, struct scoutfs_super_block *super);
+
+#endif
--- a/kmod/src/inode.c
+++ b/kmod/src/inode.c
@@ -1482,6 +1482,12 @@ static int remove_index_items(struct super_block *sb, u64 ino,
 * Return an allocated and unused inode number.  Returns -ENOSPC if
 * we're out of inode.
 *
+ * Each parent directory has its own pool of free inode numbers.  Items
+ * are sorted by their inode numbers as they're stored in segments.
+ * This will tend to group together files that are created in a
+ * directory at the same time in segments.  Concurrent creation across
+ * different directories will be stored in their own regions.
+ *
 * Inode numbers are never reclaimed.  If the inode is evicted or we're
 * unmounted the pending inode numbers will be lost.  Asking for a
 * relatively small number from the server each time will tend to
@@ -1491,18 +1497,12 @@ static int remove_index_items(struct super_block *sb, u64 ino,
 int scoutfs_alloc_ino(struct super_block *sb, bool is_dir, u64 *ino_ret)
 {
 	DECLARE_INODE_SB_INFO(sb, inf);
-	struct scoutfs_mount_options opts;
 	struct inode_allocator *ia;
 	u64 ino;
 	u64 nr;
 	int ret;

-	scoutfs_options_read(sb, &opts);
-
-	if (is_dir && opts.ino_alloc_per_lock == SCOUTFS_LOCK_INODE_GROUP_NR)
-		ia = &inf->dir_ino_alloc;
-	else
-		ia = &inf->ino_alloc;
+	ia = is_dir ? &inf->dir_ino_alloc : &inf->ino_alloc;

 	spin_lock(&ia->lock);

@@ -1523,17 +1523,6 @@ int scoutfs_alloc_ino(struct super_block *sb, bool is_dir, u64 *ino_ret)
 	*ino_ret = ia->ino++;
 	ia->nr--;

-	if (opts.ino_alloc_per_lock != SCOUTFS_LOCK_INODE_GROUP_NR) {
-		nr = ia->ino & SCOUTFS_LOCK_INODE_GROUP_MASK;
-		if (nr >= opts.ino_alloc_per_lock) {
-			nr = SCOUTFS_LOCK_INODE_GROUP_NR - nr;
-			if (nr > ia->nr)
-				nr = ia->nr;
-			ia->ino += nr;
-			ia->nr -= nr;
-		}
-	}
-
 	spin_unlock(&ia->lock);
 	ret = 0;
 out:
--- a/kmod/src/ioctl.c
+++ b/kmod/src/ioctl.c
@@ -441,6 +441,8 @@ static long scoutfs_ioc_data_wait_err(struct file *file, unsigned long arg)

 	if (!S_ISREG(inode->i_mode)) {
 		ret = -EINVAL;
+	} else if (scoutfs_inode_data_version(inode) != args.data_version) {
+		ret = -ESTALE;
 	} else {
 		ret = scoutfs_data_wait_err(inode, sblock, eblock, args.op,
 					    args.err);
--- a/kmod/src/ioctl.h
+++ b/kmod/src/ioctl.h
@@ -366,15 +366,10 @@ struct scoutfs_ioctl_statfs_more {
 *
 * Find current waiters that match the inode, op, and block range to wake
 * up and return an error.
- *
- * (*) ca. v1.25 and earlier required that the data_version passed match
- * that of the waiter, but this check is removed. It was never needed
- * because no data is modified during this ioctl. Any data_version value
- * here is thus since then ignored.
 */
 struct scoutfs_ioctl_data_wait_err {
 	__u64 ino;
-	__u64 data_version; /* Ignored, see above (*) */
+	__u64 data_version;
 	__u64 offset;
 	__u64 count;
 	__u64 op;
--- a/kmod/src/msg.h
+++ b/kmod/src/msg.h
@@ -35,12 +35,6 @@ do {									\
 	}								\
 } while (0)								\

-#define scoutfs_bug_on_err(sb, err, fmt, args...) \
-do { \
-	__typeof__(err) _err = (err); \
-	scoutfs_bug_on(sb, _err < 0 && _err != -ENOLINK, fmt, ##args); \
-} while (0)
-
 /*
 * Each message is only generated once per volume.  Remounting resets
 * the messages.
--- a/kmod/src/options.c
+++ b/kmod/src/options.c
@@ -33,7 +33,6 @@ enum {
 	Opt_acl,
 	Opt_data_prealloc_blocks,
 	Opt_data_prealloc_contig_only,
-	Opt_ino_alloc_per_lock,
 	Opt_log_merge_wait_timeout_ms,
 	Opt_metadev_path,
 	Opt_noacl,
@@ -48,7 +47,6 @@ static const match_table_t tokens = {
 	{Opt_acl, "acl"},
 	{Opt_data_prealloc_blocks, "data_prealloc_blocks=%s"},
 	{Opt_data_prealloc_contig_only, "data_prealloc_contig_only=%s"},
-	{Opt_ino_alloc_per_lock, "ino_alloc_per_lock=%s"},
 	{Opt_log_merge_wait_timeout_ms, "log_merge_wait_timeout_ms=%s"},
 	{Opt_metadev_path, "metadev_path=%s"},
 	{Opt_noacl, "noacl"},
@@ -138,7 +136,6 @@ static void init_default_options(struct scoutfs_mount_options *opts)

 	opts->data_prealloc_blocks = SCOUTFS_DATA_PREALLOC_DEFAULT_BLOCKS;
 	opts->data_prealloc_contig_only = 1;
-	opts->ino_alloc_per_lock = SCOUTFS_LOCK_INODE_GROUP_NR;
 	opts->log_merge_wait_timeout_ms = DEFAULT_LOG_MERGE_WAIT_TIMEOUT_MS;
 	opts->orphan_scan_delay_ms = -1;
 	opts->quorum_heartbeat_timeout_ms = SCOUTFS_QUORUM_DEF_HB_TIMEO_MS;
@@ -241,18 +238,6 @@ static int parse_options(struct super_block *sb, char *options, struct scoutfs_m
 			opts->data_prealloc_contig_only = nr;
 			break;

-		case Opt_ino_alloc_per_lock:
-			ret = match_int(args, &nr);
-			if (ret < 0 || nr < 1 || nr > SCOUTFS_LOCK_INODE_GROUP_NR) {
-				scoutfs_err(sb, "invalid ino_alloc_per_lock option, must be between 1 and %u",
-					    SCOUTFS_LOCK_INODE_GROUP_NR);
-				if (ret == 0)
-					ret = -EINVAL;
-				return ret;
-			}
-			opts->ino_alloc_per_lock = nr;
-			break;
-
 		case Opt_tcp_keepalive_timeout_ms:
 			ret = match_int(args, &nr);
 			ret = verify_tcp_keepalive_timeout_ms(sb, ret, nr);
@@ -408,7 +393,6 @@ int scoutfs_options_show(struct seq_file *seq, struct dentry *root)
 		seq_puts(seq, ",acl");
 	seq_printf(seq, ",data_prealloc_blocks=%llu", opts.data_prealloc_blocks);
 	seq_printf(seq, ",data_prealloc_contig_only=%u", opts.data_prealloc_contig_only);
-	seq_printf(seq, ",ino_alloc_per_lock=%u", opts.ino_alloc_per_lock);
 	seq_printf(seq, ",metadev_path=%s", opts.metadev_path);
 	if (!is_acl)
 		seq_puts(seq, ",noacl");
@@ -497,45 +481,6 @@ static ssize_t data_prealloc_contig_only_store(struct kobject *kobj, struct kobj
 }
 SCOUTFS_ATTR_RW(data_prealloc_contig_only);

-static ssize_t ino_alloc_per_lock_show(struct kobject *kobj, struct kobj_attribute *attr,
-					 char *buf)
-{
-	struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj);
-	struct scoutfs_mount_options opts;
-
-	scoutfs_options_read(sb, &opts);
-
-	return snprintf(buf, PAGE_SIZE, "%u", opts.ino_alloc_per_lock);
-}
-static ssize_t ino_alloc_per_lock_store(struct kobject *kobj, struct kobj_attribute *attr,
-					  const char *buf, size_t count)
-{
-	struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj);
-	DECLARE_OPTIONS_INFO(sb, optinf);
-	char nullterm[20]; /* more than enough for octal -U32_MAX */
-	long val;
-	int len;
-	int ret;
-
-	len = min(count, sizeof(nullterm) - 1);
-	memcpy(nullterm, buf, len);
-	nullterm[len] = '\0';
-
-	ret = kstrtol(nullterm, 0, &val);
-	if (ret < 0 || val < 1 || val > SCOUTFS_LOCK_INODE_GROUP_NR) {
-		scoutfs_err(sb, "invalid ino_alloc_per_lock option, must be between 1 and %u",
-			    SCOUTFS_LOCK_INODE_GROUP_NR);
-		return -EINVAL;
-	}
-
-	write_seqlock(&optinf->seqlock);
-	optinf->opts.ino_alloc_per_lock = val;
-	write_sequnlock(&optinf->seqlock);
-
-	return count;
-}
-SCOUTFS_ATTR_RW(ino_alloc_per_lock);
-
 static ssize_t log_merge_wait_timeout_ms_show(struct kobject *kobj, struct kobj_attribute *attr,
 						char *buf)
 {
@@ -676,7 +621,6 @@ SCOUTFS_ATTR_RO(quorum_slot_nr);
 static struct attribute *options_attrs[] = {
 	SCOUTFS_ATTR_PTR(data_prealloc_blocks),
 	SCOUTFS_ATTR_PTR(data_prealloc_contig_only),
-	SCOUTFS_ATTR_PTR(ino_alloc_per_lock),
 	SCOUTFS_ATTR_PTR(log_merge_wait_timeout_ms),
 	SCOUTFS_ATTR_PTR(metadev_path),
 	SCOUTFS_ATTR_PTR(orphan_scan_delay_ms),
--- a/kmod/src/options.h
+++ b/kmod/src/options.h
@@ -8,7 +8,6 @@
 struct scoutfs_mount_options {
 	u64 data_prealloc_blocks;
 	bool data_prealloc_contig_only;
-	unsigned int ino_alloc_per_lock;
 	unsigned int log_merge_wait_timeout_ms;
 	char *metadev_path;
 	unsigned int orphan_scan_delay_ms;
--- a/kmod/src/server.c
+++ b/kmod/src/server.c
@@ -994,11 +994,10 @@ static int for_each_rid_last_lt(struct super_block *sb, struct scoutfs_btree_roo
 }

 /*
- * Log merge range items are stored at the starting fs key of the range
- * with the zone overwritten to indicate the log merge item type.  This
- * day0 mistake loses sorting information for items in the different
- * zones in the fs root, so the range items aren't strictly sorted by
- * the starting key of their range.
+ * Log merge range items are stored at the starting fs key of the range.
+ * The only fs key field that doesn't hold information is the zone, so
+ * we use the zone to differentiate all types that we store in the log
+ * merge tree.
 */
 static void init_log_merge_key(struct scoutfs_key *key, u8 zone, u64 first,
 			       u64 second)
@@ -1030,51 +1029,6 @@ static int next_log_merge_item_key(struct super_block *sb, struct scoutfs_btree_
 	return ret;
 }

-/*
- * The range items aren't sorted by their range.start because
- * _RANGE_ZONE clobbers the range's zone.  We sweep all the items and
- * find the range with the next least starting key that's greater than
- * the caller's starting key.  We have to be careful to iterate over the
- * log_merge tree keys because the ranges can overlap as they're mapped
- * to the log_merge keys by clobbering their zone.
- */
-static int next_log_merge_range(struct super_block *sb, struct scoutfs_btree_root *root,
-				struct scoutfs_key *start, struct scoutfs_log_merge_range *rng)
-{
-	struct scoutfs_log_merge_range *next;
-	SCOUTFS_BTREE_ITEM_REF(iref);
-	struct scoutfs_key key;
-	int ret;
-
-	key = *start;
-	key.sk_zone = SCOUTFS_LOG_MERGE_RANGE_ZONE;
-	scoutfs_key_set_ones(&rng->start);
-
-	do {
-		ret = scoutfs_btree_next(sb, root, &key, &iref);
-		if (ret == 0) {
-			if (iref.key->sk_zone != SCOUTFS_LOG_MERGE_RANGE_ZONE) {
-				ret = -ENOENT;
-			} else if (iref.val_len != sizeof(struct scoutfs_log_merge_range)) {
-				ret = -EIO;
-			} else {
-				next = iref.val;
-				if (scoutfs_key_compare(&next->start, &rng->start) < 0 &&
-				    scoutfs_key_compare(&next->start, start) >= 0)
-					*rng = *next;
-				key = *iref.key;
-				scoutfs_key_inc(&key);
-			}
-			scoutfs_btree_put_iref(&iref);
-		}
-	} while (ret == 0);
-
-	if (ret == -ENOENT && !scoutfs_key_is_ones(&rng->start))
-		ret = 0;
-
-	return ret;
-}
-
 static int next_log_merge_item(struct super_block *sb,
 			       struct scoutfs_btree_root *root,
 			       u8 zone, u64 first, u64 second,
@@ -1728,7 +1682,6 @@ static int server_commit_log_trees(struct super_block *sb,
 	int ret;

 	if (arg_len != sizeof(struct scoutfs_log_trees)) {
-		err_str = "invalid message log_trees size";
 		ret = -EINVAL;
 		goto out;
 	}
@@ -1792,7 +1745,7 @@ static int server_commit_log_trees(struct super_block *sb,

 	ret = scoutfs_btree_update(sb, &server->alloc, &server->wri,
 				   &super->logs_root, &key, &lt, sizeof(lt));
-	BUG_ON(ret < 0); /* dirtying should have guaranteed success, srch item inconsistent */
+	BUG_ON(ret < 0); /* dirtying should have guaranteed success */
 	if (ret < 0)
 		err_str = "updating log trees item";

@@ -1800,10 +1753,11 @@ unlock:
 	mutex_unlock(&server->logs_mutex);

 	ret = server_apply_commit(sb, &hold, ret);
-out:
 	if (ret < 0)
 		scoutfs_err(sb, "server error %d committing client logs for rid %016llx, nr %llu: %s",
 			    ret, rid, le64_to_cpu(lt.nr), err_str);
+out:
+	WARN_ON_ONCE(ret < 0);
 	return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0);
 }

@@ -2140,7 +2094,7 @@ static int server_srch_get_compact(struct super_block *sb,

 apply:
 	ret = server_apply_commit(sb, &hold, ret);
-	WARN_ON_ONCE(ret < 0 && ret != -ENOENT && ret != -ENOLINK); /* XXX leaked busy item */
+	WARN_ON_ONCE(ret < 0 && ret != -ENOENT); /* XXX leaked busy item */
 out:
 	ret = scoutfs_net_response(sb, conn, cmd, id, ret,
 				   sc, sizeof(struct scoutfs_srch_compact));
@@ -2518,9 +2472,10 @@ out:
 		}
 	}

-	/* inconsistent */
-	scoutfs_bug_on_err(sb, ret,
-			   "server error %d splicing log merge completion: %s", ret, err_str);
+	if (ret < 0)
+		scoutfs_err(sb, "server error %d splicing log merge completion: %s", ret, err_str);
+
+	BUG_ON(ret); /* inconsistent */

 	return ret ?: einprogress;
 }
@@ -2765,7 +2720,10 @@ restart:

 	/* find the next range, always checking for splicing */
 	for (;;) {
-		ret = next_log_merge_range(sb, &super->log_merge, &stat.next_range_key, &rng);
+		key = stat.next_range_key;
+		key.sk_zone = SCOUTFS_LOG_MERGE_RANGE_ZONE;
+		ret = next_log_merge_item_key(sb, &super->log_merge, SCOUTFS_LOG_MERGE_RANGE_ZONE,
+					      &key, &rng, sizeof(rng));
 		if (ret < 0 && ret != -ENOENT) {
 			err_str = "finding merge range item";
 			goto out;
@@ -3036,13 +2994,7 @@ static int server_commit_log_merge(struct super_block *sb,
 				  SCOUTFS_LOG_MERGE_STATUS_ZONE, 0, 0,
 				  &stat, sizeof(stat));
 	if (ret < 0) {
-		/*
-		 * During a retransmission, it's possible that the server
-		 * already committed and resolved this log merge. ENOENT
-		 * is expected in that case.
-		 */
-		if (ret != -ENOENT)
-			err_str = "getting merge status item";
+		err_str = "getting merge status item";
 		goto out;
 	}

--- a/tests/README.md
+++ b/tests/README.md
@@ -117,7 +117,6 @@ used during the test.
 | T\_NR\_MOUNTS    | number of mounts     | -n              | 3                 |
 | T\_O[0-9]        | mount options        | created per run | -o server\_addr=  |
 | T\_QUORUM        | quorum count         | -q              | 2                 |
-| T\_EXTRA         | per-test file dir    | revision ctled  | tests/extra/t     |
 | T\_TMP           | per-test tmp prefix  | made for test   | results/tmp/t/tmp |
 | T\_TMPDIR        | per-test tmp dir dir | made for test   | results/tmp/t     |

--- a/tests/extra/xfstests/expected-results
+++ b/tests/extra/xfstests/expected-results
@@ -1,882 +0,0 @@
-Ran:
-generic/001
-generic/002
-generic/004
-generic/005
-generic/006
-generic/007
-generic/008
-generic/009
-generic/011
-generic/012
-generic/013
-generic/014
-generic/015
-generic/016
-generic/018
-generic/020
-generic/021
-generic/022
-generic/023
-generic/024
-generic/025
-generic/026
-generic/028
-generic/029
-generic/030
-generic/031
-generic/032
-generic/033
-generic/034
-generic/035
-generic/037
-generic/039
-generic/040
-generic/041
-generic/050
-generic/052
-generic/053
-generic/056
-generic/057
-generic/058
-generic/059
-generic/060
-generic/061
-generic/062
-generic/063
-generic/064
-generic/065
-generic/066
-generic/067
-generic/069
-generic/070
-generic/071
-generic/073
-generic/076
-generic/078
-generic/079
-generic/080
-generic/081
-generic/082
-generic/084
-generic/086
-generic/087
-generic/088
-generic/090
-generic/091
-generic/092
-generic/094
-generic/096
-generic/097
-generic/098
-generic/099
-generic/101
-generic/104
-generic/105
-generic/106
-generic/107
-generic/110
-generic/111
-generic/113
-generic/114
-generic/115
-generic/116
-generic/117
-generic/118
-generic/119
-generic/120
-generic/121
-generic/122
-generic/123
-generic/124
-generic/126
-generic/128
-generic/129
-generic/130
-generic/131
-generic/134
-generic/135
-generic/136
-generic/138
-generic/139
-generic/140
-generic/141
-generic/142
-generic/143
-generic/144
-generic/145
-generic/146
-generic/147
-generic/148
-generic/149
-generic/150
-generic/151
-generic/152
-generic/153
-generic/154
-generic/155
-generic/156
-generic/157
-generic/158
-generic/159
-generic/160
-generic/161
-generic/162
-generic/163
-generic/169
-generic/171
-generic/172
-generic/173
-generic/174
-generic/177
-generic/178
-generic/179
-generic/180
-generic/181
-generic/182
-generic/183
-generic/184
-generic/185
-generic/188
-generic/189
-generic/190
-generic/191
-generic/193
-generic/194
-generic/195
-generic/196
-generic/197
-generic/198
-generic/199
-generic/200
-generic/201
-generic/202
-generic/203
-generic/205
-generic/206
-generic/207
-generic/210
-generic/211
-generic/212
-generic/214
-generic/215
-generic/216
-generic/217
-generic/218
-generic/219
-generic/220
-generic/221
-generic/222
-generic/223
-generic/225
-generic/227
-generic/228
-generic/229
-generic/230
-generic/235
-generic/236
-generic/237
-generic/238
-generic/240
-generic/244
-generic/245
-generic/246
-generic/247
-generic/248
-generic/249
-generic/250
-generic/252
-generic/253
-generic/254
-generic/255
-generic/256
-generic/257
-generic/258
-generic/259
-generic/260
-generic/261
-generic/262
-generic/263
-generic/264
-generic/265
-generic/266
-generic/267
-generic/268
-generic/271
-generic/272
-generic/276
-generic/277
-generic/278
-generic/279
-generic/281
-generic/282
-generic/283
-generic/284
-generic/286
-generic/287
-generic/288
-generic/289
-generic/290
-generic/291
-generic/292
-generic/293
-generic/294
-generic/295
-generic/296
-generic/301
-generic/302
-generic/303
-generic/304
-generic/305
-generic/306
-generic/307
-generic/308
-generic/309
-generic/312
-generic/313
-generic/314
-generic/315
-generic/316
-generic/317
-generic/319
-generic/322
-generic/324
-generic/325
-generic/326
-generic/327
-generic/328
-generic/329
-generic/330
-generic/331
-generic/332
-generic/335
-generic/336
-generic/337
-generic/341
-generic/342
-generic/343
-generic/346
-generic/348
-generic/353
-generic/355
-generic/358
-generic/359
-generic/360
-generic/361
-generic/362
-generic/363
-generic/364
-generic/365
-generic/366
-generic/367
-generic/368
-generic/369
-generic/370
-generic/371
-generic/372
-generic/373
-generic/374
-generic/375
-generic/376
-generic/377
-generic/378
-generic/379
-generic/380
-generic/381
-generic/382
-generic/383
-generic/384
-generic/385
-generic/386
-generic/389
-generic/391
-generic/392
-generic/393
-generic/394
-generic/395
-generic/396
-generic/397
-generic/398
-generic/400
-generic/401
-generic/402
-generic/403
-generic/404
-generic/406
-generic/407
-generic/408
-generic/412
-generic/413
-generic/414
-generic/417
-generic/419
-generic/420
-generic/421
-generic/422
-generic/424
-generic/425
-generic/426
-generic/427
-generic/428
-generic/436
-generic/437
-generic/439
-generic/440
-generic/443
-generic/445
-generic/446
-generic/448
-generic/449
-generic/450
-generic/451
-generic/452
-generic/453
-generic/454
-generic/456
-generic/458
-generic/460
-generic/462
-generic/463
-generic/465
-generic/466
-generic/468
-generic/469
-generic/470
-generic/471
-generic/474
-generic/477
-generic/478
-generic/479
-generic/480
-generic/481
-generic/483
-generic/485
-generic/486
-generic/487
-generic/488
-generic/489
-generic/490
-generic/491
-generic/492
-generic/498
-generic/499
-generic/501
-generic/502
-generic/503
-generic/504
-generic/505
-generic/506
-generic/507
-generic/508
-generic/509
-generic/510
-generic/511
-generic/512
-generic/513
-generic/514
-generic/515
-generic/516
-generic/517
-generic/518
-generic/519
-generic/520
-generic/523
-generic/524
-generic/525
-generic/526
-generic/527
-generic/528
-generic/529
-generic/530
-generic/531
-generic/533
-generic/534
-generic/535
-generic/536
-generic/537
-generic/538
-generic/539
-generic/540
-generic/541
-generic/542
-generic/543
-generic/544
-generic/545
-generic/546
-generic/547
-generic/548
-generic/549
-generic/550
-generic/552
-generic/553
-generic/555
-generic/556
-generic/557
-generic/566
-generic/567
-generic/571
-generic/572
-generic/573
-generic/574
-generic/575
-generic/576
-generic/577
-generic/578
-generic/580
-generic/581
-generic/582
-generic/583
-generic/584
-generic/586
-generic/587
-generic/588
-generic/591
-generic/592
-generic/593
-generic/594
-generic/595
-generic/596
-generic/597
-generic/598
-generic/599
-generic/600
-generic/601
-generic/602
-generic/603
-generic/604
-generic/605
-generic/606
-generic/607
-generic/608
-generic/609
-generic/610
-generic/611
-generic/612
-generic/613
-generic/614
-generic/618
-generic/621
-generic/623
-generic/624
-generic/625
-generic/626
-generic/628
-generic/629
-generic/630
-generic/632
-generic/634
-generic/635
-generic/637
-generic/638
-generic/639
-generic/640
-generic/644
-generic/645
-generic/646
-generic/647
-generic/651
-generic/652
-generic/653
-generic/654
-generic/655
-generic/657
-generic/658
-generic/659
-generic/660
-generic/661
-generic/662
-generic/663
-generic/664
-generic/665
-generic/666
-generic/667
-generic/668
-generic/669
-generic/673
-generic/674
-generic/675
-generic/676
-generic/677
-generic/678
-generic/679
-generic/680
-generic/681
-generic/682
-generic/683
-generic/684
-generic/685
-generic/686
-generic/687
-generic/688
-generic/689
-shared/002
-shared/032
-Not
-run:
-generic/008
-generic/009
-generic/012
-generic/015
-generic/016
-generic/018
-generic/021
-generic/022
-generic/025
-generic/026
-generic/031
-generic/033
-generic/050
-generic/052
-generic/058
-generic/059
-generic/060
-generic/061
-generic/063
-generic/064
-generic/078
-generic/079
-generic/081
-generic/082
-generic/091
-generic/094
-generic/096
-generic/110
-generic/111
-generic/113
-generic/114
-generic/115
-generic/116
-generic/118
-generic/119
-generic/121
-generic/122
-generic/123
-generic/128
-generic/130
-generic/134
-generic/135
-generic/136
-generic/138
-generic/139
-generic/140
-generic/142
-generic/143
-generic/144
-generic/145
-generic/146
-generic/147
-generic/148
-generic/149
-generic/150
-generic/151
-generic/152
-generic/153
-generic/154
-generic/155
-generic/156
-generic/157
-generic/158
-generic/159
-generic/160
-generic/161
-generic/162
-generic/163
-generic/171
-generic/172
-generic/173
-generic/174
-generic/177
-generic/178
-generic/179
-generic/180
-generic/181
-generic/182
-generic/183
-generic/185
-generic/188
-generic/189
-generic/190
-generic/191
-generic/193
-generic/194
-generic/195
-generic/196
-generic/197
-generic/198
-generic/199
-generic/200
-generic/201
-generic/202
-generic/203
-generic/205
-generic/206
-generic/207
-generic/210
-generic/211
-generic/212
-generic/214
-generic/216
-generic/217
-generic/218
-generic/219
-generic/220
-generic/222
-generic/223
-generic/225
-generic/227
-generic/229
-generic/230
-generic/235
-generic/238
-generic/240
-generic/244
-generic/250
-generic/252
-generic/253
-generic/254
-generic/255
-generic/256
-generic/259
-generic/260
-generic/261
-generic/262
-generic/263
-generic/264
-generic/265
-generic/266
-generic/267
-generic/268
-generic/271
-generic/272
-generic/276
-generic/277
-generic/278
-generic/279
-generic/281
-generic/282
-generic/283
-generic/284
-generic/287
-generic/288
-generic/289
-generic/290
-generic/291
-generic/292
-generic/293
-generic/295
-generic/296
-generic/301
-generic/302
-generic/303
-generic/304
-generic/305
-generic/312
-generic/314
-generic/316
-generic/317
-generic/324
-generic/326
-generic/327
-generic/328
-generic/329
-generic/330
-generic/331
-generic/332
-generic/353
-generic/355
-generic/358
-generic/359
-generic/361
-generic/362
-generic/363
-generic/364
-generic/365
-generic/366
-generic/367
-generic/368
-generic/369
-generic/370
-generic/371
-generic/372
-generic/373
-generic/374
-generic/378
-generic/379
-generic/380
-generic/381
-generic/382
-generic/383
-generic/384
-generic/385
-generic/386
-generic/391
-generic/392
-generic/395
-generic/396
-generic/397
-generic/398
-generic/400
-generic/402
-generic/404
-generic/406
-generic/407
-generic/408
-generic/412
-generic/413
-generic/414
-generic/417
-generic/419
-generic/420
-generic/421
-generic/422
-generic/424
-generic/425
-generic/427
-generic/439
-generic/440
-generic/446
-generic/449
-generic/450
-generic/451
-generic/453
-generic/454
-generic/456
-generic/458
-generic/462
-generic/463
-generic/465
-generic/466
-generic/468
-generic/469
-generic/470
-generic/471
-generic/474
-generic/485
-generic/487
-generic/488
-generic/491
-generic/492
-generic/499
-generic/501
-generic/503
-generic/505
-generic/506
-generic/507
-generic/508
-generic/511
-generic/513
-generic/514
-generic/515
-generic/516
-generic/517
-generic/518
-generic/519
-generic/520
-generic/528
-generic/530
-generic/536
-generic/537
-generic/538
-generic/539
-generic/540
-generic/541
-generic/542
-generic/543
-generic/544
-generic/545
-generic/546
-generic/548
-generic/549
-generic/550
-generic/552
-generic/553
-generic/555
-generic/556
-generic/566
-generic/567
-generic/572
-generic/573
-generic/574
-generic/575
-generic/576
-generic/577
-generic/578
-generic/580
-generic/581
-generic/582
-generic/583
-generic/584
-generic/586
-generic/587
-generic/588
-generic/591
-generic/592
-generic/593
-generic/594
-generic/595
-generic/596
-generic/597
-generic/598
-generic/599
-generic/600
-generic/601
-generic/602
-generic/603
-generic/605
-generic/606
-generic/607
-generic/608
-generic/609
-generic/610
-generic/612
-generic/613
-generic/621
-generic/623
-generic/624
-generic/625
-generic/626
-generic/628
-generic/629
-generic/630
-generic/635
-generic/644
-generic/645
-generic/646
-generic/647
-generic/651
-generic/652
-generic/653
-generic/654
-generic/655
-generic/657
-generic/658
-generic/659
-generic/660
-generic/661
-generic/662
-generic/663
-generic/664
-generic/665
-generic/666
-generic/667
-generic/668
-generic/669
-generic/673
-generic/674
-generic/675
-generic/677
-generic/678
-generic/679
-generic/680
-generic/681
-generic/682
-generic/683
-generic/684
-generic/685
-generic/686
-generic/687
-generic/688
-generic/689
-shared/002
-shared/032
-Passed all 512 tests
--- a/tests/extra/xfstests/local.exclude
+++ b/tests/extra/xfstests/local.exclude
@@ -1,44 +0,0 @@
-generic/003	# missing atime update in buffered read
-generic/075	# file content mismatch failures (fds, etc)
-generic/103	# enospc causes trans commit failures
-generic/108	# mount fails on failing device?
-generic/112	# file content mismatch failures (fds, etc)
-generic/213	# enospc causes trans commit failures
-generic/318	# can't support user namespaces until v5.11
-generic/321	# requires selinux enabled for '+' in ls?
-generic/338	# BUG_ON update inode error handling
-generic/347	# _dmthin_mount doesn't work?
-generic/356	# swap
-generic/357	# swap
-generic/409	# bind mounts not scripted yet
-generic/410	# bind mounts not scripted yet
-generic/411	# bind mounts not scripted yet
-generic/423	# symlink inode size is strlen() + 1 on scoutfs
-generic/430	# xfs_io copy_range missing in el7
-generic/431	# xfs_io copy_range missing in el7
-generic/432	# xfs_io copy_range missing in el7
-generic/433	# xfs_io copy_range missing in el7
-generic/434	# xfs_io copy_range missing in el7
-generic/441	# dm-mapper
-generic/444	# el9's posix_acl_update_mode is buggy ?
-generic/467	# open_by_handle ESTALE
-generic/472	# swap
-generic/484	# dm-mapper
-generic/493	# swap
-generic/494	# swap
-generic/495	# swap
-generic/496	# swap
-generic/497	# swap
-generic/532	# xfs_io statx attrib_mask missing in el7
-generic/554	# swap
-generic/563	# cgroup+loopdev
-generic/564	# xfs_io copy_range missing in el7
-generic/565	# xfs_io copy_range missing in el7
-generic/568	# falloc not resulting in block count increase
-generic/569	# swap
-generic/570	# swap
-generic/620	# dm-hugedisk
-generic/633	# id-mapped mounts missing in el7
-generic/636	# swap
-generic/641	# swap
-generic/643	# swap
--- a/tests/fenced-local-force-unmount.sh
+++ b/tests/fenced-local-force-unmount.sh
@@ -8,33 +8,36 @@

 echo "$0 running rid '$SCOUTFS_FENCED_REQ_RID' ip '$SCOUTFS_FENCED_REQ_IP' args '$@'"

-echo_fail() {
-	echo "$@" >&2
+log() {
+	echo "$@" > /dev/stderr
 	exit 1
 }

-# silence error messages
-quiet_cat()
-{
-	cat "$@" 2>/dev/null
+echo_fail() {
+	echo "$@" > /dev/stderr
+	exit 1
 }

 rid="$SCOUTFS_FENCED_REQ_RID"

-shopt -s nullglob
 for fs in /sys/fs/scoutfs/*; do
-	fs_rid="$(quiet_cat $fs/rid)"
-	nr="$(quiet_cat $fs/data_device_maj_min)"
-	[ ! -d "$fs" -o "$fs_rid" != "$rid" ] && continue
+	[ ! -d "$fs" ] && continue

-	mnt=$(findmnt -l -n -t scoutfs -o TARGET -S $nr)
-	[ -z "$mnt" ] && continue
-
-	if ! umount -qf "$mnt"; then
-		if [ -d "$fs" ]; then
-			echo_fail "umount -qf $mnt failed"
-		fi
+	fs_rid="$(cat $fs/rid)" || \
+		echo_fail "failed to get rid in $fs"
+	if [ "$fs_rid" != "$rid" ]; then
+		continue
 	fi
+
+	nr="$(cat $fs/data_device_maj_min)" || \
+		echo_fail "failed to get data device major:minor in $fs"
+
+	mnts=$(findmnt -l -n -t scoutfs -o TARGET -S $nr) || \
+		echo_fail "findmnt -t scoutfs -S $nr failed"
+	for mnt in $mnts; do
+		umount -f "$mnt" || \
+			echo_fail "umout -f $mnt failed"
+	done
 done

 exit 0
--- a/tests/funcs/exec.sh
+++ b/tests/funcs/exec.sh
@@ -64,27 +64,21 @@ t_rc()
 }

 #
-# As run, stdout/err are redirected to a file that will be compared with
-# the stored expected golden output of the test.  This redirects
-# stdout/err in the script to stdout of the invoking run-test.  It's
-# intended to give visible output of tests without being included in the
-# golden output.
+# redirect test output back to the output of the invoking script intead
+# of the compared output.
 #
-# (see the goofy "exec" fd manipulation in the main run-tests as it runs
-# each test)
-#
-t_stdout_invoked()
+t_restore_output()
 {
 	exec >&6 2>&1
 }

 #
-# This undoes t_stdout_invokved, returning the test's stdout/err to the
-# output file as it was when it was launched.
+# redirect a command's output back to the compared output after the
+# test has restored its output
 #
-t_stdout_compare()
+t_compare_output()
 {
-	exec >&7 2>&1
+	"$@" >&7 2>&1
 }

 #
--- a/tests/funcs/filter.sh
+++ b/tests/funcs/filter.sh
@@ -121,7 +121,6 @@ t_filter_dmesg()

 	# in debugging kernels we can slow things down a bit
 	re="$re|hrtimer: interrupt took .*"
-	re="$re|clocksource: Long readout interval"

 	# fencing tests force unmounts and trigger timeouts
 	re="$re|scoutfs .* forcing unmount"
@@ -167,9 +166,6 @@ t_filter_dmesg()
 	# perf warning that it adjusted sample rate
 	re="$re|perf: interrupt took too long.*lowering kernel.perf_event_max_sample_rate.*"

-	# some ci test guests are unresponsive
-	re="$re|longest quorum heartbeat .* delay"
-
 	egrep -v "($re)" | \
 		ignore_harmless_unwind_kasan_stack_oob
 }
--- a/tests/golden/xfstests
+++ b/tests/golden/xfstests
@@ -0,0 +1,882 @@
+Ran:
+generic/001
+generic/002
+generic/004
+generic/005
+generic/006
+generic/007
+generic/008
+generic/009
+generic/011
+generic/012
+generic/013
+generic/014
+generic/015
+generic/016
+generic/018
+generic/020
+generic/021
+generic/022
+generic/023
+generic/024
+generic/025
+generic/026
+generic/028
+generic/029
+generic/030
+generic/031
+generic/032
+generic/033
+generic/034
+generic/035
+generic/037
+generic/039
+generic/040
+generic/041
+generic/050
+generic/052
+generic/053
+generic/056
+generic/057
+generic/058
+generic/059
+generic/060
+generic/061
+generic/062
+generic/063
+generic/064
+generic/065
+generic/066
+generic/067
+generic/069
+generic/070
+generic/071
+generic/073
+generic/076
+generic/078
+generic/079
+generic/080
+generic/081
+generic/082
+generic/084
+generic/086
+generic/087
+generic/088
+generic/090
+generic/091
+generic/092
+generic/094
+generic/096
+generic/097
+generic/098
+generic/099
+generic/101
+generic/104
+generic/105
+generic/106
+generic/107
+generic/110
+generic/111
+generic/113
+generic/114
+generic/115
+generic/116
+generic/117
+generic/118
+generic/119
+generic/120
+generic/121
+generic/122
+generic/123
+generic/124
+generic/126
+generic/128
+generic/129
+generic/130
+generic/131
+generic/134
+generic/135
+generic/136
+generic/138
+generic/139
+generic/140
+generic/141
+generic/142
+generic/143
+generic/144
+generic/145
+generic/146
+generic/147
+generic/148
+generic/149
+generic/150
+generic/151
+generic/152
+generic/153
+generic/154
+generic/155
+generic/156
+generic/157
+generic/158
+generic/159
+generic/160
+generic/161
+generic/162
+generic/163
+generic/169
+generic/171
+generic/172
+generic/173
+generic/174
+generic/177
+generic/178
+generic/179
+generic/180
+generic/181
+generic/182
+generic/183
+generic/184
+generic/185
+generic/188
+generic/189
+generic/190
+generic/191
+generic/193
+generic/194
+generic/195
+generic/196
+generic/197
+generic/198
+generic/199
+generic/200
+generic/201
+generic/202
+generic/203
+generic/205
+generic/206
+generic/207
+generic/210
+generic/211
+generic/212
+generic/214
+generic/215
+generic/216
+generic/217
+generic/218
+generic/219
+generic/220
+generic/221
+generic/222
+generic/223
+generic/225
+generic/227
+generic/228
+generic/229
+generic/230
+generic/235
+generic/236
+generic/237
+generic/238
+generic/240
+generic/244
+generic/245
+generic/246
+generic/247
+generic/248
+generic/249
+generic/250
+generic/252
+generic/253
+generic/254
+generic/255
+generic/256
+generic/257
+generic/258
+generic/259
+generic/260
+generic/261
+generic/262
+generic/263
+generic/264
+generic/265
+generic/266
+generic/267
+generic/268
+generic/271
+generic/272
+generic/276
+generic/277
+generic/278
+generic/279
+generic/281
+generic/282
+generic/283
+generic/284
+generic/286
+generic/287
+generic/288
+generic/289
+generic/290
+generic/291
+generic/292
+generic/293
+generic/294
+generic/295
+generic/296
+generic/301
+generic/302
+generic/303
+generic/304
+generic/305
+generic/306
+generic/307
+generic/308
+generic/309
+generic/312
+generic/313
+generic/314
+generic/315
+generic/316
+generic/317
+generic/319
+generic/322
+generic/324
+generic/325
+generic/326
+generic/327
+generic/328
+generic/329
+generic/330
+generic/331
+generic/332
+generic/335
+generic/336
+generic/337
+generic/341
+generic/342
+generic/343
+generic/346
+generic/348
+generic/353
+generic/355
+generic/358
+generic/359
+generic/360
+generic/361
+generic/362
+generic/363
+generic/364
+generic/365
+generic/366
+generic/367
+generic/368
+generic/369
+generic/370
+generic/371
+generic/372
+generic/373
+generic/374
+generic/375
+generic/376
+generic/377
+generic/378
+generic/379
+generic/380
+generic/381
+generic/382
+generic/383
+generic/384
+generic/385
+generic/386
+generic/389
+generic/391
+generic/392
+generic/393
+generic/394
+generic/395
+generic/396
+generic/397
+generic/398
+generic/400
+generic/401
+generic/402
+generic/403
+generic/404
+generic/406
+generic/407
+generic/408
+generic/412
+generic/413
+generic/414
+generic/417
+generic/419
+generic/420
+generic/421
+generic/422
+generic/424
+generic/425
+generic/426
+generic/427
+generic/428
+generic/436
+generic/437
+generic/439
+generic/440
+generic/443
+generic/445
+generic/446
+generic/448
+generic/449
+generic/450
+generic/451
+generic/452
+generic/453
+generic/454
+generic/456
+generic/458
+generic/460
+generic/462
+generic/463
+generic/465
+generic/466
+generic/468
+generic/469
+generic/470
+generic/471
+generic/474
+generic/477
+generic/478
+generic/479
+generic/480
+generic/481
+generic/483
+generic/485
+generic/486
+generic/487
+generic/488
+generic/489
+generic/490
+generic/491
+generic/492
+generic/498
+generic/499
+generic/501
+generic/502
+generic/503
+generic/504
+generic/505
+generic/506
+generic/507
+generic/508
+generic/509
+generic/510
+generic/511
+generic/512
+generic/513
+generic/514
+generic/515
+generic/516
+generic/517
+generic/518
+generic/519
+generic/520
+generic/523
+generic/524
+generic/525
+generic/526
+generic/527
+generic/528
+generic/529
+generic/530
+generic/531
+generic/533
+generic/534
+generic/535
+generic/536
+generic/537
+generic/538
+generic/539
+generic/540
+generic/541
+generic/542
+generic/543
+generic/544
+generic/545
+generic/546
+generic/547
+generic/548
+generic/549
+generic/550
+generic/552
+generic/553
+generic/555
+generic/556
+generic/557
+generic/566
+generic/567
+generic/571
+generic/572
+generic/573
+generic/574
+generic/575
+generic/576
+generic/577
+generic/578
+generic/580
+generic/581
+generic/582
+generic/583
+generic/584
+generic/586
+generic/587
+generic/588
+generic/591
+generic/592
+generic/593
+generic/594
+generic/595
+generic/596
+generic/597
+generic/598
+generic/599
+generic/600
+generic/601
+generic/602
+generic/603
+generic/604
+generic/605
+generic/606
+generic/607
+generic/608
+generic/609
+generic/610
+generic/611
+generic/612
+generic/613
+generic/614
+generic/618
+generic/621
+generic/623
+generic/624
+generic/625
+generic/626
+generic/628
+generic/629
+generic/630
+generic/632
+generic/634
+generic/635
+generic/637
+generic/638
+generic/639
+generic/640
+generic/644
+generic/645
+generic/646
+generic/647
+generic/651
+generic/652
+generic/653
+generic/654
+generic/655
+generic/657
+generic/658
+generic/659
+generic/660
+generic/661
+generic/662
+generic/663
+generic/664
+generic/665
+generic/666
+generic/667
+generic/668
+generic/669
+generic/673
+generic/674
+generic/675
+generic/676
+generic/677
+generic/678
+generic/679
+generic/680
+generic/681
+generic/682
+generic/683
+generic/684
+generic/685
+generic/686
+generic/687
+generic/688
+generic/689
+shared/002
+shared/032
+Not
+run:
+generic/008
+generic/009
+generic/012
+generic/015
+generic/016
+generic/018
+generic/021
+generic/022
+generic/025
+generic/026
+generic/031
+generic/033
+generic/050
+generic/052
+generic/058
+generic/059
+generic/060
+generic/061
+generic/063
+generic/064
+generic/078
+generic/079
+generic/081
+generic/082
+generic/091
+generic/094
+generic/096
+generic/110
+generic/111
+generic/113
+generic/114
+generic/115
+generic/116
+generic/118
+generic/119
+generic/121
+generic/122
+generic/123
+generic/128
+generic/130
+generic/134
+generic/135
+generic/136
+generic/138
+generic/139
+generic/140
+generic/142
+generic/143
+generic/144
+generic/145
+generic/146
+generic/147
+generic/148
+generic/149
+generic/150
+generic/151
+generic/152
+generic/153
+generic/154
+generic/155
+generic/156
+generic/157
+generic/158
+generic/159
+generic/160
+generic/161
+generic/162
+generic/163
+generic/171
+generic/172
+generic/173
+generic/174
+generic/177
+generic/178
+generic/179
+generic/180
+generic/181
+generic/182
+generic/183
+generic/185
+generic/188
+generic/189
+generic/190
+generic/191
+generic/193
+generic/194
+generic/195
+generic/196
+generic/197
+generic/198
+generic/199
+generic/200
+generic/201
+generic/202
+generic/203
+generic/205
+generic/206
+generic/207
+generic/210
+generic/211
+generic/212
+generic/214
+generic/216
+generic/217
+generic/218
+generic/219
+generic/220
+generic/222
+generic/223
+generic/225
+generic/227
+generic/229
+generic/230
+generic/235
+generic/238
+generic/240
+generic/244
+generic/250
+generic/252
+generic/253
+generic/254
+generic/255
+generic/256
+generic/259
+generic/260
+generic/261
+generic/262
+generic/263
+generic/264
+generic/265
+generic/266
+generic/267
+generic/268
+generic/271
+generic/272
+generic/276
+generic/277
+generic/278
+generic/279
+generic/281
+generic/282
+generic/283
+generic/284
+generic/287
+generic/288
+generic/289
+generic/290
+generic/291
+generic/292
+generic/293
+generic/295
+generic/296
+generic/301
+generic/302
+generic/303
+generic/304
+generic/305
+generic/312
+generic/314
+generic/316
+generic/317
+generic/324
+generic/326
+generic/327
+generic/328
+generic/329
+generic/330
+generic/331
+generic/332
+generic/353
+generic/355
+generic/358
+generic/359
+generic/361
+generic/362
+generic/363
+generic/364
+generic/365
+generic/366
+generic/367
+generic/368
+generic/369
+generic/370
+generic/371
+generic/372
+generic/373
+generic/374
+generic/378
+generic/379
+generic/380
+generic/381
+generic/382
+generic/383
+generic/384
+generic/385
+generic/386
+generic/391
+generic/392
+generic/395
+generic/396
+generic/397
+generic/398
+generic/400
+generic/402
+generic/404
+generic/406
+generic/407
+generic/408
+generic/412
+generic/413
+generic/414
+generic/417
+generic/419
+generic/420
+generic/421
+generic/422
+generic/424
+generic/425
+generic/427
+generic/439
+generic/440
+generic/446
+generic/449
+generic/450
+generic/451
+generic/453
+generic/454
+generic/456
+generic/458
+generic/462
+generic/463
+generic/465
+generic/466
+generic/468
+generic/469
+generic/470
+generic/471
+generic/474
+generic/485
+generic/487
+generic/488
+generic/491
+generic/492
+generic/499
+generic/501
+generic/503
+generic/505
+generic/506
+generic/507
+generic/508
+generic/511
+generic/513
+generic/514
+generic/515
+generic/516
+generic/517
+generic/518
+generic/519
+generic/520
+generic/528
+generic/530
+generic/536
+generic/537
+generic/538
+generic/539
+generic/540
+generic/541
+generic/542
+generic/543
+generic/544
+generic/545
+generic/546
+generic/548
+generic/549
+generic/550
+generic/552
+generic/553
+generic/555
+generic/556
+generic/566
+generic/567
+generic/572
+generic/573
+generic/574
+generic/575
+generic/576
+generic/577
+generic/578
+generic/580
+generic/581
+generic/582
+generic/583
+generic/584
+generic/586
+generic/587
+generic/588
+generic/591
+generic/592
+generic/593
+generic/594
+generic/595
+generic/596
+generic/597
+generic/598
+generic/599
+generic/600
+generic/601
+generic/602
+generic/603
+generic/605
+generic/606
+generic/607
+generic/608
+generic/609
+generic/610
+generic/612
+generic/613
+generic/621
+generic/623
+generic/624
+generic/625
+generic/626
+generic/628
+generic/629
+generic/630
+generic/635
+generic/644
+generic/645
+generic/646
+generic/647
+generic/651
+generic/652
+generic/653
+generic/654
+generic/655
+generic/657
+generic/658
+generic/659
+generic/660
+generic/661
+generic/662
+generic/663
+generic/664
+generic/665
+generic/666
+generic/667
+generic/668
+generic/669
+generic/673
+generic/674
+generic/675
+generic/677
+generic/678
+generic/679
+generic/680
+generic/681
+generic/682
+generic/683
+generic/684
+generic/685
+generic/686
+generic/687
+generic/688
+generic/689
+shared/002
+shared/032
+Passed all 512 tests
--- a/tests/run-tests.sh
+++ b/tests/run-tests.sh
@@ -56,7 +56,6 @@ $(basename $0) options:
              | only tests matching will be run.  Can be provided multiple
              | times
    -i        | Force removing and inserting the built scoutfs.ko module.
-    -l <nr>   | Loop each test <nr> times while passing, last run counts.
    -M <file> | Specify the filesystem's meta data device path that contains
              | the file system to be tested.  Will be clobbered by -m mkfs.
    -m        | Run mkfs on the device before mounting and running
@@ -92,14 +91,9 @@ done
 T_TRACE_DUMP="0"
 T_TRACE_PRINTK="0"
 T_PORT_START="19700"
-T_LOOP_ITER="100"

 # array declarations to be able to use array ops
 declare -a T_TRACE_GLOB
-T_TRACE_GLOB=( "scoutfs*" )
-
-# CI sets this to 3600, but, for this case we want it very short
-echo 30 > /proc/sys/kernel/hung_task_timeout_secs

 while true; do
 	case $1 in
@@ -137,12 +131,6 @@ while true; do
 	-i)
 		T_INSMOD="1"
 		;;
-	-l)
-	        test -n "$2" || die "-l must have a nr iterations argument"
-		test "$2" -eq "$2" 2>/dev/null || die "-l <nr> argument must be an integer"
-		T_LOOP_ITER="$2"
-		shift
-		;;
 	-M)
 	        test -n "$2" || die "-z must have meta device file argument"
 	        T_META_DEVICE="$2"
@@ -442,30 +430,6 @@ cmd grep .  /sys/kernel/debug/tracing/options/trace_printk \
 	    /sys/kernel/debug/tracing/buffer_size_kb \
 	    /proc/sys/kernel/ftrace_dump_on_oops

-# we can record pids to kill as we exit, we kill in reverse added order
-atexit_kill_pids=""
-add_atexit_kill_pid()
-{
-	atexit_kill_pids="$1 $atexit_kill_pids"
-}
-atexit_kill()
-{
-	local pid
-
-	# suppress bg function exited messages
-	exec {ERR}>&2 2>/dev/null
-
-	for pid in $atexit_kill_pids; do
-		if test -e "/proc/$pid/status" ; then
-			kill "$pid"
-			wait "$pid"
-		fi
-	done
-
-	exec 2>&$ERR {ERR}>&-
-}
-trap atexit_kill EXIT
-
 #
 # Build a fenced config that runs scripts out of the repository rather
 # than the default system directory
@@ -479,50 +443,26 @@ EOF
 export SCOUTFS_FENCED_CONFIG_FILE="$conf"
 T_FENCED_LOG="$T_RESULTS/fenced.log"

+#
+# Run the agent in the background, log its output, an kill it if we
+# exit
+#
+fenced_log()
+{
+	echo "[$(timestamp)] $*" >> "$T_FENCED_LOG"
+}
+fenced_pid=""
+kill_fenced()
+{
+	if test -n "$fenced_pid" -a -d "/proc/$fenced_pid" ; then
+		fenced_log "killing fenced pid $fenced_pid"
+		kill "$fenced_pid"
+	fi
+}
+trap kill_fenced EXIT
 $T_UTILS/fenced/scoutfs-fenced > "$T_FENCED_LOG" 2>&1 &
 fenced_pid=$!
-add_atexit_kill_pid $fenced_pid
-
-#
-# some critical failures will cause fs operations to hang.  We can watch
-# for evidence of them and cause the system to crash, at least.
-#
-crash_monitor()
-{
-	local bad=0
-
-	while sleep 1; do
-		if dmesg | grep -q "inserting extent.*overlaps existing"; then
-			echo "run-tests monitor saw overlapping extent message"
-			bad=1
-		fi
-
-		if dmesg | grep -q "blocked for more than"; then
-			echo "run-tests monitor saw blocked task message"
-			bad=1
-		fi
-
-		if dmesg | grep -q "error indicated by fence action" ; then
-			echo "run-tests monitor saw fence agent error message"
-			bad=1
-		fi
-
-		if [ ! -e "/proc/${fenced_pid}/status" ]; then
-			echo "run-tests monitor didn't see fenced pid $fenced_pid /proc dir"
-			bad=1
-		fi
-
-		if [ "$bad" != 0 ]; then
-			sync & # maybe this gets logs synced...
-			sleep .1
-			echo "run-tests monitor triggering crash"
-			echo c > /proc/sysrq-trigger
-			exit 1
-		fi
-	done
-}
-crash_monitor &
-add_atexit_kill_pid $!
+fenced_log "started fenced pid $fenced_pid in the background"

 # setup dm tables
 echo "0 $(blockdev --getsz $T_META_DEVICE) linear $T_META_DEVICE 0" > \
@@ -595,7 +535,7 @@ fi
 . funcs/filter.sh

 # give tests access to built binaries in src/, prefer over installed
-export PATH="$PWD/src:$PATH"
+PATH="$PWD/src:$PATH"

 msg "running tests"
 > "$T_RESULTS/skip.log"
@@ -615,112 +555,101 @@ for t in $tests; do
 	t="tests/$t"
 	test_name=$(basename "$t" | sed -e 's/.sh$//')

+	# create a temporary dir and file path for the test
+	T_TMPDIR="$T_RESULTS/tmp/$test_name"
+	T_TMP="$T_TMPDIR/tmp"
+	cmd rm -rf "$T_TMPDIR"
+	cmd mkdir -p "$T_TMPDIR"
+
+	# create a test name dir in the fs, clean up old data as needed
+	T_DS=""
+	for i in $(seq 0 $((T_NR_MOUNTS - 1))); do
+		dir="${T_M[$i]}/test/$test_name"
+
+		test $i == 0 && (
+			test -d "$dir" && cmd rm -rf "$dir"
+			cmd mkdir -p "$dir"
+		)
+
+		eval T_D$i=$dir
+		T_D[$i]=$dir
+		T_DS+="$dir "
+	done
+
+	# export all our T_ variables
+	for v in ${!T_*}; do
+		eval export $v
+	done
+	export PATH # give test access to scoutfs binary
+
+	# prepare to compare output to golden output
+	test -e "$T_RESULTS/output" || cmd mkdir -p "$T_RESULTS/output"
+	out="$T_RESULTS/output/$test_name"
+	> "$T_TMPDIR/status.msg"
+	golden="golden/$test_name"
+
 	# get stats from previous pass
 	last="$T_RESULTS/last-passed-test-stats"
 	stats=$(grep -s "^$test_name " "$last" | cut -d " " -f 2-)
 	test -n "$stats" && stats="last: $stats"
+
 	printf "  %-30s $stats" "$test_name"

 	# mark in dmesg as to what test we are running
 	echo "run scoutfs test $test_name" > /dev/kmsg

-	# let the test get at its extra files
-	T_EXTRA="$T_TESTS/extra/$test_name"
+	# record dmesg before
+	dmesg | t_filter_dmesg > "$T_TMPDIR/dmesg.before"

-	for iter in $(seq 1 $T_LOOP_ITER); do
+	# give tests stdout and compared output on specific fds
+	exec 6>&1
+	exec 7>$out

-		# create a temporary dir and file path for the test
-		T_TMPDIR="$T_RESULTS/tmp/$test_name"
-		T_TMP="$T_TMPDIR/tmp"
-		cmd rm -rf "$T_TMPDIR"
-		cmd mkdir -p "$T_TMPDIR"
+	# run the test with access to our functions
+	start_secs=$SECONDS
+	bash -c "for f in funcs/*.sh; do . \$f; done; . $t" >&7 2>&1
+	sts="$?"
+	log "test $t exited with status $sts"
+	stats="$((SECONDS - start_secs))s"

-		# create a test name dir in the fs, clean up old data as needed
-		T_DS=""
-		for i in $(seq 0 $((T_NR_MOUNTS - 1))); do
-			dir="${T_M[$i]}/test/$test_name"
+	# close our weird descriptors
+	exec 6>&-
+	exec 7>&-

-			test $i == 0 && (
-				test -d "$dir" && cmd rm -rf "$dir"
-				cmd mkdir -p "$dir"
-			)
-
-			eval T_D$i=$dir
-			T_D[$i]=$dir
-			T_DS+="$dir "
-		done
-
-		# export all our T_ variables
-		for v in ${!T_*}; do
-			eval export $v
-		done
-
-		# prepare to compare output to golden output
-		test -e "$T_RESULTS/output" || cmd mkdir -p "$T_RESULTS/output"
-		out="$T_RESULTS/output/$test_name"
-		> "$T_TMPDIR/status.msg"
-		golden="golden/$test_name"
-
-		# record dmesg before
-		dmesg | t_filter_dmesg > "$T_TMPDIR/dmesg.before"
-
-		# give tests stdout and compared output on specific fds
-		exec 6>&1
-		exec 7>$out
-
-		# run the test with access to our functions
-		start_secs=$SECONDS
-		bash -c "for f in funcs/*.sh; do . \$f; done; . $t" >&7 2>&1
-		sts="$?"
-		log "test $t exited with status $sts"
-		stats="$((SECONDS - start_secs))s"
-
-		# close our weird descriptors
-		exec 6>&-
-		exec 7>&-
-
-		# compare output if the test returned passed status
-		if [ "$sts" == "$T_PASS_STATUS" ]; then
-			if [ ! -e "$golden" ]; then
-				message="no golden output"
-				sts=$T_FAIL_STATUS
-			elif ! cmp -s "$golden" "$out"; then 
-				message="output differs"
-				sts=$T_FAIL_STATUS
-				diff -u "$golden" "$out" >> "$T_RESULTS/fail.log"
-			fi
-		else
-			# get message from t_*() functions
-			message=$(cat "$T_TMPDIR/status.msg")
-		fi
-
-		# see if anything unexpected was added to dmesg
-		if [ "$sts" == "$T_PASS_STATUS" ]; then
-			dmesg | t_filter_dmesg > "$T_TMPDIR/dmesg.after"
-			diff --old-line-format="" --unchanged-line-format="" \
-				"$T_TMPDIR/dmesg.before" "$T_TMPDIR/dmesg.after" > \
-				"$T_TMPDIR/dmesg.new"
-
-			if [ -s "$T_TMPDIR/dmesg.new" ]; then
-				message="unexpected messages in dmesg"
-				sts=$T_FAIL_STATUS
-				cat "$T_TMPDIR/dmesg.new" >> "$T_RESULTS/fail.log"
-			fi
-		fi
-
-		# record unknown exit status
-		if [ "$sts" -lt "$T_FIRST_STATUS" -o "$sts" -gt "$T_LAST_STATUS" ]; then
-			message="unknown status: $sts"
+	# compare output if the test returned passed status
+	if [ "$sts" == "$T_PASS_STATUS" ]; then
+		if [ ! -e "$golden" ]; then
+			message="no golden output"
 			sts=$T_FAIL_STATUS
+		elif ! cmp -s "$golden" "$out"; then 
+			message="output differs"
+			sts=$T_FAIL_STATUS
+			diff -u "$golden" "$out" >> "$T_RESULTS/fail.log"
 		fi
+	else
+		# get message from t_*() functions
+		message=$(cat "$T_TMPDIR/status.msg")
+	fi

-		# stop looping if we didn't pass
-		if [ "$sts" != "$T_PASS_STATUS" ]; then
-			break;
-		else
-			echo > /sys/kernel/debug/tracing/trace
+	# see if anything unexpected was added to dmesg
+	if [ "$sts" == "$T_PASS_STATUS" ]; then
+		dmesg | t_filter_dmesg > "$T_TMPDIR/dmesg.after"
+		diff --old-line-format="" --unchanged-line-format="" \
+			"$T_TMPDIR/dmesg.before" "$T_TMPDIR/dmesg.after" > \
+			"$T_TMPDIR/dmesg.new"
+
+		if [ -s "$T_TMPDIR/dmesg.new" ]; then
+			message="unexpected messages in dmesg"
+			sts=$T_FAIL_STATUS
+			cat "$T_TMPDIR/dmesg.new" >> "$T_RESULTS/fail.log"
 		fi
-	done
+	fi
+
+	# record unknown exit status
+	if [ "$sts" -lt "$T_FIRST_STATUS" -o "$sts" -gt "$T_LAST_STATUS" ]; then
+		message="unknown status: $sts"
+		sts=$T_FAIL_STATUS
+	fi

 	# show and record the result of the test
 	if [ "$sts" == "$T_PASS_STATUS" ]; then
--- a/tests/sequence
+++ b/tests/sequence
@@ -1 +1,60 @@
+export-get-name-parent.sh
+basic-block-counts.sh
+basic-bad-mounts.sh
+basic-posix-acl.sh
+inode-items-updated.sh
+simple-inode-index.sh
+simple-staging.sh
+simple-release-extents.sh
+simple-readdir.sh
+get-referring-entries.sh
+fallocate.sh
+basic-truncate.sh
+data-prealloc.sh
+setattr_more.sh
+offline-extent-waiting.sh
+move-blocks.sh
+projects.sh
+large-fragmented-free.sh
+format-version-forward-back.sh
+enospc.sh
+mmap.sh
+srch-safe-merge-pos.sh
+srch-basic-functionality.sh
+simple-xattr-unit.sh
+retention-basic.sh
+totl-xattr-tag.sh
+quota.sh
+lock-refleak.sh
+lock-shrink-consistency.sh
+lock-shrink-read-race.sh
+lock-pr-cw-conflict.sh
+lock-revoke-getcwd.sh
+lock-recover-invalidate.sh
 export-lookup-evict-race.sh
+createmany-parallel.sh
+createmany-large-names.sh
+createmany-rename-large-dir.sh
+stage-release-race-alloc.sh
+stage-multi-part.sh
+o_tmpfile.sh
+basic-posix-consistency.sh
+dirent-consistency.sh
+mkdir-rename-rmdir.sh
+lock-ex-race-processes.sh
+cross-mount-data-free.sh
+persistent-item-vers.sh
+setup-error-teardown.sh
+resize-devices.sh
+change-devices.sh
+fence-and-reclaim.sh
+quorum-heartbeat-timeout.sh
+orphan-inodes.sh
+mount-unmount-race.sh
+client-unmount-recovery.sh
+createmany-parallel-mounts.sh
+archive-light-cycle.sh
+block-stale-reads.sh
+inode-deletion.sh
+renameat2-noreplace.sh
+xfstests.sh
--- a/tests/src/mmap_stress.c
+++ b/tests/src/mmap_stress.c
@@ -19,7 +19,6 @@
 #include <sys/types.h>
 #include <stdio.h>
 #include <sys/stat.h>
-#include <inttypes.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <stdlib.h>
@@ -30,7 +29,7 @@
 #include <errno.h>

 static int size = 0;
-static int duration = 0;
+static int count = 0; /* XXX make this duration instead */

 struct thread_info {
 	int nr;
@@ -42,8 +41,6 @@ static void *run_test_func(void *ptr)
 	void *buf = NULL;
 	char *addr = NULL;
 	struct thread_info *tinfo = ptr;
-	uint64_t seconds = 0;
-	struct timespec ts;
 	int c = 0;
 	int fd;
 	ssize_t read, written, ret;
@@ -64,15 +61,9 @@ static void *run_test_func(void *ptr)

 	usleep(100000); /* 0.1sec to allow all threads to start roughly at the same time */

-	clock_gettime(CLOCK_REALTIME, &ts); /* record start time */
-	seconds = ts.tv_sec + duration;
-
 	for (;;) {
-		if (++c % 16 == 0) {
-			clock_gettime(CLOCK_REALTIME, &ts);
-			if (ts.tv_sec >= seconds)
-				break;
-		}
+		if (++c > count)
+			break;

 		switch (rand() % 4) {
 		case 0: /* pread */
@@ -108,8 +99,6 @@ static void *run_test_func(void *ptr)
 			memcpy(addr, buf, size); /* noerr */
 			break;
 		}
-
-		usleep(10000);
 	}

 	munmap(addr, size);
@@ -131,7 +120,7 @@ int main(int argc, char **argv)
 	int i;

 	if (argc != 8) {
-		fprintf(stderr, "%s requires 7 arguments - size duration file1 file2 file3 file4 file5\n", argv[0]);
+		fprintf(stderr, "%s requires 7 arguments - size count file1 file2 file3 file4 file5\n", argv[0]);
 		exit(-1);
 	}

@@ -141,9 +130,9 @@ int main(int argc, char **argv)
 		exit(-1);
 	}

-	duration = atoi(argv[2]);
-	if (duration < 0) {
-		fprintf(stderr, "invalid duration, must be greater than or equal to 0\n");
+	count = atoi(argv[2]);
+	if (count < 0) {
+		fprintf(stderr, "invalid count, must be greater than 0\n");
 		exit(-1);
 	}

--- a/tests/tests/mmap.sh
+++ b/tests/tests/mmap.sh
@@ -5,7 +5,7 @@
 t_require_commands mmap_stress mmap_validate scoutfs xfs_io

 echo "== mmap_stress"
-mmap_stress 8192 30 "$T_D0/mmap_stress" "$T_D0/mmap_stress" "$T_D0/mmap_stress" "$T_D3/mmap_stress" "$T_D3/mmap_stress" | sed 's/:.*//g' | sort
+mmap_stress 8192 2000 "$T_D0/mmap_stress" "$T_D1/mmap_stress" "$T_D2/mmap_stress" "$T_D3/mmap_stress" "$T_D4/mmap_stress" | sed 's/:.*//g' | sort

 echo "== basic mmap/read/write consistency checks"
 mmap_validate 256 1000 "$T_D0/mmap_val1" "$T_D1/mmap_val1"
--- a/tests/tests/renameat2-noreplace.sh
+++ b/tests/tests/renameat2-noreplace.sh
@@ -8,19 +8,19 @@ t_require_mounts 2
 echo "=== renameat2 noreplace flag test"

 # give each mount their own dir (lock group) to minimize create contention
-mkdir $T_D0/dir0
-mkdir $T_D1/dir1
+mkdir $T_M0/dir0
+mkdir $T_M1/dir1

 echo "=== run two asynchronous calls to renameat2 NOREPLACE"
 for i in $(seq 0 100); do
        # prepare inputs in isolation
-        touch "$T_D0/dir0/old0"
-        touch "$T_D1/dir1/old1"
+        touch "$T_M0/dir0/old0"
+        touch "$T_M1/dir1/old1"

        # race doing noreplace renames, both can't succeed
-        dumb_renameat2 -n "$T_D0/dir0/old0" "$T_D0/dir0/sharednew" 2> /dev/null &
+        dumb_renameat2 -n "$T_M0/dir0/old0" "$T_M0/dir0/sharednew" 2> /dev/null &
        pid0=$!
-        dumb_renameat2 -n "$T_D1/dir1/old1" "$T_D1/dir0/sharednew" 2> /dev/null &
+        dumb_renameat2 -n "$T_M1/dir1/old1" "$T_M1/dir0/sharednew" 2> /dev/null &
        pid1=$!

        wait $pid0
@@ -31,7 +31,7 @@ for i in $(seq 0 100); do
        test "$rc0" == 0 -a "$rc1" == 0 && t_fail "both renames succeeded"

        # blow away possible files for either race outcome
-        rm -f "$T_D0/dir0/old0" "$T_D1/dir1/old1" "$T_D0/dir0/sharednew" "$T_D1/dir1/sharednew"
+        rm -f "$T_M0/dir0/old0" "$T_M1/dir1/old1" "$T_M0/dir0/sharednew" "$T_M1/dir1/sharednew"
 done

 t_pass
--- a/tests/tests/xfstests.sh
+++ b/tests/tests/xfstests.sh
@@ -63,47 +63,73 @@ export MOUNT_OPTIONS="-o quorum_slot_nr=0,metadev_path=$T_MB0"
 export TEST_FS_MOUNT_OPTS="-o quorum_slot_nr=0,metadev_path=$T_MB0"
 EOF

-cp "$T_EXTRA/local.exclude" local.exclude
+cat << EOF > local.exclude
+generic/003	# missing atime update in buffered read
+generic/075	# file content mismatch failures (fds, etc)
+generic/103	# enospc causes trans commit failures
+generic/108	# mount fails on failing device?
+generic/112	# file content mismatch failures (fds, etc)
+generic/213	# enospc causes trans commit failures
+generic/318	# can't support user namespaces until v5.11
+generic/321	# requires selinux enabled for '+' in ls?
+generic/338	# BUG_ON update inode error handling
+generic/347	# _dmthin_mount doesn't work?
+generic/356	# swap
+generic/357	# swap
+generic/409	# bind mounts not scripted yet
+generic/410	# bind mounts not scripted yet
+generic/411	# bind mounts not scripted yet
+generic/423	# symlink inode size is strlen() + 1 on scoutfs
+generic/430	# xfs_io copy_range missing in el7
+generic/431	# xfs_io copy_range missing in el7
+generic/432	# xfs_io copy_range missing in el7
+generic/433	# xfs_io copy_range missing in el7
+generic/434	# xfs_io copy_range missing in el7
+generic/441	# dm-mapper
+generic/444	# el9's posix_acl_update_mode is buggy ?
+generic/467	# open_by_handle ESTALE
+generic/472	# swap
+generic/484	# dm-mapper
+generic/493	# swap
+generic/494	# swap
+generic/495	# swap
+generic/496	# swap
+generic/497	# swap
+generic/532	# xfs_io statx attrib_mask missing in el7
+generic/554	# swap
+generic/563	# cgroup+loopdev
+generic/564	# xfs_io copy_range missing in el7
+generic/565	# xfs_io copy_range missing in el7
+generic/568	# falloc not resulting in block count increase
+generic/569	# swap
+generic/570	# swap
+generic/620	# dm-hugedisk
+generic/633	# id-mapped mounts missing in el7
+generic/636	# swap
+generic/641	# swap
+generic/643	# swap
+EOF

-t_stdout_invoked
+t_restore_output
 echo "  (showing output of xfstests)"

 args="-E local.exclude ${T_XFSTESTS_ARGS:--g quick}"
 ./check $args
 # the fs is unmounted when check finishes

-t_stdout_compare
-
 #
-# ./check writes the results of the run to check.log.  It lists the
-# tests it ran, skipped, or failed.  Then it writes a line saying
-# everything passed or some failed.
-#
-
-#
-# If XFSTESTS_ARGS were specified then we just pass/fail to match the
-# check run.
-#
-if [ -n "$T_XFSTESTS_ARGS" ]; then
-	if tail -1 results/check.log | grep -q "Failed"; then
-		t_fail
-	else
-		t_pass
-	fi
-fi
-
-#
-# Otherwise, typically, when there were no args then we scrape the most
-# recent run and use it as the output to compare to make sure that we
-# run the right tests and get the right results.
+# ./check writes the results of the run to check.log.  It lists
+# the tests it ran, skipped, or failed.  Then it writes a line saying
+# everything passed or some failed.  We scrape the most recent run and
+# use it as the output to compare to make sure that we run the right
+# tests and get the right results.
 #
 awk '
 	/^(Ran|Not run|Failures):.*/ {
 		if (pf) {
 			res=""
 			pf=""
-		}
-		res = res "\n" $0
+		} res = res "\n" $0
 	}
 	/^(Passed|Failed).*tests$/ {
 		pf=$0
@@ -113,14 +139,10 @@ awk '
 	}' < results/check.log  > "$T_TMPDIR/results"

 # put a test per line so diff shows tests that differ
-grep -E "^(Ran|Not run|Failures):" "$T_TMPDIR/results" | fmt -w 1 > "$T_TMPDIR/results.fmt"
-grep -E "^(Passed|Failed).*tests$" "$T_TMPDIR/results" >> "$T_TMPDIR/results.fmt"
+egrep "^(Ran|Not run|Failures):" "$T_TMPDIR/results" | \
+	fmt -w 1 > "$T_TMPDIR/results.fmt"
+egrep "^(Passed|Failed).*tests$" "$T_TMPDIR/results" >> "$T_TMPDIR/results.fmt"

-diff -u "$T_EXTRA/expected-results" "$T_TMPDIR/results.fmt" > "$T_TMPDIR/results.diff"
-if [ -s "$T_TMPDIR/results.diff" ]; then
-	echo "tests that were skipped/run differed from expected:"
-	cat "$T_TMPDIR/results.diff"
-	t_fail
-fi
+t_compare_output cat "$T_TMPDIR/results.fmt"

 t_pass
--- a/utils/fenced/scoutfs-fenced
+++ b/utils/fenced/scoutfs-fenced
@@ -62,28 +62,32 @@ test -x "$SCOUTFS_FENCED_RUN" || \
 # files disappear.
 #

-# silence error messages
-quiet_cat()
+# generate failure messages to stderr while still echoing 0 for the caller
+careful_cat()
 {
-	cat "$@" 2>/dev/null
+	local path="$@"
+
+	cat "$@" || echo 0
 }

 while sleep $SCOUTFS_FENCED_DELAY; do
-	shopt -s nullglob
 	for fence in /sys/fs/scoutfs/*/fence/*; do
-
-		srv=$(basename $(dirname $(dirname $fence)))
-		fenced="$(quiet_cat $fence/fenced)"
-		error="$(quiet_cat $fence/error)"
-		rid="$(quiet_cat $fence/rid)"
-		ip="$(quiet_cat $fence/ipv4_addr)"
-		reason="$(quiet_cat $fence/reason)"
-
-		# request dirs can linger then disappear after fenced/error is set
-		if [ ! -d "$fence" -o "$fenced" == "1" -o "$error" == "1" ]; then
+		# catches unmatched regex when no dirs
+		if [ ! -d "$fence" ]; then
 			continue
 		fi

+		# skip requests that have been handled
+		if [ "$(careful_cat $fence/fenced)" == 1 -o \
+		     "$(careful_cat $fence/error)" == 1 ]; then
+			continue
+		fi
+
+		srv=$(basename $(dirname $(dirname $fence)))
+		rid="$(cat $fence/rid)"
+		ip="$(cat $fence/ipv4_addr)"
+		reason="$(cat $fence/reason)"
+
 		log_message "server $srv fencing rid $rid at IP $ip for $reason"

 		# export _REQ_ vars for run to use
--- a/utils/man/scoutfs.5
+++ b/utils/man/scoutfs.5
@@ -55,14 +55,6 @@ with initial sparse regions (perhaps by multiple threads writing to
 different regions) and wasted space isn't an issue (perhaps because the
 file population contains few small files).
 .TP
-.B ino_alloc_per_lock=<number>
-This option determines how many inode numbers are allocated in the same
-cluster lock.  The default, and maximum, is 1024.  The minimum is 1.
-Allocating fewer inodes per lock can allow more parallelism between
-mounts because there are more locks that cover the same number of
-created files.  This can be helpful when working with smaller numbers of
-large files.
-.TP
 .B log_merge_wait_timeout_ms=<number>
 This option sets the amount of time, in milliseconds, that log merge
 creation can wait before timing out.  This setting is per-mount, only