Test case for wedged full meta allocators.

Reproduce the freed-list commit deadlock and show a fixed server recovers.

The alloc_fill_freed_list trigger stuffs both server_meta_freed heads
to near-full in a single commit. It claims runs of free blocks and
appends them straight into the head blocks, leaking whatever isn't
used.

Filling both heads in one commit makes the wedge reproducible. There's
no window for the drain to empty one head before the other fills. An
unfixed server is left with both heads full and wedges on the next
commit. A fixed server drains them and stays live.

Signed-off-by: Auke Kok <auke.kok@versity.com>

kmod/src/alloc.c

@@ -25,6 +25,7 @@
 #include "alloc.h"
 #include "counters.h"
 #include "scoutfs_trace.h"
+#include "triggers.h"
 
 /*
  * The core allocator uses extent items in btrees rooted in the super.
@@ -1241,6 +1242,62 @@ out:
 	return ret;
 }
 
+/*
+ * Test: stuff a freed list head to nearly full with real free blocks.
+ *
+ * Destructive: bypass filesystem consistency. Claim free blocks, append
+ * them into the head block via list_block_add (bypassing free_meta and
+ * its active-head-only path), and leak whatever we don't use.
+ *
+ * The caller can stuff both meta_freed heads in a single commits, allowing
+ * to reproduce the wedge condition. A counter validates we did the stuffing
+ * on both heads (increased twice - once for each head). An unfixed kernel
+ * will hang on mount.
+ */
+int scoutfs_alloc_fill_freed_list(struct super_block *sb, struct scoutfs_alloc *alloc,
+				  struct scoutfs_block_writer *wri,
+				  struct scoutfs_alloc_root *root,
+				  struct scoutfs_alloc_list_head *lh)
+{
+	struct alloc_ext_args args = {
+		.alloc = alloc,
+		.wri = wri,
+		.root = root,
+		.zone = SCOUTFS_FREE_EXTENT_ORDER_ZONE,
+	};
+	const u32 target = SCOUTFS_ALLOC_LIST_MAX_BLOCKS - 8;
+	struct scoutfs_alloc_list_block *lblk;
+	struct scoutfs_block *bl = NULL;
+	struct scoutfs_extent ext;
+	int ret = 0;
+	u64 old; /* leaked */
+	u64 i;
+	u32 want;
+
+	while (le32_to_cpu(lh->first_nr) < target) {
+		want = target - le32_to_cpu(lh->first_nr) + 1;
+		ret = scoutfs_ext_alloc(sb, &alloc_ext_ops, &args, 0, 0, want, &ext);
+		if (ret < 0)
+			break;
+
+		ret = dirty_list_block(sb, alloc, wri, &lh->ref, ext.start, &old, &bl);
+		if (ret < 0)
+			break;
+		lblk = bl->data;
+
+		for (i = 1; i < ext.len && le32_to_cpu(lh->first_nr) < target; i++)
+			list_block_add(lh, lblk, ext.start + i);
+
+		scoutfs_block_put(sb, bl);
+		bl = NULL;
+	}
+
+	scoutfs_block_put(sb, bl);
+	if (ret == 0 && le32_to_cpu(lh->first_nr) >= target)
+		scoutfs_inc_counter(sb, alloc_freed_fill);
+	return ret < 0 ? ret : 0;
+}
+
 /*
  * Move blknos from all the blocks in the list into extents in the root,
  * removing empty blocks as we go.  This can return success and leave blocks

kmod/src/alloc.h

@@ -152,6 +152,10 @@ int scoutfs_alloc_splice_list(struct super_block *sb,
 			      struct scoutfs_block_writer *wri,
 			      struct scoutfs_alloc_list_head *dst,
 			      struct scoutfs_alloc_list_head *src);
+int scoutfs_alloc_fill_freed_list(struct super_block *sb, struct scoutfs_alloc *alloc,
+				  struct scoutfs_block_writer *wri,
+				  struct scoutfs_alloc_root *root,
+				  struct scoutfs_alloc_list_head *lh);
 
 bool scoutfs_alloc_meta_low(struct super_block *sb,
 			    struct scoutfs_alloc *alloc, u32 nr);

kmod/src/counters.h

@@ -16,6 +16,7 @@
 	EXPAND_COUNTER(alloc_alloc_meta)			\
 	EXPAND_COUNTER(alloc_free_data)				\
 	EXPAND_COUNTER(alloc_free_meta)				\
+	EXPAND_COUNTER(alloc_freed_fill)			\
 	EXPAND_COUNTER(alloc_list_avail_lo)			\
 	EXPAND_COUNTER(alloc_list_freed_hi)			\
 	EXPAND_COUNTER(alloc_move)				\

kmod/src/server.c

@@ -623,6 +623,16 @@ static void scoutfs_server_commit_func(struct work_struct *work)
 		goto out;
 	}
 
+	/* test-only: manufacture the deadlock by stuffing both freed heads full
+	 * in this one commit, so there's no window for empty_list to drain one
+	 * before the other fills */
+	if (scoutfs_trigger(sb, ALLOC_FILL_FREED_LIST)) {
+		scoutfs_alloc_fill_freed_list(sb, &server->alloc, &server->wri,
+					      server->meta_avail, &server->alloc.freed);
+		scoutfs_alloc_fill_freed_list(sb, &server->alloc, &server->wri,
+					      server->meta_avail, server->other_freed);
+	}
+
 	/* make sure next avail has sufficient blocks */
 	ret = scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri,
 				      server->other_avail,

kmod/src/triggers.c

@@ -47,6 +47,7 @@ static char *names[] = {
 	[SCOUTFS_TRIGGER_STATFS_LOCK_PURGE] = "statfs_lock_purge",
 	[SCOUTFS_TRIGGER_RECLAIM_SKIP_FINALIZE] = "reclaim_skip_finalize",
 	[SCOUTFS_TRIGGER_LOG_MERGE_FORCE_PARTIAL] = "log_merge_force_partial",
+	[SCOUTFS_TRIGGER_ALLOC_FILL_FREED_LIST] = "alloc_fill_freed_list",
 };
 
 bool scoutfs_trigger_test_and_clear(struct super_block *sb, unsigned int t)

kmod/src/triggers.h

@@ -10,6 +10,7 @@ enum scoutfs_trigger {
 	SCOUTFS_TRIGGER_STATFS_LOCK_PURGE,
 	SCOUTFS_TRIGGER_RECLAIM_SKIP_FINALIZE,
 	SCOUTFS_TRIGGER_LOG_MERGE_FORCE_PARTIAL,
+	SCOUTFS_TRIGGER_ALLOC_FILL_FREED_LIST,
 	SCOUTFS_TRIGGER_NR,
 };
 

tests/golden/freed-list-wedge

@@ -0,0 +1,8 @@
+== make throwaway scratch fs
+== stuff both server freed heads full in one commit
+== confirm both heads were stuffed (not a no-op)
+both freed heads stuffed
+== a fixed server drains them and keeps making progress
+one
+two
+== cleanup scratch fs

tests/sequence

@@ -62,6 +62,7 @@ client-unmount-recovery.sh
 createmany-parallel-mounts.sh
 archive-light-cycle.sh
 block-stale-reads.sh
+freed-list-wedge.sh
 inode-deletion.sh
 renameat2-noreplace.sh
 xfstests.sh

tests/tests/freed-list-wedge.sh

@@ -0,0 +1,39 @@
+#
+# Destructive: the alloc_fill_freed_list trigger claims free blocks and leaks
+# them into both server_meta_freed heads in one commit, filling them near-full.
+# Runs on a scratch fs; mkfs before re-use.
+#
+# A fixed server drains the full heads and stays live; an unfixed server wedges
+# on the next commit and hangs until the harness times it out.
+#
+
+scr_counter() {
+	cat "$(t_sysfs_path_from_mnt "$T_MSCR")/counters/$1"
+}
+
+echo "== make throwaway scratch fs"
+t_scratch_mkfs
+t_scratch_mount
+
+echo "== stuff both server freed heads full in one commit"
+old=$(scr_counter alloc_freed_fill)
+
+echo 1 > "/sys/kernel/debug/scoutfs/$(t_ident_from_mnt "$T_MSCR")/trigger/alloc_fill_freed_list"
+echo one > "$T_MSCR/one"; sync
+
+echo "== confirm both heads were stuffed (not a no-op)"
+filled=$(($(scr_counter alloc_freed_fill) - old))
+test "$filled" -ge 2 && echo "both freed heads stuffed" || \
+	echo "stuff was a no-op ($filled heads)"
+
+echo "== a fixed server drains them and keeps making progress"
+# an unfixed server wedges on this commit
+echo two > "$T_MSCR/two"; sync
+cat "$T_MSCR/one" "$T_MSCR/two"
+
+rm -f "$T_MSCR/one" "$T_MSCR/two"; sync
+
+echo "== cleanup scratch fs"
+t_scratch_umount
+
+t_pass