Update stale block reading test

The previous test that triggered re-reading blocks, as though they were
stale, was written in the era where it only hit btree blocks and
everything else was stored in LSM segments.

This reworks the test to make it clear that it affects all our block
readers today.  The test only exercise the core read retry path, but it
could be expanded to test callers retrying with newer references after
they get -ESTALE errors.

Signed-off-by: Zach Brown <zab@versity.com>

kmod/src/block.c

@@ -29,6 +29,7 @@
 #include "msg.h"
 #include "scoutfs_trace.h"
 #include "alloc.h"
+#include "triggers.h"
 
 /*
  * The scoutfs block cache manages metadata blocks that can be larger
@@ -682,7 +683,8 @@ retry:
 
 	ret = 0;
 out:
-	if (ret == -ESTALE && !retried && !block_is_dirty(bp)) {
+	if ((ret == -ESTALE || scoutfs_trigger(sb, BLOCK_REMOVE_STALE)) &&
+	    !retried && !block_is_dirty(bp)) {
 		retried = true;
 		scoutfs_inc_counter(sb, block_cache_remove_stale);
 		block_remove(sb, bp);

kmod/src/triggers.c

@@ -38,6 +38,7 @@ struct scoutfs_triggers {
 	struct scoutfs_triggers *name = SCOUTFS_SB(sb)->triggers
 
 static char *names[] = {
+	[SCOUTFS_TRIGGER_BLOCK_REMOVE_STALE] = "block_remove_stale",
 	[SCOUTFS_TRIGGER_BTREE_STALE_READ] = "btree_stale_read",
 	[SCOUTFS_TRIGGER_BTREE_ADVANCE_RING_HALF] = "btree_advance_ring_half",
 	[SCOUTFS_TRIGGER_HARD_STALE_ERROR] = "hard_stale_error",

kmod/src/triggers.h

@@ -2,6 +2,7 @@
 #define _SCOUTFS_TRIGGERS_H_
 
 enum scoutfs_trigger {
+	SCOUTFS_TRIGGER_BLOCK_REMOVE_STALE,
 	SCOUTFS_TRIGGER_BTREE_STALE_READ,
 	SCOUTFS_TRIGGER_BTREE_ADVANCE_RING_HALF,
 	SCOUTFS_TRIGGER_HARD_STALE_ERROR,

tests/golden/block-stale-reads

@@ -0,0 +1,29 @@
+== create file for xattr ping pong
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="initial"
+
+== retry btree forest reads between mounts
+trigger block_remove_stale armed: 0
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="1"
+
+trigger block_remove_stale after: 0
+counter block_cache_remove_stale diff 1
+trigger block_remove_stale armed: 0
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="2"
+
+trigger block_remove_stale after: 0
+counter block_cache_remove_stale diff 2
+trigger block_remove_stale armed: 0
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="3"
+
+trigger block_remove_stale after: 0
+counter block_cache_remove_stale diff 3
+trigger block_remove_stale armed: 0
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="4"
+
+trigger block_remove_stale after: 0
+counter block_cache_remove_stale diff 4

tests/golden/stale-btree-read

@@ -1,11 +0,0 @@
-== create file for xattr ping pong
-# file: /mnt/test/test/stale-btree-read/file
-user.xat="initial"
-
-== retry btree block read
-trigger btree_stale_read armed: 1
-# file: /mnt/test/test/stale-btree-read/file
-user.xat="btree"
-
-trigger btree_stale_read after: 0
-counter btree_stale_read diff 1

tests/sequence

@@ -28,5 +28,5 @@ setup-error-teardown.sh
 mount-unmount-race.sh
 createmany-parallel-mounts.sh
 archive-light-cycle.sh
-stale-btree-read.sh
+block-stale-reads.sh
 xfstests.sh

tests/tests/block-stale-reads.sh

@@ -0,0 +1,40 @@
+#
+# exercise stale block reading.
+#
+# It would be very difficult to manipulate the allocators, cache, and
+# persistent blocks to create stable block reading scenarios.    Instead
+# we use triggers to exercise how readers encounter stale blocks.
+#
+
+t_require_commands touch setfattr getfattr
+t_require_mounts 2
+
+GETFATTR="getfattr --absolute-names"
+SETFATTR="setfattr"
+
+#
+# force re-reading forest btree blocks as each mount reads the items
+# written by the other.
+#
+set_file="$T_D0/file"
+get_file="$T_D1/file"
+echo "== create file for xattr ping pong"
+touch "$set_file"
+$SETFATTR -n user.xat -v initial "$set_file"
+$GETFATTR -n user.xat "$get_file" 2>&1 | t_filter_fs
+
+echo "== retry btree forest reads between mounts" 
+for i in $(seq 1 4); do
+	tmp="$set_file"
+	set_file="$get_file"
+	get_file="$tmp"
+
+	$SETFATTR -n user.xat -v $i "$set_file"
+	t_trigger_arm block_remove_stale $cl
+	old=$(t_counter btree_stale_read $cl)
+	$GETFATTR -n user.xat "$get_file" 2>&1 | t_filter_fs
+	t_trigger_show block_remove_stale "after" $cl
+	t_counter_diff block_cache_remove_stale $old $cl
+done
+
+t_pass

tests/tests/stale-btree-read.sh

@@ -1,40 +0,0 @@
-#
-# verify stale btree block reading
-#
-
-t_require_commands touch stat setfattr getfattr createmany
-t_require_mounts 2
-
-GETFATTR="getfattr --absolute-names"
-SETFATTR="setfattr"
-
-#
-# This exercises the soft retry of btree blocks when
-# inconsistent cached versions are found.  It ensures that basic hard
-# error returning turns into EIO in the case where the persistent reread
-# blocks and segments really are inconsistent.
-#
-# The triggers apply across all execution in the file system.  So to
-# trigger btree block retries in the client we make sure that the server
-# is running on the other node.
-# 
-
-cl=$(t_first_client_nr)
-sv=$(t_server_nr)
-eval cl_dir="\$T_D${cl}"
-eval sv_dir="\$T_D${sv}"
-
-echo "== create file for xattr ping pong"
-touch "$sv_dir/file"
-$SETFATTR -n user.xat -v initial "$sv_dir/file"
-$GETFATTR -n user.xat "$sv_dir/file" 2>&1 | t_filter_fs
-
-echo "== retry btree block read"
-$SETFATTR -n user.xat -v btree "$sv_dir/file"
-t_trigger_arm btree_stale_read $cl
-old=$(t_counter btree_stale_read $cl)
-$GETFATTR -n user.xat "$cl_dir/file" 2>&1 | t_filter_fs
-t_trigger_show btree_stale_read "after" $cl
-t_counter_diff btree_stale_read $old $cl
-
-t_pass