From 208c51d1d24fb59c881c4c67b8ab535d761e4728 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 24 Feb 2021 12:41:14 -0800 Subject: [PATCH] Update stale block reading test The previous test that triggered re-reading blocks, as though they were stale, was written in the era where it only hit btree blocks and everything else was stored in LSM segments. This reworks the test to make it clear that it affects all our block readers today. The test only exercise the core read retry path, but it could be expanded to test callers retrying with newer references after they get -ESTALE errors. Signed-off-by: Zach Brown --- kmod/src/block.c | 4 +++- kmod/src/triggers.c | 1 + kmod/src/triggers.h | 1 + tests/golden/block-stale-reads | 29 +++++++++++++++++++++++ tests/golden/stale-btree-read | 11 --------- tests/sequence | 2 +- tests/tests/block-stale-reads.sh | 40 ++++++++++++++++++++++++++++++++ tests/tests/stale-btree-read.sh | 40 -------------------------------- 8 files changed, 75 insertions(+), 53 deletions(-) create mode 100644 tests/golden/block-stale-reads delete mode 100644 tests/golden/stale-btree-read create mode 100644 tests/tests/block-stale-reads.sh delete mode 100644 tests/tests/stale-btree-read.sh diff --git a/kmod/src/block.c b/kmod/src/block.c index 6d93e116..e13aa8e0 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -29,6 +29,7 @@ #include "msg.h" #include "scoutfs_trace.h" #include "alloc.h" +#include "triggers.h" /* * The scoutfs block cache manages metadata blocks that can be larger @@ -682,7 +683,8 @@ retry: ret = 0; out: - if (ret == -ESTALE && !retried && !block_is_dirty(bp)) { + if ((ret == -ESTALE || scoutfs_trigger(sb, BLOCK_REMOVE_STALE)) && + !retried && !block_is_dirty(bp)) { retried = true; scoutfs_inc_counter(sb, block_cache_remove_stale); block_remove(sb, bp); diff --git a/kmod/src/triggers.c b/kmod/src/triggers.c index a94f2b65..75ba07d7 100644 --- a/kmod/src/triggers.c +++ b/kmod/src/triggers.c @@ -38,6 +38,7 @@ struct scoutfs_triggers { struct scoutfs_triggers *name = SCOUTFS_SB(sb)->triggers static char *names[] = { + [SCOUTFS_TRIGGER_BLOCK_REMOVE_STALE] = "block_remove_stale", [SCOUTFS_TRIGGER_BTREE_STALE_READ] = "btree_stale_read", [SCOUTFS_TRIGGER_BTREE_ADVANCE_RING_HALF] = "btree_advance_ring_half", [SCOUTFS_TRIGGER_HARD_STALE_ERROR] = "hard_stale_error", diff --git a/kmod/src/triggers.h b/kmod/src/triggers.h index 8796cd18..d3d2e0f6 100644 --- a/kmod/src/triggers.h +++ b/kmod/src/triggers.h @@ -2,6 +2,7 @@ #define _SCOUTFS_TRIGGERS_H_ enum scoutfs_trigger { + SCOUTFS_TRIGGER_BLOCK_REMOVE_STALE, SCOUTFS_TRIGGER_BTREE_STALE_READ, SCOUTFS_TRIGGER_BTREE_ADVANCE_RING_HALF, SCOUTFS_TRIGGER_HARD_STALE_ERROR, diff --git a/tests/golden/block-stale-reads b/tests/golden/block-stale-reads new file mode 100644 index 00000000..51f11b28 --- /dev/null +++ b/tests/golden/block-stale-reads @@ -0,0 +1,29 @@ +== create file for xattr ping pong +# file: /mnt/test/test/block-stale-reads/file +user.xat="initial" + +== retry btree forest reads between mounts +trigger block_remove_stale armed: 0 +# file: /mnt/test/test/block-stale-reads/file +user.xat="1" + +trigger block_remove_stale after: 0 +counter block_cache_remove_stale diff 1 +trigger block_remove_stale armed: 0 +# file: /mnt/test/test/block-stale-reads/file +user.xat="2" + +trigger block_remove_stale after: 0 +counter block_cache_remove_stale diff 2 +trigger block_remove_stale armed: 0 +# file: /mnt/test/test/block-stale-reads/file +user.xat="3" + +trigger block_remove_stale after: 0 +counter block_cache_remove_stale diff 3 +trigger block_remove_stale armed: 0 +# file: /mnt/test/test/block-stale-reads/file +user.xat="4" + +trigger block_remove_stale after: 0 +counter block_cache_remove_stale diff 4 diff --git a/tests/golden/stale-btree-read b/tests/golden/stale-btree-read deleted file mode 100644 index 07cb3bf1..00000000 --- a/tests/golden/stale-btree-read +++ /dev/null @@ -1,11 +0,0 @@ -== create file for xattr ping pong -# file: /mnt/test/test/stale-btree-read/file -user.xat="initial" - -== retry btree block read -trigger btree_stale_read armed: 1 -# file: /mnt/test/test/stale-btree-read/file -user.xat="btree" - -trigger btree_stale_read after: 0 -counter btree_stale_read diff 1 diff --git a/tests/sequence b/tests/sequence index 8cd44c87..764ec501 100644 --- a/tests/sequence +++ b/tests/sequence @@ -28,5 +28,5 @@ setup-error-teardown.sh mount-unmount-race.sh createmany-parallel-mounts.sh archive-light-cycle.sh -stale-btree-read.sh +block-stale-reads.sh xfstests.sh diff --git a/tests/tests/block-stale-reads.sh b/tests/tests/block-stale-reads.sh new file mode 100644 index 00000000..a41d821d --- /dev/null +++ b/tests/tests/block-stale-reads.sh @@ -0,0 +1,40 @@ +# +# exercise stale block reading. +# +# It would be very difficult to manipulate the allocators, cache, and +# persistent blocks to create stable block reading scenarios. Instead +# we use triggers to exercise how readers encounter stale blocks. +# + +t_require_commands touch setfattr getfattr +t_require_mounts 2 + +GETFATTR="getfattr --absolute-names" +SETFATTR="setfattr" + +# +# force re-reading forest btree blocks as each mount reads the items +# written by the other. +# +set_file="$T_D0/file" +get_file="$T_D1/file" +echo "== create file for xattr ping pong" +touch "$set_file" +$SETFATTR -n user.xat -v initial "$set_file" +$GETFATTR -n user.xat "$get_file" 2>&1 | t_filter_fs + +echo "== retry btree forest reads between mounts" +for i in $(seq 1 4); do + tmp="$set_file" + set_file="$get_file" + get_file="$tmp" + + $SETFATTR -n user.xat -v $i "$set_file" + t_trigger_arm block_remove_stale $cl + old=$(t_counter btree_stale_read $cl) + $GETFATTR -n user.xat "$get_file" 2>&1 | t_filter_fs + t_trigger_show block_remove_stale "after" $cl + t_counter_diff block_cache_remove_stale $old $cl +done + +t_pass diff --git a/tests/tests/stale-btree-read.sh b/tests/tests/stale-btree-read.sh deleted file mode 100644 index 36c43cb7..00000000 --- a/tests/tests/stale-btree-read.sh +++ /dev/null @@ -1,40 +0,0 @@ -# -# verify stale btree block reading -# - -t_require_commands touch stat setfattr getfattr createmany -t_require_mounts 2 - -GETFATTR="getfattr --absolute-names" -SETFATTR="setfattr" - -# -# This exercises the soft retry of btree blocks when -# inconsistent cached versions are found. It ensures that basic hard -# error returning turns into EIO in the case where the persistent reread -# blocks and segments really are inconsistent. -# -# The triggers apply across all execution in the file system. So to -# trigger btree block retries in the client we make sure that the server -# is running on the other node. -# - -cl=$(t_first_client_nr) -sv=$(t_server_nr) -eval cl_dir="\$T_D${cl}" -eval sv_dir="\$T_D${sv}" - -echo "== create file for xattr ping pong" -touch "$sv_dir/file" -$SETFATTR -n user.xat -v initial "$sv_dir/file" -$GETFATTR -n user.xat "$sv_dir/file" 2>&1 | t_filter_fs - -echo "== retry btree block read" -$SETFATTR -n user.xat -v btree "$sv_dir/file" -t_trigger_arm btree_stale_read $cl -old=$(t_counter btree_stale_read $cl) -$GETFATTR -n user.xat "$cl_dir/file" 2>&1 | t_filter_fs -t_trigger_show btree_stale_read "after" $cl -t_counter_diff btree_stale_read $old $cl - -t_pass