diff --git a/tests/funcs/fs.sh b/tests/funcs/fs.sh index 03fb9888..b68bcfe7 100644 --- a/tests/funcs/fs.sh +++ b/tests/funcs/fs.sh @@ -17,6 +17,17 @@ t_sync_seq_index() t_quiet sync } +t_mount_rid() +{ + local nr="${1:-0}" + local mnt="$(eval echo \$T_M$nr)" + local rid + + rid=$(scoutfs statfs -s rid -p "$mnt") + + echo "$rid" +} + # # Output the "f.$fsid.r.$rid" identifier string for the given mount # number, 0 is used by default if none is specified. @@ -132,6 +143,16 @@ t_umount() eval t_quiet umount \$T_M$nr } +t_force_umount() +{ + local nr="$1" + + test "$nr" -lt "$T_NR_MOUNTS" || \ + t_fail "fs nr $nr invalid" + + eval t_quiet umount -f \$T_M$nr +} + # # Attempt to mount all the configured mounts, assuming that they're # not already mounted. @@ -277,3 +298,67 @@ t_counter_diff_changed() { echo "counter $which didn't change" || echo "counter $which changed" } + +# +# See if we can find a local mount with the caller's rid. +# +t_rid_is_mounted() { + local rid="$1" + local fr="$1" + + for fr in /sys/fs/scoutfs/*; do + if [ "$(cat $fr/rid)" == "$rid" ]; then + return 0 + fi + done + + return 1 +} + +# +# A given mount is being fenced if any mount has a fence request pending +# for it which hasn't finished and been removed. +# +t_rid_is_fencing() { + local rid="$1" + local fr + + for fr in /sys/fs/scoutfs/*; do + if [ -d "$fr/fence/$rid" ]; then + return 0 + fi + done + + return 1 +} + +# +# Wait until the mount identified by the first rid arg is not in any +# states specified by the remaining state description word args. +# +t_wait_if_rid_is() { + local rid="$1" + + while ( [[ $* =~ mounted ]] && t_rid_is_mounted $rid ) || + ( [[ $* =~ fencing ]] && t_rid_is_fencing $rid ) ; do + sleep .5 + done +} + +# +# Wait until any mount identifies itself as the elected leader. We can +# be waiting while tests mount and unmount so mounts may not be mounted +# at the test's expected mount points. +# +t_wait_for_leader() { + local i + + while sleep .25; do + for i in $(t_fs_nrs); do + local ldr="$(t_sysfs_path $i 2>/dev/null)/quorum/is_leader" + if [ "$(cat $ldr 2>/dev/null)" == "1" ]; then + return + fi + done + done +} diff --git a/tests/golden/fence-and-reclaim b/tests/golden/fence-and-reclaim new file mode 100644 index 00000000..dadc5edd --- /dev/null +++ b/tests/golden/fence-and-reclaim @@ -0,0 +1,5 @@ +== make sure all mounts can see each other +== force unmount one client, connection timeout, fence nop, mount +== force unmount all non-server, connection timeout, fence nop, mount +== force unmount server, quorum elects new leader, fence nop, mount +== force unmount everything, new server fences all previous diff --git a/tests/sequence b/tests/sequence index 972bebe2..0d709c1d 100644 --- a/tests/sequence +++ b/tests/sequence @@ -28,6 +28,7 @@ lock-conflicting-batch-commit.sh cross-mount-data-free.sh persistent-item-vers.sh setup-error-teardown.sh +fence-and-reclaim.sh mount-unmount-race.sh createmany-parallel-mounts.sh archive-light-cycle.sh diff --git a/tests/tests/fence-and-reclaim.sh b/tests/tests/fence-and-reclaim.sh new file mode 100644 index 00000000..1ce52048 --- /dev/null +++ b/tests/tests/fence-and-reclaim.sh @@ -0,0 +1,127 @@ +# +# Fence nodes and reclaim their resources. +# + +t_require_commands sleep touch grep sync scoutfs +t_require_mounts 2 + +# +# Make sure that all mounts can read the results of a write from each +# mount. And make sure that the greatest of all the written seqs is +# visible after the writes were commited by remote reads. +# +check_read_write() +{ + local expected + local greatest=0 + local seq + local path + local saw + local w + local r + + for w in $(t_fs_nrs); do + expected="$w wrote at $(date --rfc-3339=ns)" + eval path="\$T_D${w}/written" + echo "$expected" > "$path" + + seq=$(scoutfs stat -s meta_seq $path) + if [ "$seq" -gt "$greatest" ]; then + greatest=$seq + fi + + for r in $(t_fs_nrs); do + eval path="\$T_D${r}/written" + saw=$(cat "$path") + if [ "$saw" != "$expected" ]; then + echo "mount $r read '$saw' after mount $w wrote '$expected'" + fi + done + done + + seq=$(scoutfs statfs -s committed_seq -p $T_D0) + if [ "$seq" -lt "$greatest" ]; then + echo "committed_seq $seq less than greatest $greatest" + fi +} + +echo "== make sure all mounts can see each other" +check_read_write + +echo "== force unmount one client, connection timeout, fence nop, mount" +cl=$(t_first_client_nr) +sv=$(t_server_nr) +rid=$(t_mount_rid $cl) +echo "cl $cl sv $sv rid $rid" >> "$T_TMP.log" +sync +t_force_umount $cl +# wait for client reconnection to timeout +while grep -q $rid $(t_debugfs_path $sv)/connections; do + sleep .5 +done +while t_rid_is_fencing $rid; do + sleep .5 +done +t_mount $cl +check_read_write + +echo "== force unmount all non-server, connection timeout, fence nop, mount" +sv=$(t_server_nr) +pattern="nonsense" +sync +for cl in $(t_fs_nrs); do + if [ $cl == $sv ]; then + continue; + fi + + rid=$(t_mount_rid $cl) + pattern="$pattern|$rid" + echo "cl $cl sv $sv rid $rid" >> "$T_TMP.log" + + t_force_umount $cl +done + +# wait for all client reconnections to timeout +while egrep -q "($pattern)" $(t_debugfs_path $sv)/connections; do + sleep .5 +done +# wait for all fence requests to complete +while test -d $(echo /sys/fs/scoutfs/*/fence/* | cut -d " " -f 1); do + sleep .5 +done +# remount all the clients +for cl in $(t_fs_nrs); do + if [ $cl == $sv ]; then + continue; + fi + t_mount $cl +done +check_read_write + +echo "== force unmount server, quorum elects new leader, fence nop, mount" +sv=$(t_server_nr) +rid=$(t_mount_rid $sv) +echo "sv $sv rid $rid" >> "$T_TMP.log" +sync +t_force_umount $sv +t_wait_for_leader +# wait until new server is done fencing unmounted leader rid +while t_rid_is_fencing $rid; do + sleep .5 +done +t_mount $sv +check_read_write + +echo "== force unmount everything, new server fences all previous" +sync +for nr in $(t_fs_nrs); do + t_force_umount $nr +done +t_mount_all +# wait for all fence requests to complete +while test -d $(echo /sys/fs/scoutfs/*/fence/* | cut -d " " -f 1); do + sleep .5 +done +check_read_write + +t_pass