From 81879c693ceed62374ca5cefe141b0c1fe3dade5 Mon Sep 17 00:00:00 2001 From: Chris Kirby Date: Thu, 24 Sep 2026 16:44:08 -0500 Subject: [PATCH] Add block-quorum-port test Currently this test reproduces a scenario where disabling the quorum port on a follower results in that node trying to have an election on increasing terms. When the quorum port is reopened, a new leader is elected and fences the previous one. Signed-off-by: Chris Kirby --- tests/funcs/filter.sh | 3 + tests/funcs/require.sh | 10 +++ tests/golden/block-quorum-port | 5 ++ tests/sequence | 1 + tests/tests/block-quorum-port.sh | 101 +++++++++++++++++++++++++++++++ 5 files changed, 120 insertions(+) create mode 100644 tests/golden/block-quorum-port create mode 100644 tests/tests/block-quorum-port.sh diff --git a/tests/funcs/filter.sh b/tests/funcs/filter.sh index 831702e3..ec3b86c2 100644 --- a/tests/funcs/filter.sh +++ b/tests/funcs/filter.sh @@ -236,6 +236,9 @@ t_filter_dmesg() re="$re|File: /mnt/test.* PID:.* Comm: dd" re="$re|dio_warn_stale_pagecache*" + # block-quorum-port expects the disconnected client to advance the term + re="$re|scoutfs .* warning: saw msg type 0 from .* while leader in term" + egrep -v "($re)" | \ ignore_harmless_unwind_kasan_stack_oob | \ ignore_harmless_xfs_lockdep_warning diff --git a/tests/funcs/require.sh b/tests/funcs/require.sh index 2d0f36ae..7e410ebd 100644 --- a/tests/funcs/require.sh +++ b/tests/funcs/require.sh @@ -24,6 +24,16 @@ t_require_mounts() { t_skip "$req mounts required, only have $T_NR_MOUNTS" } +# +# make sure we have at least this many quorum members +# +t_require_quorum() { + local req="$1" + + test "$T_QUORUM" -ge "$req" || \ + t_skip "$req quorum members required, only have $T_QUORUM" +} + # # Require that the meta device be at least the size string argument, as # parsed by numfmt using single char base 2 suffixes (iec).. 64G, etc. diff --git a/tests/golden/block-quorum-port b/tests/golden/block-quorum-port new file mode 100644 index 00000000..5cee0f9b --- /dev/null +++ b/tests/golden/block-quorum-port @@ -0,0 +1,5 @@ +== disabling quorum port +== waiting for victim's term to advance +== enabling quorum port +== wait for a new leader +== make sure the original leader was not fenced diff --git a/tests/sequence b/tests/sequence index 05e9c1e5..261502da 100644 --- a/tests/sequence +++ b/tests/sequence @@ -45,6 +45,7 @@ stage-release-race-alloc.sh stage-multi-part.sh o_tmpfile.sh basic-posix-consistency.sh +block-quorum-port.sh dirent-consistency.sh mkdir-rename-rmdir.sh lock-ex-race-processes.sh diff --git a/tests/tests/block-quorum-port.sh b/tests/tests/block-quorum-port.sh new file mode 100644 index 00000000..4bdc6f6f --- /dev/null +++ b/tests/tests/block-quorum-port.sh @@ -0,0 +1,101 @@ +# +# Make sure that a fenced client regaining connectivity doesn't fence the +# current leader after a new election. +# +# 1) disable the quorum network port for a follower victim +# 2) wait for the victim to start issuing vote requests for increasing terms +# 3) reenable the quorum network port +# 4) watch for a new leader +# 5) make sure that the new leader has not fenced the previous one + +t_require_commands grep iptables scoutfs +t_require_mounts 3 +t_require_quorum 2 + +orig_leader="$(t_server_nr)" +orig_leader_sysfs="$(t_sysfs_path $orig_leader)" + +t_select_victim() +{ + local nr; + + for nr in $(t_quorum_nrs) + do + if [ "$nr" -ne "$orig_leader" ] + then + echo $nr + break + fi + done +} + +t_disable_qport() +{ + local slot="$1" + local port="$((T_PORT_START + slot))" + + iptables -I INPUT -p tcp --dport $port -j DROP + iptables -I INPUT -p udp --dport $port -j DROP + iptables -I OUTPUT -p tcp --sport $port -j DROP + iptables -I OUTPUT -p udp --sport $port -j DROP +} + +t_enable_qport() +{ + iptables -D INPUT 1 + iptables -D OUTPUT 1 + iptables -D INPUT 1 + iptables -D OUTPUT 1 +} + +t_get_victim_term() +{ + echo "$(cat $(t_sysfs_path $victim)/quorum/status | egrep '^term' | awk '{print $2}')" +} + +t_wait_for_term_advance() +{ + local new_term + + while true + do + new_term="$(t_get_victim_term)" + if [ $((new_term - orig_victim_term)) -gt 10 ] + then + break + fi + sleep 1 + done +} + +victim="$(t_select_victim)" +orig_victim_term="$(t_get_victim_term)" + +echo "== disabling quorum port" +t_disable_qport "$victim" + +echo "== waiting for victim's term to advance" +t_wait_for_term_advance + +echo "== enabling quorum port" +t_enable_qport + +echo "== wait for a new leader" +while true +do + t_wait_for_leader + if [ "$(t_fs_is_leader $orig_leader)" -eq "1" ] + then + sleep 1 + else + break + fi +done + +echo "== make sure the original leader was not fenced" +if [ ! -d "$orig_leader_sysfs" ] +then + echo Original leader was fenced +fi + +t_pass