Add block-quorum-port test

Currently this test reproduces a scenario where disabling the
quorum port on a follower results in that node trying to have
an election on increasing terms. When the quorum port is reopened,
a new leader is elected and fences the previous one.

Signed-off-by: Chris Kirby <ckirby@versity.com>

tests/funcs/filter.sh

@@ -236,6 +236,9 @@ t_filter_dmesg()
 	re="$re|File: /mnt/test.* PID:.* Comm: dd"
 	re="$re|dio_warn_stale_pagecache*"
 
+	# block-quorum-port expects the disconnected client to advance the term
+	re="$re|scoutfs .* warning: saw msg type 0 from .* while leader in term"
+
 	egrep -v "($re)" | \
 		ignore_harmless_unwind_kasan_stack_oob | \
 		ignore_harmless_xfs_lockdep_warning

tests/funcs/require.sh

@@ -24,6 +24,16 @@ t_require_mounts() {
 		t_skip "$req mounts required, only have $T_NR_MOUNTS"
 }
 
+#
+# make sure we have at least this many quorum members
+#
+t_require_quorum() {
+	local req="$1"
+
+	test "$T_QUORUM" -ge "$req" || \
+		t_skip "$req quorum members required, only have $T_QUORUM"
+}
+
 #
 # Require that the meta device be at least the size string argument, as
 # parsed by numfmt using single char base 2 suffixes (iec).. 64G, etc.

tests/golden/block-quorum-port

@@ -0,0 +1,5 @@
+== disabling quorum port
+== waiting for victim's term to advance
+== enabling quorum port
+== wait for a new leader
+== make sure the original leader was not fenced

tests/sequence

@@ -45,6 +45,7 @@ stage-release-race-alloc.sh
 stage-multi-part.sh
 o_tmpfile.sh
 basic-posix-consistency.sh
+block-quorum-port.sh
 dirent-consistency.sh
 mkdir-rename-rmdir.sh
 lock-ex-race-processes.sh

tests/tests/block-quorum-port.sh

@@ -0,0 +1,101 @@
+#
+# Make sure that a fenced client regaining connectivity doesn't fence the
+# current leader after a new election.
+#
+# 1) disable the quorum network port for a follower victim
+# 2) wait for the victim to start issuing vote requests for increasing terms
+# 3) reenable the quorum network port
+# 4) watch for a new leader
+# 5) make sure that the new leader has not fenced the previous one
+
+t_require_commands grep iptables scoutfs
+t_require_mounts 3
+t_require_quorum 2
+
+orig_leader="$(t_server_nr)"
+orig_leader_sysfs="$(t_sysfs_path $orig_leader)"
+
+t_select_victim()
+{
+	local nr;
+
+	for nr in $(t_quorum_nrs)
+	do
+		if [ "$nr" -ne "$orig_leader" ]
+		then
+			echo $nr
+			break
+		fi
+	done
+}
+
+t_disable_qport()
+{
+	local slot="$1"
+	local port="$((T_PORT_START + slot))"
+
+	iptables -I INPUT -p tcp --dport $port -j DROP
+	iptables -I INPUT -p udp --dport $port -j DROP
+	iptables -I OUTPUT -p tcp --sport $port -j DROP
+	iptables -I OUTPUT -p udp --sport $port -j DROP
+}
+
+t_enable_qport()
+{
+	iptables -D INPUT 1
+	iptables -D OUTPUT 1
+	iptables -D INPUT 1
+	iptables -D OUTPUT 1
+}
+
+t_get_victim_term()
+{
+	echo "$(cat $(t_sysfs_path $victim)/quorum/status | egrep '^term' | awk '{print $2}')"
+}
+
+t_wait_for_term_advance()
+{
+	local new_term
+
+	while true
+	do
+		new_term="$(t_get_victim_term)"
+		if [ $((new_term - orig_victim_term)) -gt 10 ]
+		then
+			break
+		fi
+		sleep 1
+	done
+}
+
+victim="$(t_select_victim)"
+orig_victim_term="$(t_get_victim_term)"
+
+echo "== disabling quorum port"
+t_disable_qport "$victim"
+
+echo "== waiting for victim's term to advance"
+t_wait_for_term_advance
+
+echo "== enabling quorum port"
+t_enable_qport
+
+echo "== wait for a new leader"
+while true
+do
+	t_wait_for_leader
+	if [ "$(t_fs_is_leader $orig_leader)" -eq "1" ]
+	then
+		sleep 1
+	else
+		break
+	fi
+done
+
+echo "== make sure the original leader was not fenced"
+if [ ! -d "$orig_leader_sysfs" ]
+then
+	echo Original leader was fenced
+fi
+
+t_pass