From 8a504cd5ae074c5a269aa1b20ab9bef6c5f527b1 Mon Sep 17 00:00:00 2001
From: "Bryant G. Duffy-Ly" <bduffyly@versity.com>
Date: Tue, 14 Dec 2021 13:39:32 -0600
Subject: [PATCH] Add client/server unmount race on lock_recov unit test

This unit test reproduces the race we have between
client and server diong lock recovery while farewell
is processed.

Signed-off-by: Bryant G. Duffy-Ly <bduffyly@versity.com>
---
 tests/golden/client-unmount-recovery   |  1 +
 tests/sequence                         |  1 +
 tests/tests/client-unmount-recovery.sh | 61 ++++++++++++++++++++++++++
 3 files changed, 63 insertions(+)
 create mode 100644 tests/golden/client-unmount-recovery
 create mode 100644 tests/tests/client-unmount-recovery.sh

diff --git a/tests/golden/client-unmount-recovery b/tests/golden/client-unmount-recovery
new file mode 100644
index 00000000..866ad611
--- /dev/null
+++ b/tests/golden/client-unmount-recovery
@@ -0,0 +1 @@
+== 60s of unmounting non-quorum clients during recovery
diff --git a/tests/sequence b/tests/sequence
index a168e287..c6e463f1 100644
--- a/tests/sequence
+++ b/tests/sequence
@@ -33,6 +33,7 @@ resize-devices.sh
 fence-and-reclaim.sh
 orphan-inodes.sh
 mount-unmount-race.sh
+client-unmount-recovery.sh
 createmany-parallel-mounts.sh
 archive-light-cycle.sh
 block-stale-reads.sh
diff --git a/tests/tests/client-unmount-recovery.sh b/tests/tests/client-unmount-recovery.sh
new file mode 100644
index 00000000..44e6233a
--- /dev/null
+++ b/tests/tests/client-unmount-recovery.sh
@@ -0,0 +1,61 @@
+#
+# Unmount Server and unmount a client as it's replaying to a remaining server
+#
+
+majority_nr=$(t_majority_count)
+quorum_nr=$T_QUORUM
+
+test "$quorum_nr" == "$majority_nr" && \
+        t_skip "all quorum members make up majority, need more mounts to unmount"
+
+test "$T_NR_MOUNTS" -lt "$T_QUORUM" && \
+        t_skip "Need enough non-quorum clients to unmount"
+
+for i in $(t_fs_nrs); do
+        mounted[$i]=1
+done
+
+LENGTH=60
+echo "== ${LENGTH}s of unmounting non-quorum clients during recovery"
+END=$((SECONDS + LENGTH))
+while [ "$SECONDS" -lt "$END" ]; do
+        sv=$(t_server_nr)
+        rid=$(t_mount_rid $sv)
+        echo "sv $sv rid $rid" >> "$T_TMP.log"
+        sync
+        t_umount $sv &
+
+        for i in $(t_fs_nrs); do
+                if [ "$i" -ge "$quorum_nr" ]; then
+                        t_umount $i &
+                        echo "umount $i pid $pid quo $quorum_nr" \
+                                >> $T_TMP.log
+                        mounted[$i]=0
+                fi
+        done
+
+        wait
+
+        t_mount $sv &
+        for i in $(t_fs_nrs); do
+                if [ "${mounted[$i]}" == 0 ]; then
+                        t_mount $i &
+                fi
+        done
+
+        wait
+
+        declare RID_LIST=$(cat /sys/fs/scoutfs/*/rid | sort -u)
+        read -a rid_arr <<< $RID_LIST
+
+        declare LOCK_LIST=$(cut -d' ' -f 5 /sys/kernel/debug/scoutfs/*/server_locks | sort -u)
+        read -a lock_arr <<< $LOCK_LIST
+
+        for i in "${lock_arr[@]}"; do
+                if [[ ! " ${rid_arr[*]} " =~ " $i " ]]; then
+                        t_fail "RID($i): exists when not mounted"
+                fi
+        done
+done
+
+t_pass