Test export-lookup-evict-race in a loop with tracing.

This test hits the unmount hang consistently in our CI the most,
so run it in a tight loop and enable tracing. Discard traces when
the run succeeded.

This will blow up if a hung task timeout occurs, so we should
crash on panic and then extract traces from the crash. Make sure
we don't wait for an hour before doing so, then, too.

Signed-off-by: Auke Kok <auke.kok@versity.com>

kmod/src/server.c

@@ -3036,7 +3036,13 @@ static int server_commit_log_merge(struct super_block *sb,
 				  SCOUTFS_LOG_MERGE_STATUS_ZONE, 0, 0,
 				  &stat, sizeof(stat));
 	if (ret < 0) {
-		err_str = "getting merge status item";
+		/*
+		 * During a retransmission, it's possible that the server
+		 * already committed and resolved this log merge. ENOENT
+		 * is expected in that case.
+		 */
+		if (ret != -ENOENT)
+			err_str = "getting merge status item";
 		goto out;
 	}
 

tests/fenced-local-force-unmount.sh

@@ -9,7 +9,7 @@
 echo "$0 running rid '$SCOUTFS_FENCED_REQ_RID' ip '$SCOUTFS_FENCED_REQ_IP' args '$@'"
 
 echo_fail() {
-	echo "$@" >> /dev/stderr
+	echo "$@" >&2
 	exit 1
 }
 
@@ -27,8 +27,7 @@ for fs in /sys/fs/scoutfs/*; do
 	nr="$(quiet_cat $fs/data_device_maj_min)"
 	[ ! -d "$fs" -o "$fs_rid" != "$rid" ] && continue
 
-	mnt=$(findmnt -l -n -t scoutfs -o TARGET -S $nr) || \
-		echo_fail "findmnt -t scoutfs -S $nr failed"
+	mnt=$(findmnt -l -n -t scoutfs -o TARGET -S $nr)
 	[ -z "$mnt" ] && continue
 
 	if ! umount -qf "$mnt"; then

tests/run-tests.sh

@@ -92,10 +92,14 @@ done
 T_TRACE_DUMP="0"
 T_TRACE_PRINTK="0"
 T_PORT_START="19700"
-T_LOOP_ITER="1"
+T_LOOP_ITER="100"
 
 # array declarations to be able to use array ops
 declare -a T_TRACE_GLOB
+T_TRACE_GLOB=( "scoutfs*" )
+
+# CI sets this to 3600, but, for this case we want it very short
+echo 30 > /proc/sys/kernel/hung_task_timeout_secs
 
 while true; do
 	case $1 in
@@ -493,6 +497,11 @@ crash_monitor()
 			bad=1
 		fi
 
+		if dmesg | grep -q "blocked for more than"; then
+			echo "run-tests monitor saw blocked task message"
+			bad=1
+		fi
+
 		if dmesg | grep -q "error indicated by fence action" ; then
 			echo "run-tests monitor saw fence agent error message"
 			bad=1
@@ -504,6 +513,8 @@ crash_monitor()
 		fi
 
 		if [ "$bad" != 0 ]; then
+			sync & # maybe this gets logs synced...
+			sleep .1
 			echo "run-tests monitor triggering crash"
 			echo c > /proc/sysrq-trigger
 			exit 1
@@ -706,6 +717,8 @@ for t in $tests; do
 		# stop looping if we didn't pass
 		if [ "$sts" != "$T_PASS_STATUS" ]; then
 			break;
+		else
+			echo > /sys/kernel/debug/tracing/trace
 		fi
 	done
 

tests/sequence

@@ -1,60 +1 @@
-export-get-name-parent.sh
-basic-block-counts.sh
-basic-bad-mounts.sh
-basic-posix-acl.sh
-inode-items-updated.sh
-simple-inode-index.sh
-simple-staging.sh
-simple-release-extents.sh
-simple-readdir.sh
-get-referring-entries.sh
-fallocate.sh
-basic-truncate.sh
-data-prealloc.sh
-setattr_more.sh
-offline-extent-waiting.sh
-move-blocks.sh
-projects.sh
-large-fragmented-free.sh
-format-version-forward-back.sh
-enospc.sh
-mmap.sh
-srch-safe-merge-pos.sh
-srch-basic-functionality.sh
-simple-xattr-unit.sh
-retention-basic.sh
-totl-xattr-tag.sh
-quota.sh
-lock-refleak.sh
-lock-shrink-consistency.sh
-lock-shrink-read-race.sh
-lock-pr-cw-conflict.sh
-lock-revoke-getcwd.sh
-lock-recover-invalidate.sh
 export-lookup-evict-race.sh
-createmany-parallel.sh
-createmany-large-names.sh
-createmany-rename-large-dir.sh
-stage-release-race-alloc.sh
-stage-multi-part.sh
-o_tmpfile.sh
-basic-posix-consistency.sh
-dirent-consistency.sh
-mkdir-rename-rmdir.sh
-lock-ex-race-processes.sh
-cross-mount-data-free.sh
-persistent-item-vers.sh
-setup-error-teardown.sh
-resize-devices.sh
-change-devices.sh
-fence-and-reclaim.sh
-quorum-heartbeat-timeout.sh
-orphan-inodes.sh
-mount-unmount-race.sh
-client-unmount-recovery.sh
-createmany-parallel-mounts.sh
-archive-light-cycle.sh
-block-stale-reads.sh
-inode-deletion.sh
-renameat2-noreplace.sh
-xfstests.sh

tests/tests/renameat2-noreplace.sh

@@ -8,19 +8,19 @@ t_require_mounts 2
 echo "=== renameat2 noreplace flag test"
 
 # give each mount their own dir (lock group) to minimize create contention
-mkdir $T_M0/dir0
-mkdir $T_M1/dir1
+mkdir $T_D0/dir0
+mkdir $T_D1/dir1
 
 echo "=== run two asynchronous calls to renameat2 NOREPLACE"
 for i in $(seq 0 100); do
         # prepare inputs in isolation
-        touch "$T_M0/dir0/old0"
-        touch "$T_M1/dir1/old1"
+        touch "$T_D0/dir0/old0"
+        touch "$T_D1/dir1/old1"
 
         # race doing noreplace renames, both can't succeed
-        dumb_renameat2 -n "$T_M0/dir0/old0" "$T_M0/dir0/sharednew" 2> /dev/null &
+        dumb_renameat2 -n "$T_D0/dir0/old0" "$T_D0/dir0/sharednew" 2> /dev/null &
         pid0=$!
-        dumb_renameat2 -n "$T_M1/dir1/old1" "$T_M1/dir0/sharednew" 2> /dev/null &
+        dumb_renameat2 -n "$T_D1/dir1/old1" "$T_D1/dir0/sharednew" 2> /dev/null &
         pid1=$!
 
         wait $pid0
@@ -31,7 +31,7 @@ for i in $(seq 0 100); do
         test "$rc0" == 0 -a "$rc1" == 0 && t_fail "both renames succeeded"
 
         # blow away possible files for either race outcome
-        rm -f "$T_M0/dir0/old0" "$T_M1/dir1/old1" "$T_M0/dir0/sharednew" "$T_M1/dir1/sharednew"
+        rm -f "$T_D0/dir0/old0" "$T_D1/dir1/old1" "$T_D0/dir0/sharednew" "$T_D1/dir1/sharednew"
 done
 
 t_pass

utils/fenced/scoutfs-fenced

@@ -7,7 +7,7 @@ message_output()
 
 error_message()
 {
-	message_output "$@" >> /dev/stderr
+	message_output "$@" >&2
 }
 
 error_exit()