Check for fenced old leader in mounted test.

The old mounted check only considered begin/end quorum data, and
not whether the old leader that is now disconnected was fenced by
a new quorum leader.

Since this is the quaranteed case if the leader is disconnected
forcefully, this check must account for this case, so that quorum slots
can be modified if the node is permanently removed or replaced.

Signed-off-by: Auke Kok <auke.kok@versity.com>

tests/fenced-local-force-unmount.sh

@@ -9,7 +9,7 @@
 echo "$0 running rid '$SCOUTFS_FENCED_REQ_RID' ip '$SCOUTFS_FENCED_REQ_IP' args '$@'"
 
 echo_fail() {
-	echo "$@" >&2
+	echo "$@" >> /dev/stderr
 	exit 1
 }
 

tests/run-tests.sh

@@ -92,14 +92,10 @@ done
 T_TRACE_DUMP="0"
 T_TRACE_PRINTK="0"
 T_PORT_START="19700"
-T_LOOP_ITER="100"
+T_LOOP_ITER="1"
 
 # array declarations to be able to use array ops
 declare -a T_TRACE_GLOB
-T_TRACE_GLOB=( "scoutfs*" )
-
-# CI sets this to 3600, but, for this case we want it very short
-echo 30 > /proc/sys/kernel/hung_task_timeout_secs
 
 while true; do
 	case $1 in
@@ -497,11 +493,6 @@ crash_monitor()
 			bad=1
 		fi
 
-		if dmesg | grep -q "blocked for more than"; then
-			echo "run-tests monitor saw blocked task message"
-			bad=1
-		fi
-
 		if dmesg | grep -q "error indicated by fence action" ; then
 			echo "run-tests monitor saw fence agent error message"
 			bad=1
@@ -513,8 +504,6 @@ crash_monitor()
 		fi
 
 		if [ "$bad" != 0 ]; then
-			sync & # maybe this gets logs synced...
-			sleep .1
 			echo "run-tests monitor triggering crash"
 			echo c > /proc/sysrq-trigger
 			exit 1
@@ -717,8 +706,6 @@ for t in $tests; do
 		# stop looping if we didn't pass
 		if [ "$sts" != "$T_PASS_STATUS" ]; then
 			break;
-		else
-			echo > /sys/kernel/debug/tracing/trace
 		fi
 	done
 

tests/sequence

@@ -1 +1,60 @@
+export-get-name-parent.sh
+basic-block-counts.sh
+basic-bad-mounts.sh
+basic-posix-acl.sh
+inode-items-updated.sh
+simple-inode-index.sh
+simple-staging.sh
+simple-release-extents.sh
+simple-readdir.sh
+get-referring-entries.sh
+fallocate.sh
+basic-truncate.sh
+data-prealloc.sh
+setattr_more.sh
+offline-extent-waiting.sh
+move-blocks.sh
+projects.sh
+large-fragmented-free.sh
+format-version-forward-back.sh
+enospc.sh
+mmap.sh
+srch-safe-merge-pos.sh
+srch-basic-functionality.sh
+simple-xattr-unit.sh
+retention-basic.sh
+totl-xattr-tag.sh
+quota.sh
+lock-refleak.sh
+lock-shrink-consistency.sh
+lock-shrink-read-race.sh
+lock-pr-cw-conflict.sh
+lock-revoke-getcwd.sh
+lock-recover-invalidate.sh
 export-lookup-evict-race.sh
+createmany-parallel.sh
+createmany-large-names.sh
+createmany-rename-large-dir.sh
+stage-release-race-alloc.sh
+stage-multi-part.sh
+o_tmpfile.sh
+basic-posix-consistency.sh
+dirent-consistency.sh
+mkdir-rename-rmdir.sh
+lock-ex-race-processes.sh
+cross-mount-data-free.sh
+persistent-item-vers.sh
+setup-error-teardown.sh
+resize-devices.sh
+change-devices.sh
+fence-and-reclaim.sh
+quorum-heartbeat-timeout.sh
+orphan-inodes.sh
+mount-unmount-race.sh
+client-unmount-recovery.sh
+createmany-parallel-mounts.sh
+archive-light-cycle.sh
+block-stale-reads.sh
+inode-deletion.sh
+renameat2-noreplace.sh
+xfstests.sh

tests/tests/renameat2-noreplace.sh

@@ -8,19 +8,19 @@ t_require_mounts 2
 echo "=== renameat2 noreplace flag test"
 
 # give each mount their own dir (lock group) to minimize create contention
-mkdir $T_D0/dir0
-mkdir $T_D1/dir1
+mkdir $T_M0/dir0
+mkdir $T_M1/dir1
 
 echo "=== run two asynchronous calls to renameat2 NOREPLACE"
 for i in $(seq 0 100); do
         # prepare inputs in isolation
-        touch "$T_D0/dir0/old0"
-        touch "$T_D1/dir1/old1"
+        touch "$T_M0/dir0/old0"
+        touch "$T_M1/dir1/old1"
 
         # race doing noreplace renames, both can't succeed
-        dumb_renameat2 -n "$T_D0/dir0/old0" "$T_D0/dir0/sharednew" 2> /dev/null &
+        dumb_renameat2 -n "$T_M0/dir0/old0" "$T_M0/dir0/sharednew" 2> /dev/null &
         pid0=$!
-        dumb_renameat2 -n "$T_D1/dir1/old1" "$T_D1/dir0/sharednew" 2> /dev/null &
+        dumb_renameat2 -n "$T_M1/dir1/old1" "$T_M1/dir0/sharednew" 2> /dev/null &
         pid1=$!
 
         wait $pid0
@@ -31,7 +31,7 @@ for i in $(seq 0 100); do
         test "$rc0" == 0 -a "$rc1" == 0 && t_fail "both renames succeeded"
 
         # blow away possible files for either race outcome
-        rm -f "$T_D0/dir0/old0" "$T_D1/dir1/old1" "$T_D0/dir0/sharednew" "$T_D1/dir1/sharednew"
+        rm -f "$T_M0/dir0/old0" "$T_M1/dir1/old1" "$T_M0/dir0/sharednew" "$T_M1/dir1/sharednew"
 done
 
 t_pass

utils/fenced/scoutfs-fenced

@@ -7,7 +7,7 @@ message_output()
 
 error_message()
 {
-	message_output "$@" >&2
+	message_output "$@" >> /dev/stderr
 }
 
 error_exit()

utils/src/util.c

@@ -198,11 +198,13 @@ int write_block_sync(int fd, u32 magic, __le64 fsid, u64 seq, u64 blkno,
  */
 int meta_super_in_use(int meta_fd, struct scoutfs_super_block *meta_super)
 {
-	struct scoutfs_quorum_block *qblk = NULL;
+	struct scoutfs_quorum_block *qblk[SCOUTFS_QUORUM_BLOCKS] = {NULL,};
 	struct scoutfs_quorum_block_event *beg;
 	struct scoutfs_quorum_block_event *end;
+	struct scoutfs_quorum_block_event *fence;
+	bool beg_was_fenced;
 	int ret = 0;
-	int i;
+	int i, j;
 
 	if (meta_super->mounted_clients.ref.blkno != 0) {
 		fprintf(stderr, "meta superblock mounted clients btree is not empty.\n");
@@ -210,36 +212,61 @@ int meta_super_in_use(int meta_fd, struct scoutfs_super_block *meta_super)
 		goto out;
 	}
 
-	/* check for active quorum slots */
+	/* read all blocks */
 	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
 		if (!quorum_slot_present(meta_super, i))
 			continue;
 		ret = read_block(meta_fd, SCOUTFS_QUORUM_BLKNO + i, SCOUTFS_BLOCK_SM_SHIFT,
-				 (void **)&qblk);
+				 (void **)&qblk[i]);
 		if (ret < 0) {
 			fprintf(stderr, "error reading quorum block for slot %u\n", i);
 			goto out;
 		}
+	}
 
-		beg = &qblk->events[SCOUTFS_QUORUM_EVENT_BEGIN];
-		end = &qblk->events[SCOUTFS_QUORUM_EVENT_END];
+	/* check for active quorum slots */
+	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
+		if (!qblk[i])
+			continue;
 
-		if (le64_to_cpu(beg->write_nr) > le64_to_cpu(end->write_nr)) {
-			fprintf(stderr, "mount in quorum slot %u could still be running.\n"
-					"  begin event: write_nr %llu timestamp %llu.%08u\n"
-					"    end event: write_nr %llu timestamp %llu.%08u\n",
-				i, le64_to_cpu(beg->write_nr), le64_to_cpu(beg->ts.sec),
-				le32_to_cpu(beg->ts.nsec),
-				le64_to_cpu(end->write_nr), le64_to_cpu(end->ts.sec),
-				le32_to_cpu(end->ts.nsec));
-			ret = -EBUSY;
-			goto out;
+		beg = &qblk[i]->events[SCOUTFS_QUORUM_EVENT_BEGIN];
+		end = &qblk[i]->events[SCOUTFS_QUORUM_EVENT_END];
+
+		if (le64_to_cpu(beg->write_nr) <= le64_to_cpu(end->write_nr))
+			continue;
+
+		/* check if this term was fenced by others in a later term */
+		beg_was_fenced = false;
+		for (j = 0; j < SCOUTFS_QUORUM_BLOCKS; j++) {
+			if ((!qblk[j]) || (i == j))
+				continue;
+
+			fence = &qblk[j]->events[SCOUTFS_QUORUM_EVENT_FENCE];
+			if (le64_to_cpu(fence->term) > le64_to_cpu(beg->term)) {
+				beg_was_fenced = true;
+				break;
+			}
 		}
 
-		free(qblk);
-		qblk = NULL;
+		if (beg_was_fenced)
+			continue;
+
+		fprintf(stderr, "mount in quorum slot %u could still be running.\n"
+				"  begin event: write_nr %llu timestamp %llu.%08u\n"
+				"    end event: write_nr %llu timestamp %llu.%08u\n",
+			i, le64_to_cpu(beg->write_nr), le64_to_cpu(beg->ts.sec),
+			le32_to_cpu(beg->ts.nsec),
+			le64_to_cpu(end->write_nr), le64_to_cpu(end->ts.sec),
+			le32_to_cpu(end->ts.nsec));
+		ret = -EBUSY;
+		goto out;
 	}
 
 out:
+	/* free any allocated blocks */
+	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++)
+		if (qblk[i] != NULL)
+			free(qblk[i]);
+
 	return ret;
 }