Check for fenced old leader in mounted test.

The old mounted check only considered begin/end quorum data, and
not whether the old leader that is now disconnected was fenced by
a new quorum leader.

Since this is the guaranteed case if the leader is disconnected
forcefully, this check must account for this case, so that quorum slots
can be modified if the node is permanently removed or replaced.

Signed-off-by: Auke Kok <auke.kok@versity.com>

kmod/src/lock.c

@@ -980,7 +980,7 @@ static bool lock_flags_invalid(int flags)
  */
 static int lock_key_range(struct super_block *sb, enum scoutfs_lock_mode mode, int flags,
 			  struct scoutfs_key *start, struct scoutfs_key *end,
-			  u64 ino, struct scoutfs_lock **ret_lock)
+			  struct scoutfs_lock **ret_lock)
 {
 	DECLARE_LOCK_INFO(sb, linfo);
 	struct scoutfs_lock *lock;
@@ -1028,8 +1028,6 @@ static int lock_key_range(struct super_block *sb, enum scoutfs_lock_mode mode, i
 		/* the fast path where we can use the granted mode */
 		if (lock_modes_match(lock->mode, mode)) {
 			lock_inc_count(lock->users, mode);
-			lock->last_user_pid[mode] = task_pid_nr(current);
-			lock->last_user_ino[mode] = ino;
 			*ret_lock = lock;
 			ret = 0;
 			break;
@@ -1110,7 +1108,7 @@ int scoutfs_lock_ino(struct super_block *sb, enum scoutfs_lock_mode mode, int fl
 	end.sk_zone = SCOUTFS_FS_ZONE;
 	end.ski_ino = cpu_to_le64(ino | SCOUTFS_LOCK_INODE_GROUP_MASK);
 
-	return lock_key_range(sb, mode, flags, &start, &end, ino, ret_lock);
+	return lock_key_range(sb, mode, flags, &start, &end, ret_lock);
 }
 
 /*
@@ -1240,7 +1238,7 @@ int scoutfs_lock_rename(struct super_block *sb, enum scoutfs_lock_mode mode, int
 		.sk_type = SCOUTFS_RENAME_TYPE,
 	};
 
-	return lock_key_range(sb, mode, flags, &key, &key, 0, lock);
+	return lock_key_range(sb, mode, flags, &key, &key, lock);
 }
 
 /*
@@ -1288,7 +1286,7 @@ int scoutfs_lock_inode_index(struct super_block *sb, enum scoutfs_lock_mode mode
 
 	scoutfs_lock_get_index_item_range(type, major, ino, &start, &end);
 
-	return lock_key_range(sb, mode, 0, &start, &end, ino, ret_lock);
+	return lock_key_range(sb, mode, 0, &start, &end, ret_lock);
 }
 
 /*
@@ -1315,7 +1313,7 @@ int scoutfs_lock_orphan(struct super_block *sb, enum scoutfs_lock_mode mode, int
 	end.sko_ino = cpu_to_le64(U64_MAX);
 	end.sk_type = SCOUTFS_ORPHAN_TYPE;
 
-	return lock_key_range(sb, mode, flags, &start, &end, ino, lock);
+	return lock_key_range(sb, mode, flags, &start, &end, lock);
 }
 
 int scoutfs_lock_xattr_totl(struct super_block *sb, enum scoutfs_lock_mode mode, int flags,
@@ -1326,7 +1324,7 @@ int scoutfs_lock_xattr_totl(struct super_block *sb, enum scoutfs_lock_mode mode,
 
 	scoutfs_totl_set_range(&start, &end);
 
-	return lock_key_range(sb, mode, flags, &start, &end, 0, lock);
+	return lock_key_range(sb, mode, flags, &start, &end, lock);
 }
 
 int scoutfs_lock_xattr_indx(struct super_block *sb, enum scoutfs_lock_mode mode, int flags,
@@ -1337,7 +1335,7 @@ int scoutfs_lock_xattr_indx(struct super_block *sb, enum scoutfs_lock_mode mode,
 
 	scoutfs_xattr_indx_get_range(&start, &end);
 
-	return lock_key_range(sb, mode, flags, &start, &end, 0, lock);
+	return lock_key_range(sb, mode, flags, &start, &end, lock);
 }
 
 int scoutfs_lock_quota(struct super_block *sb, enum scoutfs_lock_mode mode, int flags,
@@ -1348,7 +1346,7 @@ int scoutfs_lock_quota(struct super_block *sb, enum scoutfs_lock_mode mode, int
 
 	scoutfs_quota_get_lock_range(&start, &end);
 
-	return lock_key_range(sb, mode, flags, &start, &end, 0, lock);
+	return lock_key_range(sb, mode, flags, &start, &end, lock);
 }
 
 void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, enum scoutfs_lock_mode mode)
@@ -1465,7 +1463,7 @@ static void lock_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent)
 	struct scoutfs_lock *lock =
 		container_of(ent, struct scoutfs_lock, tseq_entry);
 
-	seq_printf(m, "start "SK_FMT" end "SK_FMT" refresh_gen %llu mode %d waiters: rd %u wr %u wo %u users: rd %u wr %u wo %u ino: rd %llu wr %llu wo %llu pid: rd %d wr %d wo %d\n",
+	seq_printf(m, "start "SK_FMT" end "SK_FMT" refresh_gen %llu mode %d waiters: rd %u wr %u wo %u users: rd %u wr %u wo %u\n",
 			   SK_ARG(&lock->start), SK_ARG(&lock->end),
 			   lock->refresh_gen, lock->mode,
 			   lock->waiters[SCOUTFS_LOCK_READ],
@@ -1473,13 +1471,7 @@ static void lock_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent)
 			   lock->waiters[SCOUTFS_LOCK_WRITE_ONLY],
 			   lock->users[SCOUTFS_LOCK_READ],
 			   lock->users[SCOUTFS_LOCK_WRITE],
-			   lock->users[SCOUTFS_LOCK_WRITE_ONLY],
-			   lock->last_user_ino[SCOUTFS_LOCK_READ],
-			   lock->last_user_ino[SCOUTFS_LOCK_WRITE],
-			   lock->last_user_ino[SCOUTFS_LOCK_WRITE_ONLY],
-			   lock->last_user_pid[SCOUTFS_LOCK_READ],
-			   lock->last_user_pid[SCOUTFS_LOCK_WRITE],
-			   lock->last_user_pid[SCOUTFS_LOCK_WRITE_ONLY]);
+			   lock->users[SCOUTFS_LOCK_WRITE_ONLY]);
 }
 
 /*

kmod/src/lock.h

@@ -42,8 +42,6 @@ struct scoutfs_lock {
 	enum scoutfs_lock_mode invalidating_mode;
 	unsigned int waiters[SCOUTFS_LOCK_NR_MODES];
 	unsigned int users[SCOUTFS_LOCK_NR_MODES];
-	pid_t last_user_pid[SCOUTFS_LOCK_NR_MODES];
-	u64 last_user_ino[SCOUTFS_LOCK_NR_MODES];
 
 	struct scoutfs_tseq_entry tseq_entry;
 

tests/golden/lock-pid-ino

@@ -1,6 +0,0 @@
-== set up file
-== exercise read, write, and write-only modes
-== verify FS-zone lock recorded read and write ino+pid
-== verify orphan-zone lock recorded write-only ino+pid
-== contend on a single inode with concurrent read and write loops
-== verify both rd and wr slots populated by concurrent contention

tests/sequence

@@ -32,7 +32,6 @@ totl-merge-read.sh
 quota-invalidate-race.sh
 totl-delta-inject.sh
 lock-refleak.sh
-lock-pid-ino.sh
 lock-shrink-consistency.sh
 lock-shrink-read-race.sh
 lock-pr-cw-conflict.sh

tests/tests/lock-pid-ino.sh

@@ -1,68 +0,0 @@
-#
-# verify debugfs client_locks reports per-mode last-user PID and inode.
-#
-
-t_require_commands stat touch awk rm
-
-FILE="$T_D0/file"
-
-echo "== set up file"
-touch "$FILE"
-INO=$(stat -c %i "$FILE")
-GROUP_START=$(( INO & ~1023 ))
-
-echo "== exercise read, write, and write-only modes"
-t_quiet stat "$FILE"
-echo data > "$FILE"
-rm -f "$FILE"
-
-echo "== verify FS-zone lock recorded read and write ino+pid"
-ERR=$(awk -v group="$GROUP_START" -v ino="$INO" '
-	$2 == "16." group ".0.0.0.0" {
-		if ($25 != ino || $32 <= 0)
-			print "read mode: ino=" $25 " pid=" $32 " want ino=" ino " pid>0"
-		if ($27 != ino || $34 <= 0)
-			print "write mode: ino=" $27 " pid=" $34 " want ino=" ino " pid>0"
-		found = 1
-	}
-	END { if (!found) print "no FS-zone client_locks line for group " group }
-' < "$(t_debugfs_path)/client_locks")
-[ -n "$ERR" ] && t_fail "$ERR"
-
-echo "== verify orphan-zone lock recorded write-only ino+pid"
-ERR=$(awk -v ino="$INO" '
-	$2 == "8.0.4.0.0.0" {
-		if ($29 != ino || $36 <= 0)
-			print "write-only mode: ino=" $29 " pid=" $36 " want ino=" ino " pid>0"
-		found = 1
-	}
-	END { if (!found) print "no orphan-zone client_locks line" }
-' < "$(t_debugfs_path)/client_locks")
-[ -n "$ERR" ] && t_fail "$ERR"
-
-echo "== contend on a single inode with concurrent read and write loops"
-FILE2="$T_D0/file2"
-touch "$FILE2"
-INO2=$(stat -c %i "$FILE2")
-GROUP2=$(( INO2 & ~1023 ))
-
-for i in $(seq 1 5); do t_quiet stat "$FILE2"; done &
-RPID=$!
-for i in $(seq 1 5); do echo $i > "$FILE2"; done &
-WPID=$!
-wait $RPID $WPID
-
-echo "== verify both rd and wr slots populated by concurrent contention"
-ERR=$(awk -v group="$GROUP2" -v ino="$INO2" '
-	$2 == "16." group ".0.0.0.0" {
-		if ($25 != ino || $32 <= 0)
-			print "concurrent read: ino=" $25 " pid=" $32 " want ino=" ino " pid>0"
-		if ($27 != ino || $34 <= 0)
-			print "concurrent write: ino=" $27 " pid=" $34 " want ino=" ino " pid>0"
-		found = 1
-	}
-	END { if (!found) print "no FS-zone client_locks line for group " group }
-' < "$(t_debugfs_path)/client_locks")
-[ -n "$ERR" ] && t_fail "$ERR"
-
-t_pass

utils/src/util.c

@@ -198,11 +198,13 @@ int write_block_sync(int fd, u32 magic, __le64 fsid, u64 seq, u64 blkno,
  */
 int meta_super_in_use(int meta_fd, struct scoutfs_super_block *meta_super)
 {
-	struct scoutfs_quorum_block *qblk = NULL;
+	struct scoutfs_quorum_block *qblk[SCOUTFS_QUORUM_BLOCKS] = {NULL,};
 	struct scoutfs_quorum_block_event *beg;
 	struct scoutfs_quorum_block_event *end;
+	struct scoutfs_quorum_block_event *fence;
+	bool beg_was_fenced;
 	int ret = 0;
-	int i;
+	int i, j;
 
 	if (meta_super->mounted_clients.ref.blkno != 0) {
 		fprintf(stderr, "meta superblock mounted clients btree is not empty.\n");
@@ -210,36 +212,61 @@ int meta_super_in_use(int meta_fd, struct scoutfs_super_block *meta_super)
 		goto out;
 	}
 
-	/* check for active quorum slots */
+	/* read all blocks */
 	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
 		if (!quorum_slot_present(meta_super, i))
 			continue;
 		ret = read_block(meta_fd, SCOUTFS_QUORUM_BLKNO + i, SCOUTFS_BLOCK_SM_SHIFT,
-				 (void **)&qblk);
+				 (void **)&qblk[i]);
 		if (ret < 0) {
 			fprintf(stderr, "error reading quorum block for slot %u\n", i);
 			goto out;
 		}
+	}
 
-		beg = &qblk->events[SCOUTFS_QUORUM_EVENT_BEGIN];
-		end = &qblk->events[SCOUTFS_QUORUM_EVENT_END];
+	/* check for active quorum slots */
+	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
+		if (!qblk[i])
+			continue;
 
-		if (le64_to_cpu(beg->write_nr) > le64_to_cpu(end->write_nr)) {
-			fprintf(stderr, "mount in quorum slot %u could still be running.\n"
-					"  begin event: write_nr %llu timestamp %llu.%08u\n"
-					"    end event: write_nr %llu timestamp %llu.%08u\n",
-				i, le64_to_cpu(beg->write_nr), le64_to_cpu(beg->ts.sec),
-				le32_to_cpu(beg->ts.nsec),
-				le64_to_cpu(end->write_nr), le64_to_cpu(end->ts.sec),
-				le32_to_cpu(end->ts.nsec));
-			ret = -EBUSY;
-			goto out;
+		beg = &qblk[i]->events[SCOUTFS_QUORUM_EVENT_BEGIN];
+		end = &qblk[i]->events[SCOUTFS_QUORUM_EVENT_END];
+
+		if (le64_to_cpu(beg->write_nr) <= le64_to_cpu(end->write_nr))
+			continue;
+
+		/* check if this term was fenced by others in a later term */
+		beg_was_fenced = false;
+		for (j = 0; j < SCOUTFS_QUORUM_BLOCKS; j++) {
+			if ((!qblk[j]) || (i == j))
+				continue;
+
+			fence = &qblk[j]->events[SCOUTFS_QUORUM_EVENT_FENCE];
+			if (le64_to_cpu(fence->term) > le64_to_cpu(beg->term)) {
+				beg_was_fenced = true;
+				break;
+			}
 		}
 
-		free(qblk);
-		qblk = NULL;
+		if (beg_was_fenced)
+			continue;
+
+		fprintf(stderr, "mount in quorum slot %u could still be running.\n"
+				"  begin event: write_nr %llu timestamp %llu.%08u\n"
+				"    end event: write_nr %llu timestamp %llu.%08u\n",
+			i, le64_to_cpu(beg->write_nr), le64_to_cpu(beg->ts.sec),
+			le32_to_cpu(beg->ts.nsec),
+			le64_to_cpu(end->write_nr), le64_to_cpu(end->ts.sec),
+			le32_to_cpu(end->ts.nsec));
+		ret = -EBUSY;
+		goto out;
 	}
 
 out:
+	/* free any allocated blocks */
+	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++)
+		if (qblk[i] != NULL)
+			free(qblk[i]);
+
 	return ret;
 }