Check for fenced old leader in mounted test.

The old mounted check only considered begin/end quorum data, and
not whether the old leader that is now disconnected was fenced by
a new quorum leader.

Since this is the guaranteed case if the leader is disconnected
forcefully, this check must account for this case, so that quorum slots
can be modified if the node is permanently removed or replaced.

Signed-off-by: Auke Kok <auke.kok@versity.com>

kmod/src/data.c

@@ -79,8 +79,10 @@ static void item_from_extent(struct scoutfs_key *key,
 		.skdx_end = cpu_to_le64(start + len - 1),
 		.skdx_len = cpu_to_le64(len),
 	};
-	dv->blkno = cpu_to_le64(map);
-	dv->flags = flags;
+	*dv = (struct scoutfs_data_extent_val) {
+		.blkno = cpu_to_le64(map),
+		.flags = flags,
+	};
 }
 
 static void ext_from_item(struct scoutfs_extent *ext,

kmod/src/dir.c

@@ -587,10 +587,12 @@ static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 hash,
 	}
 
 	/* initialize the dent */
-	dent->ino = cpu_to_le64(ino);
-	dent->hash = cpu_to_le64(hash);
-	dent->pos = cpu_to_le64(pos);
-	dent->type = mode_to_type(mode);
+	*dent = (struct scoutfs_dirent) {
+		.ino = cpu_to_le64(ino),
+		.hash = cpu_to_le64(hash),
+		.pos = cpu_to_le64(pos),
+		.type = mode_to_type(mode),
+	};
 	memcpy(dent->name, name, name_len);
 
 	init_dirent_key(&ent_key, SCOUTFS_DIRENT_TYPE, dir_ino, hash, pos);

kmod/src/forest.c

@@ -793,7 +793,7 @@ out:
 	if (ret)
 		scoutfs_forest_destroy(sb);
 
-	return 0;
+	return ret;
 }
 
 void scoutfs_forest_start(struct super_block *sb)

kmod/src/ioctl.c

@@ -415,8 +415,6 @@ static long scoutfs_ioc_data_wait_err(struct file *file, unsigned long arg)
 		return 0;
 	if ((args.op & SCOUTFS_IOC_DWO_UNKNOWN) || !IS_ERR_VALUE(args.err))
 		return -EINVAL;
-	if ((args.op & SCOUTFS_IOC_DWO_UNKNOWN) || !IS_ERR_VALUE(args.err))
-		return -EINVAL;
 
 	trace_scoutfs_ioc_data_wait_err(sb, &args);
 

kmod/src/net.c

@@ -336,7 +336,7 @@ static inline u8 net_err_from_host(struct super_block *sb, int error)
 				     error);
 		}
 
-		return -EINVAL;
+		return SCOUTFS_NET_ERR_EINVAL;
 	}
 
 	return net_errs[ind];

kmod/src/quorum.c

@@ -1195,8 +1195,8 @@ static struct attribute *quorum_attrs[] = {
 
 static inline bool valid_ipv4_unicast(__be32 addr)
 {
-	return !(ipv4_is_multicast(addr) && ipv4_is_lbcast(addr) &&
-		 ipv4_is_zeronet(addr) && ipv4_is_local_multicast(addr));
+	return !(ipv4_is_multicast(addr) || ipv4_is_lbcast(addr) ||
+		 ipv4_is_zeronet(addr) || ipv4_is_local_multicast(addr));
 }
 
 static inline bool valid_ipv4_port(__be16 port)

tests/tests/simple-inode-index.sh

@@ -92,13 +92,13 @@ test "$before" -lt "$after" || \
 # didn't skip past deleted dirty items
 #
 echo "== make sure dirtying doesn't livelock walk"
-dd if=/dev/urandom of="$DIR/dirtying" bs=4K count=1 >> $seqres.full 2>&1
+dd if=/dev/urandom of="$DIR/dirtying" bs=4K count=1 >> "$T_TMPDIR/seqres.full" 2>&1
 nr=1
 while [ "$nr" -lt 100 ]; do
-	echo "dirty/walk attempt $nr" >> $seqres.full
+	echo "dirty/walk attempt $nr" >> "$T_TMPDIR/seqres.full"
 	sync
 	dd if=/dev/urandom of="$DIR/dirtying" bs=4K count=1 conv=notrunc \
-		>> $seqres.full 2>&1
+		>> "$T_TMPDIR/seqres.full" 2>&1
 	scoutfs walk-inodes data_seq 0 -1 $DIR/dirtying >& /dev/null 
 	((nr++))
 done

tests/tests/simple-staging.sh

@@ -12,12 +12,12 @@ create_file() {
 
 	if [ "$blocks" != 0 ]; then
 		dd if=/dev/urandom bs=4096 count=$blocks of="$file" \
-			>> $seqres.full 2>&1
+			>> "$T_TMPDIR/seqres.full" 2>&1
 	fi
 
 	if [ "$remainder" != 0 ]; then
 		dd if=/dev/urandom bs="$remainder" count=1 of="$file" \
-			conv=notrunc oflag=append >> $seqres.full 2>&1
+			conv=notrunc oflag=append >> "$T_TMPDIR/seqres.full" 2>&1
 	fi
 }
 
@@ -78,7 +78,7 @@ create_file "$FILE" $((4096 * 1024))
 cp "$FILE"  "$T_TMP"
 nr=1
 while [ "$nr" -lt 10 ]; do
-	echo "attempt $nr" >> $seqres.full 2>&1
+	echo "attempt $nr" >> "$T_TMPDIR/$seqres.full" 2>&1
 	release_vers "$FILE" stat 0 4096K
 	sync
 	echo 3 > /proc/sys/vm/drop_caches

utils/src/util.c

@@ -198,11 +198,13 @@ int write_block_sync(int fd, u32 magic, __le64 fsid, u64 seq, u64 blkno,
  */
 int meta_super_in_use(int meta_fd, struct scoutfs_super_block *meta_super)
 {
-	struct scoutfs_quorum_block *qblk = NULL;
+	struct scoutfs_quorum_block *qblk[SCOUTFS_QUORUM_BLOCKS] = {NULL,};
 	struct scoutfs_quorum_block_event *beg;
 	struct scoutfs_quorum_block_event *end;
+	struct scoutfs_quorum_block_event *fence;
+	bool beg_was_fenced;
 	int ret = 0;
-	int i;
+	int i, j;
 
 	if (meta_super->mounted_clients.ref.blkno != 0) {
 		fprintf(stderr, "meta superblock mounted clients btree is not empty.\n");
@@ -210,36 +212,61 @@ int meta_super_in_use(int meta_fd, struct scoutfs_super_block *meta_super)
 		goto out;
 	}
 
-	/* check for active quorum slots */
+	/* read all blocks */
 	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
 		if (!quorum_slot_present(meta_super, i))
 			continue;
 		ret = read_block(meta_fd, SCOUTFS_QUORUM_BLKNO + i, SCOUTFS_BLOCK_SM_SHIFT,
-				 (void **)&qblk);
+				 (void **)&qblk[i]);
 		if (ret < 0) {
 			fprintf(stderr, "error reading quorum block for slot %u\n", i);
 			goto out;
 		}
+	}
 
-		beg = &qblk->events[SCOUTFS_QUORUM_EVENT_BEGIN];
-		end = &qblk->events[SCOUTFS_QUORUM_EVENT_END];
+	/* check for active quorum slots */
+	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
+		if (!qblk[i])
+			continue;
 
-		if (le64_to_cpu(beg->write_nr) > le64_to_cpu(end->write_nr)) {
-			fprintf(stderr, "mount in quorum slot %u could still be running.\n"
-					"  begin event: write_nr %llu timestamp %llu.%08u\n"
-					"    end event: write_nr %llu timestamp %llu.%08u\n",
-				i, le64_to_cpu(beg->write_nr), le64_to_cpu(beg->ts.sec),
-				le32_to_cpu(beg->ts.nsec),
-				le64_to_cpu(end->write_nr), le64_to_cpu(end->ts.sec),
-				le32_to_cpu(end->ts.nsec));
-			ret = -EBUSY;
-			goto out;
+		beg = &qblk[i]->events[SCOUTFS_QUORUM_EVENT_BEGIN];
+		end = &qblk[i]->events[SCOUTFS_QUORUM_EVENT_END];
+
+		if (le64_to_cpu(beg->write_nr) <= le64_to_cpu(end->write_nr))
+			continue;
+
+		/* check if this term was fenced by others in a later term */
+		beg_was_fenced = false;
+		for (j = 0; j < SCOUTFS_QUORUM_BLOCKS; j++) {
+			if ((!qblk[j]) || (i == j))
+				continue;
+
+			fence = &qblk[j]->events[SCOUTFS_QUORUM_EVENT_FENCE];
+			if (le64_to_cpu(fence->term) > le64_to_cpu(beg->term)) {
+				beg_was_fenced = true;
+				break;
+			}
 		}
 
-		free(qblk);
-		qblk = NULL;
+		if (beg_was_fenced)
+			continue;
+
+		fprintf(stderr, "mount in quorum slot %u could still be running.\n"
+				"  begin event: write_nr %llu timestamp %llu.%08u\n"
+				"    end event: write_nr %llu timestamp %llu.%08u\n",
+			i, le64_to_cpu(beg->write_nr), le64_to_cpu(beg->ts.sec),
+			le32_to_cpu(beg->ts.nsec),
+			le64_to_cpu(end->write_nr), le64_to_cpu(end->ts.sec),
+			le32_to_cpu(end->ts.nsec));
+		ret = -EBUSY;
+		goto out;
 	}
 
 out:
+	/* free any allocated blocks */
+	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++)
+		if (qblk[i] != NULL)
+			free(qblk[i]);
+
 	return ret;
 }