Superblock checks for meta and data dev.

We check superblock magic, crc, flags. data device superblock is
checked but a little less thorough.  We check whether the device is
still mounted, since that would make checking invalid to begin with.
Quorum blocks are validated to have sane contents.

We add a global problem counter so we can trivially measure and
report whether any problem was found at all, instead of iterating
over all the problems and checking each individual count.

We pick the standard exit code values from `fsck` and mirror their
intentional behavior. This results in `fsck.scoutfs` can now be
trivially created by making it a wrapper around `scoutfs check`.

Signed-off-by: Auke Kok <auke.kok@versity.com>
Signed-off-by: Hunter Shaffer <hunter.shaffer@versity.com>

utils/src/check/block.c

@@ -317,8 +317,15 @@ int block_hdr_valid(struct block *blk, u64 blkno, int bf, u32 magic)
 
 	crc = crc_block(hdr, size);
 
-	if ((le32_to_cpu(hdr->crc) != crc) ||
-	    (le32_to_cpu(hdr->magic) != magic))
+	/*
+	 * a bad CRC is easy to repair, so we pass a different error code
+	 * back. Unless the other data is also wrong - then it's EINVAL
+	 * to signal that this isn't a valid block hdr at all.
+	 */
+	if (le32_to_cpu(hdr->crc) != crc)
+		ret = -EIO; /* keep checking other fields */
+
+	if (le32_to_cpu(hdr->magic) != magic)
 		ret = -EINVAL;
 
 	/*

utils/src/check/check.c

@@ -25,6 +25,7 @@
 #include "debug.h"
 #include "meta.h"
 #include "super.h"
+#include "problem.h"
 
 struct check_args {
 	char *meta_device;
@@ -74,8 +75,26 @@ static int do_check(struct check_args *args)
 	if (ret < 0)
 		goto out;
 
-	ret = check_supers() ?:
-	      check_meta_alloc();
+	/*
+	 * At some point we may convert this to a multi-pass system where we may
+	 * try and repair items, and, as long as repairs are made, we will rerun
+	 * the checks more times. We may need to start counting how many problems we
+	 * fix in the process of these loops, so that we don't stall on unrepairable
+	 * problems and are making actual repair progress. IOW - when we do a full
+	 * check loop without any problems fixed, we stop trying.
+	 */
+	ret = check_supers(data_fd) ?:
+	      check_super_in_use(meta_fd) ?:
+	      check_meta_alloc() ?:
+	      check_super_crc();
+
+	if (ret < 0)
+		goto out;
+
+	debug("problem count %lu", problems_count());
+	if (problems_count() > 0)
+		printf("Problems detected.\n");
+
 out:
 	/* and tear it all down */
 	block_shutdown();
@@ -134,6 +153,12 @@ static struct argp argp = {
 	"Check filesystem consistency"
 };
 
+/* Exit codes used by fsck-type programs */
+#define FSCK_EX_NONDESTRUCT	1	/* File system errors corrected */
+#define FSCK_EX_UNCORRECTED	4	/* File system errors left uncorrected */
+#define FSCK_EX_ERROR		8	/* Operational error */
+#define FSCK_EX_USAGE		16	/* Usage or syntax error */
+
 static int check_cmd(int argc, char **argv)
 {
 	struct check_args check_args = {NULL};
@@ -141,9 +166,16 @@ static int check_cmd(int argc, char **argv)
 
 	ret = argp_parse(&argp, argc, argv, 0, NULL, &check_args);
 	if (ret)
-		return ret;
+		exit(FSCK_EX_USAGE);
 
-	return do_check(&check_args);
+	ret = do_check(&check_args);
+	if (ret < 0)
+		ret = FSCK_EX_ERROR;
+
+	if (problems_count() > 0)
+		ret |= FSCK_EX_UNCORRECTED;
+
+	exit(ret);
 }
 
 static void __attribute__((constructor)) check_ctor(void)

utils/src/check/image.c

@@ -434,7 +434,7 @@ static int do_image(struct image_args *args)
 		}
 
 		ret = block_setup(meta_fd, 128 * 1024 * 1024, 32 * 1024 * 1024) ?:
-		      check_supers() ?:
+		      check_supers(-1) ?:
 		      get_ref_bits(&bm) ?:
 		      read_image(args, meta_fd, &bm);
 		block_shutdown();

utils/src/check/problem.c

@@ -3,16 +3,29 @@
 
 #include "problem.h"
 
-#if 0
 #define PROB_STR(pb) [pb] = #pb
-static char *prob_strs[] = {
+char *prob_strs[] = {
 	PROB_STR(PB_META_EXTENT_INVALID),
-	PROB_STR(PB_META_EXTENT_OVERLAPS_EXISTING),
+	PROB_STR(PB_META_REF_OVERLAPS_EXISTING),
+	PROB_STR(PB_META_FREE_OVERLAPS_EXISTING),
+	PROB_STR(PB_BTREE_BLOCK_BAD_LEVEL),
+	PROB_STR(PB_SB_HDR_CRC_INVALID),
+	PROB_STR(PB_SB_HDR_MAGIC_INVALID),
+	PROB_STR(PB_FS_IN_USE),
+	PROB_STR(PB_MOUNTED_CLIENTS_REF_BLKNO),
+	PROB_STR(PB_SB_BAD_FLAG),
+	PROB_STR(PB_SB_BAD_FMT_VERS),
+	PROB_STR(PB_QCONF_WRONG_VERSION),
+	PROB_STR(PB_QSLOT_BAD_FAM),
+	PROB_STR(PB_QSLOT_BAD_PORT),
+	PROB_STR(PB_QSLOT_NO_ADDR),
+	PROB_STR(PB_QSLOT_BAD_ADDR),
+	PROB_STR(PB_DATA_DEV_SB_INVALID),
 };
-#endif
 
 static struct problem_data {
 	uint64_t counts[PB__NR];
+	uint64_t count;
 } global_pdat;
 
 void problem_record(prob_t pb)
@@ -20,4 +33,12 @@ void problem_record(prob_t pb)
 	struct problem_data *pdat = &global_pdat;
 
 	pdat->counts[pb]++;
+	pdat->count++;
+}
+
+uint64_t problems_count(void)
+{
+	struct problem_data *pdat = &global_pdat;
+
+	return pdat->count;
 }

utils/src/check/problem.h

@@ -9,9 +9,23 @@ typedef enum {
 	PB_META_REF_OVERLAPS_EXISTING,
 	PB_META_FREE_OVERLAPS_EXISTING,
 	PB_BTREE_BLOCK_BAD_LEVEL,
+	PB_SB_HDR_CRC_INVALID,
+	PB_SB_HDR_MAGIC_INVALID,
+	PB_FS_IN_USE,
+	PB_MOUNTED_CLIENTS_REF_BLKNO,
+	PB_SB_BAD_FLAG,
+	PB_SB_BAD_FMT_VERS,
+	PB_QCONF_WRONG_VERSION,
+	PB_QSLOT_BAD_FAM,
+	PB_QSLOT_BAD_PORT,
+	PB_QSLOT_NO_ADDR,
+	PB_QSLOT_BAD_ADDR,
+	PB_DATA_DEV_SB_INVALID,
 	PB__NR,
 } prob_t;
 
+extern char *prob_strs[];
+
 #define problem(pb, fmt, ...)							\
 do {										\
 	debug("problem found: "#pb": %s: "fmt, sns_str(), __VA_ARGS__);	\
@@ -19,5 +33,6 @@ do {										\
 } while (0)
 
 void problem_record(prob_t pb);
+uint64_t problems_count(void);
 
 #endif

utils/src/check/super.c

@@ -2,13 +2,17 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <errno.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
 
 #include "sparse.h"
 #include "util.h"
 #include "format.h"
+#include "crc.h"
 
 #include "block.h"
 #include "super.h"
+#include "problem.h"
 
 /*
  * After we check the super blocks we provide a global buffer to track
@@ -18,14 +22,135 @@
  */
 struct scoutfs_super_block *global_super;
 
+/*
+ * Check superblock crc. We can't use global_super here since it's not the
+ * whole block itself, but only the struct scoutfs_super_block, so it needs
+ * to reload a copy here.
+ */
+int check_super_crc(void)
+{
+	struct scoutfs_super_block *super = NULL;
+	struct scoutfs_block_header *hdr;
+	struct block *blk = NULL;
+	u32 crc;
+	int ret;
+
+	ret = block_get(&blk, SCOUTFS_SUPER_BLKNO, BF_SM | BF_DIRTY);
+	if (ret < 0) {
+		fprintf(stderr, "error reading super block\n");
+		return ret;
+	}
+
+	super = block_buf(blk);
+	crc = crc_block((struct scoutfs_block_header *)super, block_size(blk));
+	hdr = &global_super->hdr;
+	debug("superblock crc 0x%04x calculated 0x%04x " "%s", le32_to_cpu(hdr->crc), crc, le32_to_cpu(hdr->crc) == crc ? "(match)" : "(mismatch)");
+
+	if (crc != le32_to_cpu(hdr->crc))
+		problem(PB_SB_HDR_CRC_INVALID, "crc 0x%04x calculated 0x%04x", le32_to_cpu(hdr->crc), crc);
+	block_put(&blk);
+
+	return 0;
+}
+
+/*
+ * Crude check for the unlikely cases where the fs appears to still be mounted.
+ */
+int check_super_in_use(int meta_fd)
+{
+	int ret = meta_super_in_use(meta_fd, global_super);
+	debug("meta_super_in_use ret %d", ret);
+
+	if (ret < 0)
+		problem(PB_FS_IN_USE, "File system appears in use. ret %d", ret);
+
+	debug("global_super->mounted_clients.ref.blkno 0x%08llx", global_super->mounted_clients.ref.blkno);
+	if (global_super->mounted_clients.ref.blkno != 0)
+		problem(PB_MOUNTED_CLIENTS_REF_BLKNO, "Mounted clients ref blkno 0x%08llx",
+			 global_super->mounted_clients.ref.blkno);
+
+	return ret;
+}
+
+/*
+ * quick glance data device superblock checks.
+ *
+ * -EIO for crc failures, all others -EINVAL
+ *
+ * caller must have run check_supers() first so that global_super is
+ * setup, so that we can cross-ref to it.
+ */
+static int check_data_super(int data_fd)
+{
+	struct scoutfs_super_block *super = NULL;
+	char *buf;
+	int ret = 0;
+	u32 crc;
+	ssize_t size = SCOUTFS_BLOCK_SM_SIZE;
+	off_t off = SCOUTFS_SUPER_BLKNO << SCOUTFS_BLOCK_SM_SHIFT;
+
+	buf = aligned_alloc(4096, size); /* XXX static alignment :/ */
+	if (!buf)
+		return -ENOMEM;
+
+	memset(buf, 0, size);
+
+	if (lseek(data_fd, off, SEEK_SET) != off)
+		return -errno;
+
+	if (read(data_fd, buf, size) < 0) {
+		ret = -errno;
+		goto out;
+	}
+
+	super = (struct scoutfs_super_block *)buf;
+
+	crc = crc_block((struct scoutfs_block_header *)buf, size);
+
+	debug("data fsid 0x%016llx", le64_to_cpu(super->hdr.fsid));
+	debug("data super magic 0x%04x", super->hdr.magic);
+	debug("data crc calc 0x%08x exp 0x%08x %s", crc, le32_to_cpu(super->hdr.crc),
+	      crc == le32_to_cpu(super->hdr.crc) ? "(match)" : "(mismatch)");
+	debug("data flags %llu fmt_vers %llu", le64_to_cpu(super->flags), le64_to_cpu(super->fmt_vers));
+
+	if (crc != le32_to_cpu(super->hdr.crc))
+		/* tis but a scratch */
+		ret = -EIO;
+
+	if (le64_to_cpu(super->hdr.fsid) != le64_to_cpu(global_super->hdr.fsid))
+		/* mismatched data bdev? not good */
+		ret = -EINVAL;
+
+	if (le32_to_cpu(super->hdr.magic) != SCOUTFS_BLOCK_MAGIC_SUPER)
+		/* fsid matched but not a superblock? yikes */
+		ret = -EINVAL;
+
+	if (le64_to_cpu(super->flags) != 0) /* !SCOUTFS_FLAG_IS_META_BDEV */
+		ret = -EINVAL;
+
+	if ((le64_to_cpu(super->fmt_vers) < SCOUTFS_FORMAT_VERSION_MIN) ||
+	    (le64_to_cpu(super->fmt_vers) > SCOUTFS_FORMAT_VERSION_MAX))
+		ret = -EINVAL;
+
+	if (ret != 0)
+		problem(PB_DATA_DEV_SB_INVALID, "data device is invalid or corrupt (%d)", ret);
+out:
+	free(buf);
+	return ret;
+}
+
 /*
  * After checking the supers we save a copy of it in a global buffer that's used by
  * other modules to track the current super.  It can be modified and written during commits.
  */
-int check_supers(void)
+int check_supers(int data_fd)
 {
 	struct scoutfs_super_block *super = NULL;
 	struct block *blk = NULL;
+	struct scoutfs_quorum_slot* slot = NULL;
+	struct in_addr in;
+	uint16_t family;
+	uint16_t port;
 	int ret;
 
 	sns_push("supers", 0, 0);
@@ -44,13 +169,75 @@ int check_supers(void)
 	}
 
 	ret = block_hdr_valid(blk, SCOUTFS_SUPER_BLKNO, BF_SM, SCOUTFS_BLOCK_MAGIC_SUPER);
-	if (ret < 0)
-		return ret;
 
 	super = block_buf(blk);
 
+	if (ret < 0) {
+		/* */
+		if (ret == -EINVAL) {
+			/* that's really bad */
+			fprintf(stderr, "superblock invalid magic\n");
+			goto out;
+		} else if (ret == -EIO)
+			/* just report/count a CRC error */
+			problem(PB_SB_HDR_MAGIC_INVALID, "superblock magic invalid: 0x%04x is not 0x%04x",
+				super->hdr.magic, SCOUTFS_BLOCK_MAGIC_SUPER);
+	}
+
 	memcpy(global_super, super, sizeof(struct scoutfs_super_block));
-	ret = 0;
+
+	debug("Superblock flag: %llu", global_super->flags);
+	if (le64_to_cpu(global_super->flags) != SCOUTFS_FLAG_IS_META_BDEV)
+		problem(PB_SB_BAD_FLAG, "Bad flag: %llu expecting: 1 or 0", global_super->flags);
+
+	debug("Superblock fmt_vers: %llu", le64_to_cpu(global_super->fmt_vers));
+	if ((le64_to_cpu(global_super->fmt_vers) < SCOUTFS_FORMAT_VERSION_MIN) ||
+	    (le64_to_cpu(global_super->fmt_vers) > SCOUTFS_FORMAT_VERSION_MAX))
+		problem(PB_SB_BAD_FMT_VERS, "Bad fmt_vers: %llu outside supported range (%d-%d)",
+			le64_to_cpu(global_super->fmt_vers), SCOUTFS_FORMAT_VERSION_MIN,
+			SCOUTFS_FORMAT_VERSION_MAX);
+
+	debug("Quorum Config Version: %llu", global_super->qconf.version);
+	if (le64_to_cpu(global_super->qconf.version) != 1)
+		problem(PB_QCONF_WRONG_VERSION, "Wrong Version: %llu (expected 1)", global_super->qconf.version);
+
+	for (int i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
+		slot = &global_super->qconf.slots[i];
+		family = le16_to_cpu(slot->addr.v4.family);
+		port = le16_to_cpu(slot->addr.v4.port);
+		in.s_addr = le32_to_cpu(slot->addr.v4.addr);
+
+		if (family == SCOUTFS_AF_NONE) {
+			debug("Quorum slot %u is empty", i);
+			continue;
+		}
+
+		debug("Quorum slot %u family: %u, port: %u, address: %s", i, family, port, inet_ntoa(in));
+		if (family != SCOUTFS_AF_IPV4)
+			problem(PB_QSLOT_BAD_FAM, "Quorum Slot %u doesn't have valid address", i);
+
+		if (port == 0)
+			problem(PB_QSLOT_BAD_PORT, "Quorum Slot %u has bad port", i);
+
+		if (!in.s_addr) {
+			problem(PB_QSLOT_NO_ADDR, "Quorum Slot %u has not been assigned ipv4 address", i);
+		} else if (!(in.s_addr & 0xff000000)) {
+			problem(PB_QSLOT_BAD_ADDR, "Quorum Slot %u has invalid ipv4 address", i);
+		} else if ((in.s_addr & 0xff) == 0xff) {
+			problem(PB_QSLOT_BAD_ADDR, "Quorum Slot %u has invalid ipv4 address", i);
+		}
+	}
+
+	debug("super magic 0x%04x", global_super->hdr.magic);
+	if (le32_to_cpu(global_super->hdr.magic) != SCOUTFS_BLOCK_MAGIC_SUPER)
+		problem(PB_SB_HDR_MAGIC_INVALID, "superblock magic invalid: 0x%04x is not 0x%04x",
+			global_super->hdr.magic, SCOUTFS_BLOCK_MAGIC_SUPER);
+
+	/* `scoutfs image` command doesn't open data_fd */
+	if (data_fd < 0)
+		ret = 0;
+	else
+		ret = check_data_super(data_fd);
 out:
 	block_put(&blk);
 

utils/src/check/super.h

@@ -3,7 +3,10 @@
 
 extern struct scoutfs_super_block *global_super;
 
-int check_supers(void);
+int check_super_crc();
+int check_supers(int data_fd);
+int super_commit(void);
+int check_super_in_use(int meta_fd);
 void super_shutdown(void);
 
 #endif