diff --git a/utils/src/check/block.c b/utils/src/check/block.c index 13eeb867..08535a5a 100644 --- a/utils/src/check/block.c +++ b/utils/src/check/block.c @@ -317,8 +317,15 @@ int block_hdr_valid(struct block *blk, u64 blkno, int bf, u32 magic) crc = crc_block(hdr, size); - if ((le32_to_cpu(hdr->crc) != crc) || - (le32_to_cpu(hdr->magic) != magic)) + /* + * a bad CRC is easy to repair, so we pass a different error code + * back. Unless the other data is also wrong - then it's EINVAL + * to signal that this isn't a valid block hdr at all. + */ + if (le32_to_cpu(hdr->crc) != crc) + ret = -EIO; /* keep checking other fields */ + + if (le32_to_cpu(hdr->magic) != magic) ret = -EINVAL; /* diff --git a/utils/src/check/check.c b/utils/src/check/check.c index df07f403..0fa8a870 100644 --- a/utils/src/check/check.c +++ b/utils/src/check/check.c @@ -25,6 +25,7 @@ #include "debug.h" #include "meta.h" #include "super.h" +#include "problem.h" struct check_args { char *meta_device; @@ -74,8 +75,26 @@ static int do_check(struct check_args *args) if (ret < 0) goto out; - ret = check_supers() ?: - check_meta_alloc(); + /* + * At some point we may convert this to a multi-pass system where we may + * try and repair items, and, as long as repairs are made, we will rerun + * the checks more times. We may need to start counting how many problems we + * fix in the process of these loops, so that we don't stall on unrepairable + * problems and are making actual repair progress. IOW - when we do a full + * check loop without any problems fixed, we stop trying. + */ + ret = check_supers(data_fd) ?: + check_super_in_use(meta_fd) ?: + check_meta_alloc() ?: + check_super_crc(); + + if (ret < 0) + goto out; + + debug("problem count %lu", problems_count()); + if (problems_count() > 0) + printf("Problems detected.\n"); + out: /* and tear it all down */ block_shutdown(); @@ -134,6 +153,12 @@ static struct argp argp = { "Check filesystem consistency" }; +/* Exit codes used by fsck-type programs */ +#define FSCK_EX_NONDESTRUCT 1 /* File system errors corrected */ +#define FSCK_EX_UNCORRECTED 4 /* File system errors left uncorrected */ +#define FSCK_EX_ERROR 8 /* Operational error */ +#define FSCK_EX_USAGE 16 /* Usage or syntax error */ + static int check_cmd(int argc, char **argv) { struct check_args check_args = {NULL}; @@ -141,9 +166,16 @@ static int check_cmd(int argc, char **argv) ret = argp_parse(&argp, argc, argv, 0, NULL, &check_args); if (ret) - return ret; + exit(FSCK_EX_USAGE); - return do_check(&check_args); + ret = do_check(&check_args); + if (ret < 0) + ret = FSCK_EX_ERROR; + + if (problems_count() > 0) + ret |= FSCK_EX_UNCORRECTED; + + exit(ret); } static void __attribute__((constructor)) check_ctor(void) diff --git a/utils/src/check/image.c b/utils/src/check/image.c index 061076c6..0932ece6 100644 --- a/utils/src/check/image.c +++ b/utils/src/check/image.c @@ -434,7 +434,7 @@ static int do_image(struct image_args *args) } ret = block_setup(meta_fd, 128 * 1024 * 1024, 32 * 1024 * 1024) ?: - check_supers() ?: + check_supers(-1) ?: get_ref_bits(&bm) ?: read_image(args, meta_fd, &bm); block_shutdown(); diff --git a/utils/src/check/problem.c b/utils/src/check/problem.c index 2191726f..fd8d42a9 100644 --- a/utils/src/check/problem.c +++ b/utils/src/check/problem.c @@ -3,16 +3,29 @@ #include "problem.h" -#if 0 #define PROB_STR(pb) [pb] = #pb -static char *prob_strs[] = { +char *prob_strs[] = { PROB_STR(PB_META_EXTENT_INVALID), - PROB_STR(PB_META_EXTENT_OVERLAPS_EXISTING), + PROB_STR(PB_META_REF_OVERLAPS_EXISTING), + PROB_STR(PB_META_FREE_OVERLAPS_EXISTING), + PROB_STR(PB_BTREE_BLOCK_BAD_LEVEL), + PROB_STR(PB_SB_HDR_CRC_INVALID), + PROB_STR(PB_SB_HDR_MAGIC_INVALID), + PROB_STR(PB_FS_IN_USE), + PROB_STR(PB_MOUNTED_CLIENTS_REF_BLKNO), + PROB_STR(PB_SB_BAD_FLAG), + PROB_STR(PB_SB_BAD_FMT_VERS), + PROB_STR(PB_QCONF_WRONG_VERSION), + PROB_STR(PB_QSLOT_BAD_FAM), + PROB_STR(PB_QSLOT_BAD_PORT), + PROB_STR(PB_QSLOT_NO_ADDR), + PROB_STR(PB_QSLOT_BAD_ADDR), + PROB_STR(PB_DATA_DEV_SB_INVALID), }; -#endif static struct problem_data { uint64_t counts[PB__NR]; + uint64_t count; } global_pdat; void problem_record(prob_t pb) @@ -20,4 +33,12 @@ void problem_record(prob_t pb) struct problem_data *pdat = &global_pdat; pdat->counts[pb]++; + pdat->count++; +} + +uint64_t problems_count(void) +{ + struct problem_data *pdat = &global_pdat; + + return pdat->count; } diff --git a/utils/src/check/problem.h b/utils/src/check/problem.h index ce7b7fde..6ac49bb5 100644 --- a/utils/src/check/problem.h +++ b/utils/src/check/problem.h @@ -9,9 +9,23 @@ typedef enum { PB_META_REF_OVERLAPS_EXISTING, PB_META_FREE_OVERLAPS_EXISTING, PB_BTREE_BLOCK_BAD_LEVEL, + PB_SB_HDR_CRC_INVALID, + PB_SB_HDR_MAGIC_INVALID, + PB_FS_IN_USE, + PB_MOUNTED_CLIENTS_REF_BLKNO, + PB_SB_BAD_FLAG, + PB_SB_BAD_FMT_VERS, + PB_QCONF_WRONG_VERSION, + PB_QSLOT_BAD_FAM, + PB_QSLOT_BAD_PORT, + PB_QSLOT_NO_ADDR, + PB_QSLOT_BAD_ADDR, + PB_DATA_DEV_SB_INVALID, PB__NR, } prob_t; +extern char *prob_strs[]; + #define problem(pb, fmt, ...) \ do { \ debug("problem found: "#pb": %s: "fmt, sns_str(), __VA_ARGS__); \ @@ -19,5 +33,6 @@ do { \ } while (0) void problem_record(prob_t pb); +uint64_t problems_count(void); #endif diff --git a/utils/src/check/super.c b/utils/src/check/super.c index 40f815ea..e3c14fae 100644 --- a/utils/src/check/super.c +++ b/utils/src/check/super.c @@ -2,13 +2,17 @@ #include #include #include +#include +#include #include "sparse.h" #include "util.h" #include "format.h" +#include "crc.h" #include "block.h" #include "super.h" +#include "problem.h" /* * After we check the super blocks we provide a global buffer to track @@ -18,14 +22,135 @@ */ struct scoutfs_super_block *global_super; +/* + * Check superblock crc. We can't use global_super here since it's not the + * whole block itself, but only the struct scoutfs_super_block, so it needs + * to reload a copy here. + */ +int check_super_crc(void) +{ + struct scoutfs_super_block *super = NULL; + struct scoutfs_block_header *hdr; + struct block *blk = NULL; + u32 crc; + int ret; + + ret = block_get(&blk, SCOUTFS_SUPER_BLKNO, BF_SM | BF_DIRTY); + if (ret < 0) { + fprintf(stderr, "error reading super block\n"); + return ret; + } + + super = block_buf(blk); + crc = crc_block((struct scoutfs_block_header *)super, block_size(blk)); + hdr = &global_super->hdr; + debug("superblock crc 0x%04x calculated 0x%04x " "%s", le32_to_cpu(hdr->crc), crc, le32_to_cpu(hdr->crc) == crc ? "(match)" : "(mismatch)"); + + if (crc != le32_to_cpu(hdr->crc)) + problem(PB_SB_HDR_CRC_INVALID, "crc 0x%04x calculated 0x%04x", le32_to_cpu(hdr->crc), crc); + block_put(&blk); + + return 0; +} + +/* + * Crude check for the unlikely cases where the fs appears to still be mounted. + */ +int check_super_in_use(int meta_fd) +{ + int ret = meta_super_in_use(meta_fd, global_super); + debug("meta_super_in_use ret %d", ret); + + if (ret < 0) + problem(PB_FS_IN_USE, "File system appears in use. ret %d", ret); + + debug("global_super->mounted_clients.ref.blkno 0x%08llx", global_super->mounted_clients.ref.blkno); + if (global_super->mounted_clients.ref.blkno != 0) + problem(PB_MOUNTED_CLIENTS_REF_BLKNO, "Mounted clients ref blkno 0x%08llx", + global_super->mounted_clients.ref.blkno); + + return ret; +} + +/* + * quick glance data device superblock checks. + * + * -EIO for crc failures, all others -EINVAL + * + * caller must have run check_supers() first so that global_super is + * setup, so that we can cross-ref to it. + */ +static int check_data_super(int data_fd) +{ + struct scoutfs_super_block *super = NULL; + char *buf; + int ret = 0; + u32 crc; + ssize_t size = SCOUTFS_BLOCK_SM_SIZE; + off_t off = SCOUTFS_SUPER_BLKNO << SCOUTFS_BLOCK_SM_SHIFT; + + buf = aligned_alloc(4096, size); /* XXX static alignment :/ */ + if (!buf) + return -ENOMEM; + + memset(buf, 0, size); + + if (lseek(data_fd, off, SEEK_SET) != off) + return -errno; + + if (read(data_fd, buf, size) < 0) { + ret = -errno; + goto out; + } + + super = (struct scoutfs_super_block *)buf; + + crc = crc_block((struct scoutfs_block_header *)buf, size); + + debug("data fsid 0x%016llx", le64_to_cpu(super->hdr.fsid)); + debug("data super magic 0x%04x", super->hdr.magic); + debug("data crc calc 0x%08x exp 0x%08x %s", crc, le32_to_cpu(super->hdr.crc), + crc == le32_to_cpu(super->hdr.crc) ? "(match)" : "(mismatch)"); + debug("data flags %llu fmt_vers %llu", le64_to_cpu(super->flags), le64_to_cpu(super->fmt_vers)); + + if (crc != le32_to_cpu(super->hdr.crc)) + /* tis but a scratch */ + ret = -EIO; + + if (le64_to_cpu(super->hdr.fsid) != le64_to_cpu(global_super->hdr.fsid)) + /* mismatched data bdev? not good */ + ret = -EINVAL; + + if (le32_to_cpu(super->hdr.magic) != SCOUTFS_BLOCK_MAGIC_SUPER) + /* fsid matched but not a superblock? yikes */ + ret = -EINVAL; + + if (le64_to_cpu(super->flags) != 0) /* !SCOUTFS_FLAG_IS_META_BDEV */ + ret = -EINVAL; + + if ((le64_to_cpu(super->fmt_vers) < SCOUTFS_FORMAT_VERSION_MIN) || + (le64_to_cpu(super->fmt_vers) > SCOUTFS_FORMAT_VERSION_MAX)) + ret = -EINVAL; + + if (ret != 0) + problem(PB_DATA_DEV_SB_INVALID, "data device is invalid or corrupt (%d)", ret); +out: + free(buf); + return ret; +} + /* * After checking the supers we save a copy of it in a global buffer that's used by * other modules to track the current super. It can be modified and written during commits. */ -int check_supers(void) +int check_supers(int data_fd) { struct scoutfs_super_block *super = NULL; struct block *blk = NULL; + struct scoutfs_quorum_slot* slot = NULL; + struct in_addr in; + uint16_t family; + uint16_t port; int ret; sns_push("supers", 0, 0); @@ -44,13 +169,75 @@ int check_supers(void) } ret = block_hdr_valid(blk, SCOUTFS_SUPER_BLKNO, BF_SM, SCOUTFS_BLOCK_MAGIC_SUPER); - if (ret < 0) - return ret; super = block_buf(blk); + if (ret < 0) { + /* */ + if (ret == -EINVAL) { + /* that's really bad */ + fprintf(stderr, "superblock invalid magic\n"); + goto out; + } else if (ret == -EIO) + /* just report/count a CRC error */ + problem(PB_SB_HDR_MAGIC_INVALID, "superblock magic invalid: 0x%04x is not 0x%04x", + super->hdr.magic, SCOUTFS_BLOCK_MAGIC_SUPER); + } + memcpy(global_super, super, sizeof(struct scoutfs_super_block)); - ret = 0; + + debug("Superblock flag: %llu", global_super->flags); + if (le64_to_cpu(global_super->flags) != SCOUTFS_FLAG_IS_META_BDEV) + problem(PB_SB_BAD_FLAG, "Bad flag: %llu expecting: 1 or 0", global_super->flags); + + debug("Superblock fmt_vers: %llu", le64_to_cpu(global_super->fmt_vers)); + if ((le64_to_cpu(global_super->fmt_vers) < SCOUTFS_FORMAT_VERSION_MIN) || + (le64_to_cpu(global_super->fmt_vers) > SCOUTFS_FORMAT_VERSION_MAX)) + problem(PB_SB_BAD_FMT_VERS, "Bad fmt_vers: %llu outside supported range (%d-%d)", + le64_to_cpu(global_super->fmt_vers), SCOUTFS_FORMAT_VERSION_MIN, + SCOUTFS_FORMAT_VERSION_MAX); + + debug("Quorum Config Version: %llu", global_super->qconf.version); + if (le64_to_cpu(global_super->qconf.version) != 1) + problem(PB_QCONF_WRONG_VERSION, "Wrong Version: %llu (expected 1)", global_super->qconf.version); + + for (int i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { + slot = &global_super->qconf.slots[i]; + family = le16_to_cpu(slot->addr.v4.family); + port = le16_to_cpu(slot->addr.v4.port); + in.s_addr = le32_to_cpu(slot->addr.v4.addr); + + if (family == SCOUTFS_AF_NONE) { + debug("Quorum slot %u is empty", i); + continue; + } + + debug("Quorum slot %u family: %u, port: %u, address: %s", i, family, port, inet_ntoa(in)); + if (family != SCOUTFS_AF_IPV4) + problem(PB_QSLOT_BAD_FAM, "Quorum Slot %u doesn't have valid address", i); + + if (port == 0) + problem(PB_QSLOT_BAD_PORT, "Quorum Slot %u has bad port", i); + + if (!in.s_addr) { + problem(PB_QSLOT_NO_ADDR, "Quorum Slot %u has not been assigned ipv4 address", i); + } else if (!(in.s_addr & 0xff000000)) { + problem(PB_QSLOT_BAD_ADDR, "Quorum Slot %u has invalid ipv4 address", i); + } else if ((in.s_addr & 0xff) == 0xff) { + problem(PB_QSLOT_BAD_ADDR, "Quorum Slot %u has invalid ipv4 address", i); + } + } + + debug("super magic 0x%04x", global_super->hdr.magic); + if (le32_to_cpu(global_super->hdr.magic) != SCOUTFS_BLOCK_MAGIC_SUPER) + problem(PB_SB_HDR_MAGIC_INVALID, "superblock magic invalid: 0x%04x is not 0x%04x", + global_super->hdr.magic, SCOUTFS_BLOCK_MAGIC_SUPER); + + /* `scoutfs image` command doesn't open data_fd */ + if (data_fd < 0) + ret = 0; + else + ret = check_data_super(data_fd); out: block_put(&blk); diff --git a/utils/src/check/super.h b/utils/src/check/super.h index 7c75ad2d..f14417ba 100644 --- a/utils/src/check/super.h +++ b/utils/src/check/super.h @@ -3,7 +3,10 @@ extern struct scoutfs_super_block *global_super; -int check_supers(void); +int check_super_crc(); +int check_supers(int data_fd); +int super_commit(void); +int check_super_in_use(int meta_fd); void super_shutdown(void); #endif