From 87fcad5428bedd48192716645556d9ee0738c783 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 29 Jan 2021 15:43:03 -0800 Subject: [PATCH] Update scoutfs mkfs and print for quorum slots Signed-off-by: Zach Brown --- utils/src/mkfs.c | 94 ++++++++++++++++++++++++++++++++++++++--------- utils/src/parse.c | 65 ++++++++++++++++++++++++++++++++ utils/src/parse.h | 3 ++ utils/src/print.c | 81 +++++++++++++++++++++------------------- 4 files changed, 188 insertions(+), 55 deletions(-) diff --git a/utils/src/mkfs.c b/utils/src/mkfs.c index f5306dd1..b151c72a 100644 --- a/utils/src/mkfs.c +++ b/utils/src/mkfs.c @@ -101,12 +101,13 @@ static int write_alloc_root(struct scoutfs_super_block *super, int fd, } struct mkfs_args { - unsigned long long quorum_count; char *meta_device; char *data_device; unsigned long long max_meta_size; unsigned long long max_data_size; bool force; + int nr_slots; + struct scoutfs_quorum_slot slots[SCOUTFS_QUORUM_MAX_SLOTS]; }; /* @@ -124,12 +125,14 @@ static int do_mkfs(struct mkfs_args *args) struct scoutfs_inode inode; struct scoutfs_alloc_list_block *lblk; struct scoutfs_btree_block *bt = NULL; + struct scoutfs_block_header *hdr; struct scoutfs_key key; struct timeval tv; int meta_fd = -1; int data_fd = -1; char uuid_str[37]; void *zeros = NULL; + char *indent; u64 blkno; u64 meta_size; u64 data_size; @@ -191,10 +194,7 @@ static int do_mkfs(struct mkfs_args *args) if (ret) goto out; - /* metadata blocks start after the quorum blocks */ - next_meta = (SCOUTFS_QUORUM_BLKNO + SCOUTFS_QUORUM_BLOCKS) >> - SCOUTFS_BLOCK_SM_LG_SHIFT; - /* rest of meta dev is available for metadata blocks */ + next_meta = SCOUTFS_META_DEV_START_BLKNO; last_meta = (meta_size >> SCOUTFS_BLOCK_LG_SHIFT) - 1; /* Data blocks go on the data dev */ first_data = SCOUTFS_DATA_DEV_START_BLKNO; @@ -215,7 +215,10 @@ static int do_mkfs(struct mkfs_args *args) super->total_data_blocks = cpu_to_le64(last_data - first_data + 1); super->first_data_blkno = cpu_to_le64(first_data); super->last_data_blkno = cpu_to_le64(last_data); - super->quorum_count = args->quorum_count; + + assert(sizeof(args->slots) == + member_sizeof(struct scoutfs_super_block, qconf.slots)); + memcpy(super->qconf.slots, args->slots, sizeof(args->slots)); /* fs root starts with root inode and its index items */ blkno = next_meta++; @@ -309,9 +312,11 @@ static int do_mkfs(struct mkfs_args *args) } /* zero out quorum blocks */ + hdr = zeros; + hdr->magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_QUORUM); for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { - ret = write_raw_block(meta_fd, SCOUTFS_QUORUM_BLKNO + i, - SCOUTFS_BLOCK_SM_SHIFT, zeros); + ret = write_block(meta_fd, SCOUTFS_QUORUM_BLKNO + i, + SCOUTFS_BLOCK_SM_SHIFT, super, hdr); if (ret < 0) { fprintf(stderr, "error zeroing quorum block: %s (%d)\n", strerror(-errno), -errno); @@ -356,7 +361,7 @@ static int do_mkfs(struct mkfs_args *args) " uuid: %s\n" " 64KB metadata blocks: "SIZE_FMT"\n" " 4KB data blocks: "SIZE_FMT"\n" - " quorum count: %u\n", + " quorum slots: ", args->meta_device, args->data_device, le64_to_cpu(super->hdr.fsid), @@ -365,8 +370,22 @@ static int do_mkfs(struct mkfs_args *args) SIZE_ARGS(le64_to_cpu(super->total_meta_blocks), SCOUTFS_BLOCK_LG_SIZE), SIZE_ARGS(le64_to_cpu(super->total_data_blocks), - SCOUTFS_BLOCK_SM_SIZE), - super->quorum_count); + SCOUTFS_BLOCK_SM_SIZE)); + + indent = ""; + for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { + struct scoutfs_quorum_slot *sl = &super->qconf.slots[i]; + struct in_addr in; + + if (sl->addr.addr == 0) + continue; + + in.s_addr = htonl(le32_to_cpu(sl->addr.addr)); + printf("%s%u: %s:%u", indent, + i, inet_ntoa(in), le16_to_cpu(sl->addr.port)); + indent = "\n "; + } + printf("\n"); ret = 0; out: @@ -383,16 +402,55 @@ out: return ret; } +static bool valid_quorum_slots(struct scoutfs_quorum_slot *slots) +{ + struct in_addr in; + bool valid = true; + char *addr; + int i; + int j; + + for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { + if (slots[i].addr.addr == 0) + continue; + + for (j = i + 1; j < SCOUTFS_QUORUM_MAX_SLOTS; j++) { + if (slots[j].addr.addr == 0) + continue; + + if (slots[i].addr.addr == slots[j].addr.addr && + slots[i].addr.port == slots[j].addr.port) { + + in.s_addr = + htonl(le32_to_cpu(slots[i].addr.addr)); + addr = inet_ntoa(in); + fprintf(stderr, "quorum slot nr %u and %u have the same address %s:%u\n", + i, j, addr, + le16_to_cpu(slots[i].addr.port)); + valid = false; + } + } + } + + return valid; +} + static int parse_opt(int key, char *arg, struct argp_state *state) { struct mkfs_args *args = state->input; + struct scoutfs_quorum_slot slot; int ret; switch (key) { case 'Q': - ret = parse_u64(arg, &args->quorum_count); - if (ret) + ret = parse_quorum_slot(&slot, arg); + if (ret < 0) return ret; + if (args->slots[ret].addr.addr != 0) + argp_error(state, "Quorum slot %u already specified before slot '%s'\n", + ret, arg); + args->slots[ret] = slot; + args->nr_slots++; break; case 'f': args->force = true; @@ -432,12 +490,14 @@ static int parse_opt(int key, char *arg, struct argp_state *state) argp_error(state, "more than two arguments given"); break; case ARGP_KEY_FINI: - if (!args->quorum_count) - argp_error(state, "must provide nonzero quorum count with --quorum-count|-Q option"); + if (!args->nr_slots) + argp_error(state, "must specify at least one quorum slot with --quorum-count|-Q"); if (!args->meta_device) argp_error(state, "no metadata device argument given"); if (!args->data_device) argp_error(state, "no data device argument given"); + if (!valid_quorum_slots(args->slots)) + argp_error(state, "invalid quorum slot configuration"); break; default: break; @@ -447,7 +507,7 @@ static int parse_opt(int key, char *arg, struct argp_state *state) } static struct argp_option options[] = { - { "quorum-count", 'Q', "NUM", 0, "Number of voters required to use the filesystem [Required]"}, + { "quorum-slot", 'Q', "NR,ADDR,PORT", 0, "Specify quorum slot addresses [Required]"}, { "force", 'f', NULL, 0, "Overwrite existing data on block devices"}, { "max-meta-size", 'm', "SIZE", 0, "Use a size less than the base metadata device size (bytes or KMGTP units)"}, { "max-data-size", 'd', "SIZE", 0, "Use a size less than the base data device size (bytes or KMGTP units)"}, @@ -463,7 +523,7 @@ static struct argp argp = { static int mkfs_cmd(int argc, char *argv[]) { - struct mkfs_args mkfs_args = {0}; + struct mkfs_args mkfs_args = {NULL,}; int ret; ret = argp_parse(&argp, argc, argv, 0, NULL, &mkfs_args); diff --git a/utils/src/parse.c b/utils/src/parse.c index 761129f9..49d566b8 100644 --- a/utils/src/parse.c +++ b/utils/src/parse.c @@ -3,6 +3,9 @@ #include #include #include +#include +#include +#include #include "sparse.h" #include "util.h" @@ -152,3 +155,65 @@ int parse_timespec(char *str, struct timespec *ts) return 0; } + +/* + * Parse a quorum slot specification string "NR,ADDR,PORT" into its + * component parts. We use sscanf to both parse the leading NR and + * trailing PORT integers, and to pull out the inner ADDR string which + * is then parsed to make sure that it's a valid unicast ipv4 address. + * We require that all components be specified, and sccanf will check + * this by the number of matches it returns. + */ +int parse_quorum_slot(struct scoutfs_quorum_slot *slot, char *arg) +{ +#define ADDR_CHARS 45 /* max ipv6 */ + char addr[ADDR_CHARS + 1] = {'\0',}; + struct in_addr in; + int port; + int parsed; + int nr; + int ret; + + /* leading and trailing ints, an inner sized string without ,, all separated by , */ + ret = sscanf(arg, "%u,%"__stringify(ADDR_CHARS)"[^,],%u%n", + &nr, addr, &port, &parsed); + if (ret == EOF) { + printf("error parsing quorum slot '%s': %s\n", + arg, strerror(errno)); + return -EINVAL; + } + + if (parsed != strlen(arg)) { + printf("extra unparsed trailing characters in quorum slot '%s'\n", + arg); + return -EINVAL; + } + + if (ret != 3) { + printf("failed to parse all three NR,ADDR,PORT tokens in quorum slot '%s'\n", arg); + return -EINVAL; + } + + if (nr < 0 || nr >= SCOUTFS_QUORUM_MAX_SLOTS) { + printf("invalid nr '%d' in quorum slot '%s', must be between 0 and %u\n", + nr, arg, SCOUTFS_QUORUM_MAX_SLOTS - 1); + return -EINVAL; + } + + if (port <= 0 || port > USHRT_MAX) { + printf("invalid ipv4 port '%u' in quorum slot '%s', must be between 1 and %u\n", + port, arg, USHRT_MAX); + return -EINVAL; + } + + if (inet_aton(addr, &in) == 0 || htonl(in.s_addr) == 0 || + htonl(in.s_addr) == UINT_MAX) { + printf("invalid ipv4 address '%s' in quorum slot '%s'\n", + addr, arg); + return -EINVAL; + } + + slot->addr.addr = cpu_to_le32(htonl(in.s_addr)); + slot->addr.port = cpu_to_le16(port); + return nr; +} diff --git a/utils/src/parse.h b/utils/src/parse.h index b7ed51f4..f83afeec 100644 --- a/utils/src/parse.h +++ b/utils/src/parse.h @@ -4,11 +4,14 @@ #include #include +struct scoutfs_quorum_slot; + int parse_human(char* str, u64 *val_ret); int parse_u64(char *str, u64 *val_ret); int parse_s64(char *str, s64 *val_ret); int parse_u32(char *str, u32 *val_ret); int parse_timespec(char *str, struct timespec *ts); +int parse_quorum_slot(struct scoutfs_quorum_slot *slot, char *arg); static inline char* strdup_or_error(const struct argp_state *state, char *str) { diff --git a/utils/src/print.c b/utils/src/print.c index c2ba30a2..d6df9d29 100644 --- a/utils/src/print.c +++ b/utils/src/print.c @@ -796,14 +796,25 @@ static char *alloc_addr_str(struct scoutfs_inet_addr *ia) return str; } +#define OFF_NAME(x) \ + { offsetof(struct scoutfs_quorum_block, x), __stringify_1(x) } + static int print_quorum_blocks(int fd, struct scoutfs_super_block *super) { + struct print_events { + size_t offset; + char *name; + } events[] = { + OFF_NAME(write), OFF_NAME(update_term), OFF_NAME(set_leader), + OFF_NAME(clear_leader), OFF_NAME(fenced), + }; struct scoutfs_quorum_block *blk = NULL; + struct scoutfs_quorum_block_event *ev; char *log_addr = NULL; u64 blkno; int ret; int i; - int j; + int e; for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { blkno = SCOUTFS_QUORUM_BLKNO + i; @@ -812,31 +823,21 @@ static int print_quorum_blocks(int fd, struct scoutfs_super_block *super) if (ret) goto out; - if (blk->voter_rid != 0) { - printf("quorum block blkno %llu\n" - " fsid %llx blkno %llu crc 0x%08x\n" - " term %llu write_nr %llu voter_rid %016llx " - "vote_for_rid %016llx\n" - " log_nr %u\n", - blkno, le64_to_cpu(blk->fsid), - le64_to_cpu(blk->blkno), le32_to_cpu(blk->crc), - le64_to_cpu(blk->term), - le64_to_cpu(blk->write_nr), - le64_to_cpu(blk->voter_rid), - le64_to_cpu(blk->vote_for_rid), - blk->log_nr); - for (j = 0; j < blk->log_nr; j++) { - free(log_addr); - log_addr = alloc_addr_str(&blk->log[j].addr); - if (!log_addr) { - ret = -ENOMEM; - goto out; - } - printf(" [%u]: term %llu rid %llu addr %s\n", - j, le64_to_cpu(blk->log[j].term), - le64_to_cpu(blk->log[j].rid), - log_addr); - } + printf("quorum blkno %llu (slot %llu)\n", + blkno, blkno - SCOUTFS_QUORUM_BLKNO); + print_block_header(&blk->hdr, SCOUTFS_BLOCK_SM_SIZE); + printf(" term %llu random_write_mark 0x%llx flags 0x%llx\n", + le64_to_cpu(blk->term), + le64_to_cpu(blk->random_write_mark), + le64_to_cpu(blk->flags)); + + for (e = 0; e < array_size(events); e++) { + ev = (void *)blk + events[e].offset; + + printf(" %12s: rid %016llx ts %llu.%08u\n", + events[e].name, le64_to_cpu(ev->rid), + le64_to_cpu(ev->ts.sec), + le32_to_cpu(ev->ts.nsec)); } } @@ -850,7 +851,8 @@ out: static void print_super_block(struct scoutfs_super_block *super, u64 blkno) { char uuid_str[37]; - char *server_addr; + char *addr; + int i; uuid_unparse(super->uuid, uuid_str); @@ -864,16 +866,11 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno) le64_to_cpu(super->version), uuid_str); printf(" flags: 0x%016llx\n", le64_to_cpu(super->flags)); - server_addr = alloc_addr_str(&super->server_addr); - if (!server_addr) - return; - /* XXX these are all in a crazy order */ printf(" next_ino %llu next_trans_seq %llu\n" " total_meta_blocks %llu first_meta_blkno %llu last_meta_blkno %llu\n" " total_data_blocks %llu first_data_blkno %llu last_data_blkno %llu\n" - " quorum_fenced_term %llu quorum_server_term %llu unmount_barrier %llu\n" - " quorum_count %u server_addr %s\n" + " unmount_barrier %llu\n" " meta_alloc[0]: "ALCROOT_F"\n" " meta_alloc[1]: "ALCROOT_F"\n" " data_alloc: "ALCROOT_F"\n" @@ -894,11 +891,7 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno) le64_to_cpu(super->total_data_blocks), le64_to_cpu(super->first_data_blkno), le64_to_cpu(super->last_data_blkno), - le64_to_cpu(super->quorum_fenced_term), - le64_to_cpu(super->quorum_server_term), le64_to_cpu(super->unmount_barrier), - super->quorum_count, - server_addr, ALCROOT_A(&super->meta_alloc[0]), ALCROOT_A(&super->meta_alloc[1]), ALCROOT_A(&super->data_alloc), @@ -922,7 +915,19 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno) le64_to_cpu(super->fs_root.ref.blkno), le64_to_cpu(super->fs_root.ref.seq)); - free(server_addr); + printf(" quorum config version %llu\n", + le64_to_cpu(super->qconf.version)); + for (i = 0; i < array_size(super->qconf.slots); i++) { + if (!super->qconf.slots[i].addr.addr && + !super->qconf.slots[i].addr.port) + continue; + + addr = alloc_addr_str(&super->qconf.slots[i].addr); + if (addr) { + printf(" quorum slot %2u: %s\n", i, addr); + free(addr); + } + } } struct print_args {