Merge pull request #62 from versity/zab/change_quorum_config

Zab/change quorum config

ReleaseNotes.md

@@ -1,6 +1,19 @@
 Versity ScoutFS Release Notes
 =============================
 
+---
+v1.x
+\
+*TBD*
+
+
+* **Add scoutfs(1) change-quorum-config command**
+\
+  Add a change-quorum-config command to scoutfs(1) to change the quorum
+  configuration stored in the metadata device while the file system is
+  unmounted.   This can be used to change the mounts that will
+  participate in quorum and the IP addresses they use.
+
 ---
 v1.0
 \

utils/man/scoutfs.8

@@ -42,6 +42,40 @@ the super blocks on both devices.
 .RE
 .PD
 
+.TP
+.BI "change-quorum-config {-Q|--quorum-slot} NR,ADDR,PORT [-F|--offline META-DEVICE DATA-DEVICE]"
+.sp
+Change the quorum configuration for an existing file system.  The new
+configuration completely replaces the old configuration.   Any slots
+from the old configuration that should be retained must be described
+with arguments in the new configuration.
+.sp
+Currently the configuration may only be changed offline.
+.sp
+.RS 1.0i
+.PD 0
+.TP
+.B "-Q, --quorum-slot NR,ADDR,PORT"
+The quorum configuration is built by specifying configured slots with
+multiple arguments as described in the
+.B mkfs
+command.
+.TP
+.B "-F, --offline META-DEVICE"
+Perform the change offline by updating the superblock in the metadata
+device.   The command will read the super block and refuse to make the
+change if it sees any evidence that the metadata device is currently in
+use.   The file system must be successfully unmounted after possibly
+recovering any previously unresolved mounts for the change to be
+successful.   After the change succeeds the newly configured slots can
+be used by mounts.
+.sp
+The offline change directly reads from and writes to the device and does
+not protect against concurrent use of the device.   It must be carefully
+run when the file system will not be mounted.
+.RE
+.PD
+
 .TP
 .BI "df [-h|--human-readable] [-p|--path PATH]"
 .sp

utils/src/change_format_version.c

@@ -45,16 +45,11 @@ static int do_change_fmt_vers(struct change_fmt_vers_args *args)
 {
 	struct scoutfs_super_block *meta_super = NULL;
 	struct scoutfs_super_block *data_super = NULL;
-	struct scoutfs_quorum_block *qblk = NULL;
-	struct scoutfs_quorum_block_event *beg;
-	struct scoutfs_quorum_block_event *end;
 	bool wrote_meta = false;
-	bool in_use = false;
 	char uuid_str[37];
 	int meta_fd = -1;
 	int data_fd = -1;
 	int ret;
-	int i;
 
 	meta_fd = open(args->meta_device, O_DIRECT | O_SYNC | O_RDWR | O_EXCL);
 	if (meta_fd < 0) {
@@ -117,44 +112,13 @@ static int do_change_fmt_vers(struct change_fmt_vers_args *args)
 		goto out;
 	}
 
-	if (meta_super->mounted_clients.ref.blkno != 0) {
-		fprintf(stderr, "meta superblock mounted clients btree is not empty.\n");
-		ret = -EBUSY;
-		in_use = true;
+	ret = meta_super_in_use(meta_fd, meta_super);
+	if (ret < 0) {
+		if (ret == -EBUSY)
+			fprintf(stderr, "The filesystem must be fully recovered and cleanly unmounted to change the format version\n");
 		goto out;
 	}
 
-	/* check for active quorum slots */
-	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
-		if (!quorum_slot_present(meta_super, i))
-			continue;
-		ret = read_block(meta_fd, SCOUTFS_QUORUM_BLKNO + i, SCOUTFS_BLOCK_SM_SHIFT,
-				 (void **)&qblk);
-		if (ret < 0) {
-			fprintf(stderr, "error reading quorum block for slot %u\n", i);
-			goto out;
-		}
-
-		beg = &qblk->events[SCOUTFS_QUORUM_EVENT_BEGIN];
-		end = &qblk->events[SCOUTFS_QUORUM_EVENT_END];
-
-		if (le64_to_cpu(beg->write_nr) > le64_to_cpu(end->write_nr)) {
-			fprintf(stderr, "mount in quorum slot %u could still be running.\n"
-					"  begin event: write_nr %llu timestamp %llu.%08u\n"
-					"    end event: write_nr %llu timestamp %llu.%08u\n",
-				i, le64_to_cpu(beg->write_nr), le64_to_cpu(beg->ts.sec),
-				le32_to_cpu(beg->ts.nsec),
-				le64_to_cpu(end->write_nr), le64_to_cpu(end->ts.sec),
-				le32_to_cpu(end->ts.nsec));
-			ret = -EBUSY;
-			in_use = true;
-			goto out;
-		}
-
-		free(qblk);
-		qblk = NULL;
-	}
-
 	if (le64_to_cpu(meta_super->fmt_vers) != args->fmt_vers) {
 		meta_super->fmt_vers = cpu_to_le64(args->fmt_vers);
 
@@ -195,11 +159,7 @@ static int do_change_fmt_vers(struct change_fmt_vers_args *args)
 		le64_to_cpu(meta_super->fmt_vers));
 
 out:
-	if (in_use)
-		fprintf(stderr, "The filesystem must be fully recovered and cleanly unmounted to change the format version\n");
 
-	if (qblk)
-		free(qblk);
 	if (meta_super)
 		free(meta_super);
 	if (data_super)

utils/src/change_quorum_config.c

@@ -0,0 +1,171 @@
+#define _GNU_SOURCE /* O_DIRECT */
+#include <unistd.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <uuid/uuid.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <assert.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <inttypes.h>
+#include <argp.h>
+
+#include "sparse.h"
+#include "cmd.h"
+#include "util.h"
+#include "format.h"
+#include "parse.h"
+#include "dev.h"
+#include "quorum.h"
+
+struct change_quorum_args {
+	char *meta_device;
+	bool offline;
+	int nr_slots;
+	struct scoutfs_quorum_slot slots[SCOUTFS_QUORUM_MAX_SLOTS];
+};
+
+static int do_change_quorum(struct change_quorum_args *args)
+{
+	struct scoutfs_super_block *meta_super = NULL;
+	char uuid_str[37];
+	int meta_fd = -1;
+	int ret;
+
+	meta_fd = open(args->meta_device, O_DIRECT | O_SYNC | O_RDWR | O_EXCL);
+	if (meta_fd < 0) {
+		ret = -errno;
+		fprintf(stderr, "failed to open meta device '%s': %s (%d)\n",
+			args->meta_device, strerror(errno), errno);
+		goto out;
+	}
+
+	ret = read_block_verify(meta_fd, SCOUTFS_BLOCK_MAGIC_SUPER, 0, SCOUTFS_SUPER_BLKNO,
+				SCOUTFS_BLOCK_SM_SHIFT, (void **)&meta_super);
+	if (ret) {
+		ret = -errno;
+		fprintf(stderr, "failed to read meta super block: %s (%d)\n",
+			strerror(errno), errno);
+		goto out;
+	}
+
+	ret = meta_super_in_use(meta_fd, meta_super);
+	if (ret < 0) {
+		if (ret == -EBUSY)
+			fprintf(stderr, "The filesystem must be fully recovered and cleanly unmounted to change the quorum config\n");
+		goto out;
+	}
+
+	assert(sizeof(meta_super->qconf.slots) == sizeof(args->slots));
+	memcpy(meta_super->qconf.slots, args->slots, sizeof(meta_super->qconf.slots));
+	le64_add_cpu(&meta_super->qconf.version, 1);
+
+	ret = write_block(meta_fd, SCOUTFS_BLOCK_MAGIC_SUPER, meta_super->hdr.fsid, 1,
+			  SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT, &meta_super->hdr);
+	if (ret)
+		goto out;
+
+	uuid_unparse(meta_super->uuid, uuid_str);
+
+	printf("Successfully changed quorum config for scoutfs filesystem:\n"
+	       "  meta device path:       %s\n"
+	       "  fsid:                   %llx\n"
+	       "  uuid:                   %s\n"
+	       "  quorum config version:  %llu\n"
+	       "  quorum slots:           ",
+		args->meta_device,
+		le64_to_cpu(meta_super->hdr.fsid),
+		uuid_str,
+		le64_to_cpu(meta_super->qconf.version));
+
+	print_quorum_slots(meta_super->qconf.slots, array_size(meta_super->qconf.slots),
+			   "                          ");
+
+out:
+
+	if (meta_super)
+		free(meta_super);
+	if (meta_fd != -1)
+		close(meta_fd);
+	return ret;
+}
+
+static int parse_opt(int key, char *arg, struct argp_state *state)
+{
+	struct change_quorum_args *args = state->input;
+	struct scoutfs_quorum_slot slot;
+	int ret;
+
+	switch (key) {
+	case 'F':
+		args->offline = true;
+		break;
+	case 'Q':
+		ret = parse_quorum_slot(&slot, arg);
+		if (ret < 0)
+			return ret;
+		if (args->slots[ret].addr.v4.family != cpu_to_le16(SCOUTFS_AF_NONE))
+			argp_error(state, "Quorum slot %u already specified before slot '%s'\n",
+				   ret, arg);
+		args->slots[ret] = slot;
+		args->nr_slots++;
+		break;
+	case ARGP_KEY_ARG:
+		if (!args->meta_device)
+			args->meta_device = strdup_or_error(state, arg);
+		else
+			argp_error(state, "more than one metadata device argument given");
+		break;
+	case ARGP_KEY_FINI:
+		if (!args->offline)
+			argp_error(state, "must specify --offline");
+		if (!args->meta_device)
+			argp_error(state, "no metadata device argument given");
+		if (!args->nr_slots)
+			argp_error(state, "must specify at least one quorum slot with --quorum-slot|-Q");
+		if (!valid_quorum_slots(args->slots))
+			argp_error(state, "invalid quorum slot configuration");
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static struct argp_option options[] = {
+	{ "quorum-slot", 'Q', "NR,ADDR,PORT", 0, "Specify quorum slot addresses [Required]"},
+	{ "offline", 'F', NULL, 0, "Write format version in offline device super blocks [Currently Required]"},
+	{ NULL }
+};
+
+static struct argp argp = {
+	options,
+	parse_opt,
+	"",
+	"Change quorum slots and addresses of an existing ScoutFS filesystem"
+};
+
+static int change_quorum_cmd(int argc, char *argv[])
+{
+	struct change_quorum_args change_quorum_args = {
+		.offline = false,
+	};
+	int ret;
+
+	ret = argp_parse(&argp, argc, argv, 0, NULL, &change_quorum_args);
+	if (ret)
+		return ret;
+
+	return do_change_quorum(&change_quorum_args);
+}
+
+static void __attribute__((constructor)) change_quorum_ctor(void)
+{
+	cmd_register_argp("change-quorum-config", &argp, GROUP_CORE, change_quorum_cmd);
+}

utils/src/mkfs.c

@@ -31,6 +31,7 @@
 #include "btree.h"
 #include "leaf_item_hash.h"
 #include "blkid.h"
+#include "quorum.h"
 
 
 /*
@@ -139,7 +140,6 @@ static int do_mkfs(struct mkfs_args *args)
 	int data_fd = -1;
 	char uuid_str[37];
 	void *zeros = NULL;
-	char *indent;
 	u64 blkno;
 	u64 meta_size;
 	u64 data_size;
@@ -224,6 +224,7 @@ static int do_mkfs(struct mkfs_args *args)
 	assert(sizeof(args->slots) ==
 		     member_sizeof(struct scoutfs_super_block, qconf.slots));
 	memcpy(super->qconf.slots, args->slots, sizeof(args->slots));
+	super->qconf.version = cpu_to_le64(1);
 
 	if (invalid_data_alloc_zone_blocks(le64_to_cpu(super->total_data_blocks),
 					   args->data_alloc_zone_blocks)) {
@@ -350,14 +351,15 @@ static int do_mkfs(struct mkfs_args *args)
 	uuid_unparse(super->uuid, uuid_str);
 
 	printf("Created scoutfs filesystem:\n"
-	       "  meta device path:     %s\n"
-	       "  data device path:     %s\n"
-	       "  fsid:                 %llx\n"
-	       "  uuid:                 %s\n"
-	       "  format version:       %llu\n"
-	       "  64KB metadata blocks: "SIZE_FMT"\n"
-	       "  4KB data blocks:      "SIZE_FMT"\n"
-	       "  quorum slots:         ",
+	       "  meta device path:       %s\n"
+	       "  data device path:       %s\n"
+	       "  fsid:                   %llx\n"
+	       "  uuid:                   %s\n"
+	       "  format version:         %llu\n"
+	       "  64KB metadata blocks:   "SIZE_FMT"\n"
+	       "  4KB data blocks:        "SIZE_FMT"\n"
+	       "  quorum config version:  %llu\n"
+	       "  quorum slots:           ",
 		args->meta_device,
 	        args->data_device,
 		le64_to_cpu(super->hdr.fsid),
@@ -366,22 +368,11 @@ static int do_mkfs(struct mkfs_args *args)
 		SIZE_ARGS(le64_to_cpu(super->total_meta_blocks),
 			  SCOUTFS_BLOCK_LG_SIZE),
 		SIZE_ARGS(le64_to_cpu(super->total_data_blocks),
-			  SCOUTFS_BLOCK_SM_SIZE));
+			  SCOUTFS_BLOCK_SM_SIZE),
+		le64_to_cpu(super->qconf.version));
 
-	indent = "";
-	for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
-		struct scoutfs_quorum_slot *sl = &super->qconf.slots[i];
-		struct in_addr in;
-
-		if (sl->addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4))
-			continue;
-
-		in.s_addr = htonl(le32_to_cpu(sl->addr.v4.addr));
-		printf("%s%u: %s:%u", indent,
-		       i, inet_ntoa(in), le16_to_cpu(sl->addr.v4.port));
-		indent = "\n                        ";
-	}
-	printf("\n");
+	print_quorum_slots(super->qconf.slots, array_size(super->qconf.slots),
+			   "                          ");
 
 	ret = 0;
 out:
@@ -398,45 +389,6 @@ out:
 	return ret;
 }
 
-static bool valid_quorum_slots(struct scoutfs_quorum_slot *slots)
-{
-	struct in_addr in;
-	bool valid = true;
-	char *addr;
-	int i;
-	int j;
-
-	for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
-		if (slots[i].addr.v4.family == cpu_to_le16(SCOUTFS_AF_NONE))
-			continue;
-
-		if (slots[i].addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4)) {
-			fprintf(stderr, "quorum slot nr %u has invalid family %u\n",
-				i, le16_to_cpu(slots[i].addr.v4.family));
-			valid = false;
-		}
-
-		for (j = i + 1; j < SCOUTFS_QUORUM_MAX_SLOTS; j++) {
-			if (slots[i].addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4))
-				continue;
-
-			if (slots[i].addr.v4.addr == slots[j].addr.v4.addr &&
-			    slots[i].addr.v4.port == slots[j].addr.v4.port) {
-
-				in.s_addr =
-					htonl(le32_to_cpu(slots[i].addr.v4.addr));
-				addr = inet_ntoa(in);
-				fprintf(stderr, "quorum slot nr %u and %u have the same address %s:%u\n",
-					i, j, addr,
-					le16_to_cpu(slots[i].addr.v4.port));
-				valid = false;
-			}
-		}
-	}
-
-	return valid;
-}
-
 static int parse_opt(int key, char *arg, struct argp_state *state)
 {
 	struct mkfs_args *args = state->input;
@@ -517,7 +469,7 @@ static int parse_opt(int key, char *arg, struct argp_state *state)
 		break;
 	case ARGP_KEY_FINI:
 		if (!args->nr_slots)
-			argp_error(state, "must specify at least one quorum slot with --quorum-count|-Q");
+			argp_error(state, "must specify at least one quorum slot with --quorum-slot|-Q");
 		if (!args->meta_device)
 			argp_error(state, "no metadata device argument given");
 		if (!args->data_device)

utils/src/quorum.c

@@ -1,3 +1,7 @@
+#include <stdio.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
 #include "sparse.h"
 #include "util.h"
 #include "format.h"
@@ -8,3 +12,68 @@ bool quorum_slot_present(struct scoutfs_super_block *super, int i)
 {
 	return super->qconf.slots[i].addr.v4.family == cpu_to_le16(SCOUTFS_AF_IPV4);
 }
+
+bool valid_quorum_slots(struct scoutfs_quorum_slot *slots)
+{
+	struct in_addr in;
+	bool valid = true;
+	char *addr;
+	int i;
+	int j;
+
+	for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
+		if (slots[i].addr.v4.family == cpu_to_le16(SCOUTFS_AF_NONE))
+			continue;
+
+		if (slots[i].addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4)) {
+			fprintf(stderr, "quorum slot nr %u has invalid family %u\n",
+				i, le16_to_cpu(slots[i].addr.v4.family));
+			valid = false;
+		}
+
+		for (j = i + 1; j < SCOUTFS_QUORUM_MAX_SLOTS; j++) {
+			if (slots[i].addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4))
+				continue;
+
+			if (slots[i].addr.v4.addr == slots[j].addr.v4.addr &&
+			    slots[i].addr.v4.port == slots[j].addr.v4.port) {
+
+				in.s_addr =
+					htonl(le32_to_cpu(slots[i].addr.v4.addr));
+				addr = inet_ntoa(in);
+				fprintf(stderr, "quorum slot nr %u and %u have the same address %s:%u\n",
+					i, j, addr,
+					le16_to_cpu(slots[i].addr.v4.port));
+				valid = false;
+			}
+		}
+	}
+
+	return valid;
+}
+
+/*
+ * Print quorum slots to stdout, a line at a time.   The first line is
+ * not indented and the rest of the lines use the indent string from the
+ * caller.
+ */
+void print_quorum_slots(struct scoutfs_quorum_slot *slots, int nr, char *indent)
+{
+	struct scoutfs_quorum_slot *sl;
+	struct in_addr in;
+	bool first = true;
+	int i;
+
+	for (i = 0, sl = slots; i < SCOUTFS_QUORUM_MAX_SLOTS; i++, sl++) {
+
+		if (sl->addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4))
+			continue;
+
+		in.s_addr = htonl(le32_to_cpu(sl->addr.v4.addr));
+		printf("%s%u: %s:%u\n", first ? "" : indent,
+		       i, inet_ntoa(in), le16_to_cpu(sl->addr.v4.port));
+
+		first = false;
+	}
+}
+

utils/src/quorum.h

@@ -4,5 +4,7 @@
 #include <stdbool.h>
 
 bool quorum_slot_present(struct scoutfs_super_block *super, int i);
+bool valid_quorum_slots(struct scoutfs_quorum_slot *slots);
+void print_quorum_slots(struct scoutfs_quorum_slot *slots, int nr, char *indent);
 
 #endif

utils/src/util.c

@@ -12,6 +12,7 @@
 #include "util.h"
 #include "format.h"
 #include "crc.h"
+#include "quorum.h"
 
 #define ENV_PATH "SCOUTFS_MOUNT_PATH"
 
@@ -201,3 +202,56 @@ int write_block_sync(int fd, u32 magic, __le64 fsid, u64 seq, u64 blkno,
 
 	return 0;
 }
+
+/*
+ * Check to see if the metadata super block indicates that there might
+ * be active mounts using the system.  Returns -errno, 0, or -EBUSY if
+ * we found evidence that the device might be in use.
+ */
+int meta_super_in_use(int meta_fd, struct scoutfs_super_block *meta_super)
+{
+	struct scoutfs_quorum_block *qblk = NULL;
+	struct scoutfs_quorum_block_event *beg;
+	struct scoutfs_quorum_block_event *end;
+	int ret = 0;
+	int i;
+
+	if (meta_super->mounted_clients.ref.blkno != 0) {
+		fprintf(stderr, "meta superblock mounted clients btree is not empty.\n");
+		ret = -EBUSY;
+		goto out;
+	}
+
+	/* check for active quorum slots */
+	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
+		if (!quorum_slot_present(meta_super, i))
+			continue;
+		ret = read_block(meta_fd, SCOUTFS_QUORUM_BLKNO + i, SCOUTFS_BLOCK_SM_SHIFT,
+				 (void **)&qblk);
+		if (ret < 0) {
+			fprintf(stderr, "error reading quorum block for slot %u\n", i);
+			goto out;
+		}
+
+		beg = &qblk->events[SCOUTFS_QUORUM_EVENT_BEGIN];
+		end = &qblk->events[SCOUTFS_QUORUM_EVENT_END];
+
+		if (le64_to_cpu(beg->write_nr) > le64_to_cpu(end->write_nr)) {
+			fprintf(stderr, "mount in quorum slot %u could still be running.\n"
+					"  begin event: write_nr %llu timestamp %llu.%08u\n"
+					"    end event: write_nr %llu timestamp %llu.%08u\n",
+				i, le64_to_cpu(beg->write_nr), le64_to_cpu(beg->ts.sec),
+				le32_to_cpu(beg->ts.nsec),
+				le64_to_cpu(end->write_nr), le64_to_cpu(end->ts.sec),
+				le32_to_cpu(end->ts.nsec));
+			ret = -EBUSY;
+			goto out;
+		}
+
+		free(qblk);
+		qblk = NULL;
+	}
+
+out:
+	return ret;
+}

utils/src/util.h

@@ -117,10 +117,12 @@ int read_block_crc(int fd, u64 blkno, int shift, void **ret_val);
 int read_block_verify(int fd, u32 magic, u64 fsid, u64 blkno, int shift, void **ret_val);
 
 struct scoutfs_block_header;
+struct scoutfs_super_block;
 int write_block(int fd, u32 magic, __le64 fsid, u64 seq, u64 blkno,
 		int shift, struct scoutfs_block_header *hdr);
 int write_block_sync(int fd, u32 magic, __le64 fsid, u64 seq, u64 blkno,
 		     int shift, struct scoutfs_block_header *hdr);
+int meta_super_in_use(int meta_fd, struct scoutfs_super_block *meta_super);
 
 #define __stringify_1(x) #x
 #define __stringify(x) __stringify_1(x)