mirrors/scoutfs
1
0
Fork 0
mirror of https://github.com/versity/scoutfs.git synced 2026-01-07 12:35:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

scoutfs-utils: remove quorum slot config

Browse Source 
The format no longer has statically configured named slots.  The only
persistent config is the number of monts that must be voting to reach
quorum.  The quorum blocks now have a log of successfull elections.

Signed-off-by: Zach Brown <zab@versity.com>
This commit is contained in:
Zach Brown
2019-06-25 10:03:07 -07:00
committed by Zach Brown
parent fc15b816b0
commit 3670a5b80d
3 changed files with 141 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
utils/src/format.h
View File

@@ -42,11 +42,14 @@
/*
* A reasonably large region of aligned quorum blocks follow the super
* block.
* block. Each voting cycle reads the entire region so we don't want it
* to be too enormous. 256K seems like a reasonably chunky single IO.
* The number of blocks in the region also determines the number of
* mounts that have a reasonable probability of not overwriting each
* other's random block locations.
*/
#define SCOUTFS_QUORUM_BLKNO ((128ULL * 1024) >> SCOUTFS_BLOCK_SHIFT)
#define SCOUTFS_QUORUM_BLOCKS ((128ULL * 1024) >> SCOUTFS_BLOCK_SHIFT)
#define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS
#define SCOUTFS_QUORUM_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SHIFT)
#define SCOUTFS_QUORUM_BLOCKS ((256ULL * 1024) >> SCOUTFS_BLOCK_SHIFT)
#define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */
@@ -304,9 +307,10 @@ struct scoutfs_mounted_client_btree_key {
} __packed;
struct scoutfs_mounted_client_btree_val {
__u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES];
__u8 flags;
} __packed;
#define SCOUTFS_MOUNTED_CLIENT_VOTER (1 << 0)
/*
* The max number of links defines the max number of entries that we can
@@ -421,70 +425,47 @@ struct scoutfs_xattr {
#define SCOUTFS_UUID_BYTES 16
/*
* During each quorum voting interval the fabric has to process 2 reads
* and a write for each voting mount. The only reason we limit the
* number of active quorum mounts is to limit the number of IOs per
* interval. We use a pretty conservative interval given that IOs will
* generally be faster than our constant and we'll have fewer active
* than the max.
* Mounts read all the quorum blocks and write to one random quorum
* block during a cycle. The min cycle time limits the per-mount iop
* load during elections. The random cycle delay makes it less likely
* that mounts will read and write at the same time and miss each
* other's writes. An election only completes if a quorum of mounts
* vote for a leader before any of their elections timeout. This is
* made less likely by the probability that mounts will overwrite each
* others random block locations. The max quorum count limits that
* probability. 9 mounts only have a 55% chance of writing to unique 4k
* blocks in a 256k region. The election timeout is set to include
* enough cycles to usually complete the election. Once a leader is
* elected it spends a number of cycles writing out blocks with itself
* logged as a leader. This reduces the possibility that servers
* will have their log entries overwritten and not be fenced.
*/
#define SCOUTFS_QUORUM_MAX_ACTIVE 7
#define SCOUTFS_QUORUM_IO_LATENCY_MS 10
#define SCOUTFS_QUORUM_INTERVAL_MS \
(SCOUTFS_QUORUM_MAX_ACTIVE * 3 * SCOUTFS_QUORUM_IO_LATENCY_MS)
#define SCOUTFS_QUORUM_MAX_COUNT 9
#define SCOUTFS_QUORUM_CYCLE_LO_MS 10
#define SCOUTFS_QUORUM_CYCLE_HI_MS 20
#define SCOUTFS_QUORUM_TERM_LO_MS 250
#define SCOUTFS_QUORUM_TERM_HI_MS 500
#define SCOUTFS_QUORUM_ELECTED_LOG_CYCLES 10
/*
* Each mount that is found in the quorum config in the super block can
* write to quorum blocks indicating which mount they vote for as
* the leader.
*
* @config_gen: references the config gen in the super block
* @write_nr: incremented for every write, only 0 when never written
* @elected_nr: incremented when elected, 0 otherwise
* @unmount_barrier: incremented by servers when all members have unmounted
* @vote_slot: the active config slot that the writer is voting for
*/
struct scoutfs_quorum_block {
__le64 fsid;
__le64 blkno;
__le64 config_gen;
__le64 term;
__le64 write_nr;
__le64 elected_nr;
__le64 unmount_barrier;
__le64 voter_rid;
__le64 vote_for_rid;
__le32 crc;
__u8 vote_slot;
__u8 flags;
} __packed;
#define SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED (1 << 0)
#define SCOUTFS_QUORUM_BLOCK_FLAG_LISTENING (1 << 1)
#define SCOUTFS_QUORUM_BLOCK_FLAGS_UNKNOWN (U8_MAX << 2)
#define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS
/*
* Each quorum voter is described by a slot which corresponds to the
* block that the voter will write to.
*
* The stale flag is used to support config migration. A new
* configuration is written in free slots and the old configuration is
* marked stale. Stale slots can only be reclaimed once we have
* evidence that the named mount won't try and write to it by seeing it
* write to other slots or connect with the new gen.
*/
struct scoutfs_quorum_config {
__le64 gen;
struct scoutfs_quorum_slot {
__u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES];
__u8 log_nr;
struct scoutfs_quorum_log {
__le64 term;
__le64 rid;
struct scoutfs_inet_addr addr;
__u8 vote_priority;
__u8 flags;
} __packed slots[SCOUTFS_QUORUM_MAX_SLOTS];
} __packed log[0];
} __packed;
#define SCOUTFS_QUORUM_SLOT_ACTIVE (1 << 0)
#define SCOUTFS_QUORUM_SLOT_STALE (1 << 1)
#define SCOUTFS_QUORUM_SLOT_FLAGS_UNKNOWN (U8_MAX << 2)
#define SCOUTFS_QUORUM_LOG_MAX \
((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_quorum_block)) / \
sizeof(struct scoutfs_quorum_log))
struct scoutfs_super_block {
struct scoutfs_block_header hdr;
@@ -500,9 +481,13 @@ struct scoutfs_super_block {
__le64 next_seg_seq;
__le64 next_node_id;
__le64 next_compact_id;
__le64 quorum_fenced_term;
__le64 quorum_server_term;
__le64 unmount_barrier;
__u8 quorum_count;
struct scoutfs_inet_addr server_addr;
struct scoutfs_btree_root alloc_root;
struct scoutfs_manifest manifest;
struct scoutfs_quorum_config quorum_config;
struct scoutfs_btree_root lock_clients;
struct scoutfs_btree_root trans_seqs;
struct scoutfs_btree_root mounted_clients;
@@ -624,8 +609,6 @@ enum {
* Greetings verify identity of communicating nodes. The sender sends
* their credentials and the receiver verifies them.
*
* @name: The client sends its unique name to the server.
*
* @server_term: The raft term that elected the server. Initially 0
* from the client, sent by the server, then sent by the client as it
* tries to reconnect. Used to identify a client reconnecting to a
@@ -634,7 +617,7 @@ enum {
* @unmount_barrier: Incremented every time the remaining majority of
* quorum members all agree to leave. The server tells a quorum member
* the value that it's connecting under so that if the client sees the
* value increase in a quorum block it knows that the server has
* value increase in the super block then it knows that the server has
* processed its farewell and can safely unmount.
*
* @node_id: The id of the client. Initially 0 from the client,
@@ -643,7 +626,6 @@ enum {
* state must be dealt with.
*/
struct scoutfs_net_greeting {
__u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES];
__le64 fsid;
__le64 format_hash;
__le64 server_term;
@@ -653,7 +635,8 @@ struct scoutfs_net_greeting {
} __packed;
#define SCOUTFS_NET_GREETING_FLAG_FAREWELL (1 << 0)
#define SCOUTFS_NET_GREETING_FLAG_INVALID (~(__u64)0 << 1)
#define SCOUTFS_NET_GREETING_FLAG_VOTER (1 << 1)
#define SCOUTFS_NET_GREETING_FLAG_INVALID (~(__u64)0 << 2)
/*
* This header precedes and describes all network messages sent over

175
utils/src/mkfs.c
View File

@@ -163,7 +163,7 @@ static char *size_str(u64 nr, unsigned size)
* - btree ring blocks with manifest and allocator btree blocks
* - segment with root inode items
*/
static int write_new_fs(char *path, int fd, struct scoutfs_quorum_config *conf)
static int write_new_fs(char *path, int fd, u8 quorum_count)
{
struct scoutfs_super_block *super;
struct scoutfs_key *ino_key;
@@ -176,9 +176,7 @@ static int write_new_fs(char *path, int fd, struct scoutfs_quorum_config *conf)
struct scoutfs_btree_block *bt;
struct scoutfs_btree_item *btitem;
struct scoutfs_segment_item *item;
struct scoutfs_quorum_slot *slot;
struct scoutfs_key key;
struct in_addr in;
__le32 *prev_link;
struct timeval tv;
char uuid_str[37];
@@ -239,9 +237,7 @@ static int write_new_fs(char *path, int fd, struct scoutfs_quorum_config *conf)
super->next_seg_seq = cpu_to_le64(2);
super->next_node_id = cpu_to_le64(1);
super->next_compact_id = cpu_to_le64(1);
super->quorum_config = *conf;
super->quorum_config.gen = cpu_to_le64(1);
super->quorum_count = quorum_count;
/* align the btree ring to the segment after the super */
blkno = round_up(SCOUTFS_SUPER_BLKNO + 1, SCOUTFS_SEGMENT_BLOCKS);
@@ -435,7 +431,8 @@ static int write_new_fs(char *path, int fd, struct scoutfs_quorum_config *conf)
" device bytes: "SIZE_FMT"\n"
" device blocks: "SIZE_FMT"\n"
" btree ring blocks: "SIZE_FMT"\n"
" free blocks: "SIZE_FMT"\n",
" free blocks: "SIZE_FMT"\n"
" quorum count: %u\n",
path,
le64_to_cpu(super->hdr.fsid),
le64_to_cpu(super->format_hash),
@@ -445,20 +442,8 @@ static int write_new_fs(char *path, int fd, struct scoutfs_quorum_config *conf)
SIZE_ARGS(le64_to_cpu(super->bring.nr_blocks),
SCOUTFS_BLOCK_SIZE),
SIZE_ARGS(le64_to_cpu(super->free_blocks),
SCOUTFS_BLOCK_SIZE));
printf(" quorum slots:\n");
for (i = 0; i < array_size(super->quorum_config.slots); i++) {
slot = &super->quorum_config.slots[i];
if (slot->flags == 0)
continue;
in.s_addr = htonl(le32_to_cpu(slot->addr.addr));
printf(" [%2u]: name %s priority %u addr:port %s:%u\n",
i, slot->name, slot->vote_priority,
inet_ntoa(in), le16_to_cpu(slot->addr.port));
}
SCOUTFS_BLOCK_SIZE),
super->quorum_count);
ret = 0;
out:
@@ -474,138 +459,16 @@ out:
}
static struct option long_ops[] = {
{ "quorum_slot", 1, NULL, 'Q' },
{ "quorum_count", 1, NULL, 'Q' },
{ NULL, 0, NULL, 0}
};
enum { NAME, PRIORITY, ADDR, PORT };
static int parse_quorum_slot(struct scoutfs_quorum_config *conf, char *arg)
{
struct scoutfs_quorum_slot *slot;
struct scoutfs_quorum_slot *sl;
struct in_addr in;
unsigned long port;
int free_slot;
char *save;
char *tok;
char *dup;
char *s;
int ret;
int i;
dup = strdup(arg);
if (!dup) {
printf("allocation failure while parsing quorum slot '%s'\n",
arg);
return -EINVAL;
}
for (i = 0; i < array_size(conf->slots); i++) {
if (conf->slots[i].flags == 0)
break;
}
if (i == array_size(conf->slots)) {
printf("too many quorum slots provided\n");
ret = -EINVAL;
goto out;
}
slot = &conf->slots[i];
free_slot = i;
slot->addr.port = cpu_to_le16(23853); /* randomly chosen */
for (save = NULL, s = dup, i = NAME; i <= PORT; i++, s = NULL) {
tok = strtok_r(s, ":", &save);
if (tok == NULL)
break;
/* assume flags and a default port */
if (i == PORT && !isdigit(tok[0]))
i = PRIORITY;
switch(i) {
case NAME:
if (strlen(tok) >= SCOUTFS_UNIQUE_NAME_MAX_BYTES) {
printf("quorum slot name too long: %s\n", tok);
return -EINVAL;
}
strcpy((char *)slot->name, tok);
break;
case PRIORITY:
slot->vote_priority = strtoul(tok, NULL, 0);
if (slot->vote_priority > 255) {
printf("invalid quorum slot priority: %s\n",
tok);
ret = -EINVAL;
goto out;
}
break;
case ADDR:
if (inet_aton(tok, &in) == 0) {
printf("invalid quorum slot address: %s\n", tok);
ret = -EINVAL;
goto out;
}
slot->addr.addr = cpu_to_le32(htonl(in.s_addr));
break;
case PORT:
port = strtoul(tok, NULL, 0);
if (port == 0 || port >= 65535) {
printf("invalid quorum slot port: %s\n", tok);
ret = -EINVAL;
goto out;
}
slot->addr.port = cpu_to_le16(port);
break;
}
}
if (slot->name[0] == '\0') {
printf("quorum slot must specify name: %s\n", arg);
ret = -EINVAL;
goto out;
}
if (slot->addr.addr == 0) {
printf("quorum slot must specify address: %s\n", arg);
ret = -EINVAL;
goto out;
}
for (i = 0; i < free_slot; i++) {
sl = &conf->slots[i];
if (strcmp((char *)slot->name, (char *)sl->name) == 0) {
printf("duplicate quorum slot name: %s\n", arg);
ret = -EINVAL;
goto out;
}
if (memcmp(&slot->addr, &sl->addr, sizeof(slot->addr)) == 0) {
printf("duplicate quorum slot addr: %s\n", arg);
ret = -EINVAL;
goto out;
}
}
slot->flags = SCOUTFS_QUORUM_SLOT_ACTIVE;
ret = 0;
out:
free(dup);
return ret;
}
static int mkfs_func(int argc, char *argv[])
{
struct scoutfs_quorum_config conf = {0,};
bool have_quorum = false;
unsigned long long ull;
char *path = argv[1];
u8 quorum_count = 0;
char *end = NULL;
int ret;
int fd;
int c;
@@ -613,10 +476,14 @@ static int mkfs_func(int argc, char *argv[])
while ((c = getopt_long(argc, argv, "Q:", long_ops, NULL)) != -1) {
switch (c) {
case 'Q':
ret = parse_quorum_slot(&conf, optarg);
if (ret)
return ret;
have_quorum = true;
ull = strtoull(optarg, &end, 0);
if (*end != '\0' || ull == 0 ||
ull > SCOUTFS_QUORUM_MAX_COUNT) {
printf("scoutfs: invalid quorum count '%s'\n",
optarg);
return -EINVAL;
}
quorum_count = ull;
break;
case '?':
default:
@@ -631,8 +498,8 @@ static int mkfs_func(int argc, char *argv[])
path = argv[optind];
if (!have_quorum) {
printf("must configure quorum with --quorum_slot|-Q options\n");
if (!quorum_count) {
printf("provide quorum count with --quorum_count|-Q option\n");
return -EINVAL;
}
@@ -644,7 +511,7 @@ static int mkfs_func(int argc, char *argv[])
return ret;
}
ret = write_new_fs(path, fd, &conf);
ret = write_new_fs(path, fd, quorum_count);
close(fd);
return ret;

128
utils/src/print.c
View File

@@ -382,8 +382,8 @@ static int print_mounted_client_entry(void *key, unsigned key_len, void *val,
struct scoutfs_mounted_client_btree_key *mck = key;
struct scoutfs_mounted_client_btree_val *mcv = val;
printf(" node_id %llu name %s\n",
be64_to_cpu(mck->node_id), mcv->name);
printf(" node_id %llu flags 0x%x\n",
be64_to_cpu(mck->node_id), mcv->flags);
return 0;
}
@@ -489,70 +489,89 @@ static int print_btree(int fd, struct scoutfs_super_block *super, char *which,
return ret;
}
static char *alloc_addr_str(struct scoutfs_inet_addr *ia)
{
struct in_addr addr;
char *quad;
char *str;
int len;
memset(&addr, 0, sizeof(addr));
addr.s_addr = htonl(le32_to_cpu(ia->addr));
quad = inet_ntoa(addr);
if (quad == NULL)
return NULL;
len = snprintf(NULL, 0, "%s:%u", quad, le16_to_cpu(ia->port));
if (len < 1 || len > 22)
return NULL;
len++; /* null */
str = malloc(len);
if (!str)
return NULL;
snprintf(str, len, "%s:%u", quad, le16_to_cpu(ia->port));
return str;
}
static int print_quorum_blocks(int fd, struct scoutfs_super_block *super)
{
struct scoutfs_quorum_block *blk;
struct scoutfs_quorum_block *blk = NULL;
char *log_addr = NULL;
u64 blkno;
int ret;
int i;
int j;
for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
blkno = SCOUTFS_QUORUM_BLKNO + i;
free(blk);
blk = read_block(fd, blkno);
if (!blk) {
ret = -ENOMEM;
break;
goto out;
}
if (blk->fsid != 0 || blk->write_nr != 0) {
if (blk->voter_rid != 0) {
printf("quorum block blkno %llu\n"
" fsid %llx blkno %llu config_gen %llu crc 0x%08x\n"
" write_nr %llu elected_nr %llu "
"unmount_barrier %llu vote_slot %u flags %02x\n",
" fsid %llx blkno %llu crc 0x%08x\n"
" term %llu write_nr %llu voter_rid %016llx "
"vote_for_rid %016llx\n"
" log_nr %u\n",
blkno, le64_to_cpu(blk->fsid),
le64_to_cpu(blk->blkno),
le64_to_cpu(blk->config_gen),
le32_to_cpu(blk->crc),
le64_to_cpu(blk->blkno), le32_to_cpu(blk->crc),
le64_to_cpu(blk->term),
le64_to_cpu(blk->write_nr),
le64_to_cpu(blk->elected_nr),
le64_to_cpu(blk->unmount_barrier),
blk->vote_slot, blk->flags);
le64_to_cpu(blk->voter_rid),
le64_to_cpu(blk->vote_for_rid),
blk->log_nr);
for (j = 0; j < blk->log_nr; j++) {
free(log_addr);
log_addr = alloc_addr_str(&blk->log[j].addr);
if (!log_addr) {
ret = -ENOMEM;
goto out;
}
printf(" [%u]: term %llu rid %llu addr %s\n",
j, le64_to_cpu(blk->log[j].term),
le64_to_cpu(blk->log[j].rid),
log_addr);
}
}
free(blk);
ret = 0;
}
ret = 0;
out:
free(log_addr);
return ret;
}
static void print_slot_flags(unsigned long flags)
{
if (flags == 0) {
printf("-");
return;
}
while (flags) {
if (flags & SCOUTFS_QUORUM_SLOT_ACTIVE) {
printf("active");
flags &= ~SCOUTFS_QUORUM_SLOT_ACTIVE;
} else if (flags & SCOUTFS_QUORUM_SLOT_STALE) {
printf("stale");
flags &= ~SCOUTFS_QUORUM_SLOT_STALE;
}
if (flags)
printf(",");
}
}
static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
{
struct scoutfs_quorum_slot *slot;
char uuid_str[37];
struct in_addr in;
char *server_addr;
u64 count;
int i;
@@ -563,10 +582,16 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
printf(" format_hash %llx uuid %s\n",
le64_to_cpu(super->format_hash), uuid_str);
server_addr = alloc_addr_str(&super->server_addr);
if (!server_addr)
return;
/* XXX these are all in a crazy order */
printf(" next_ino %llu next_trans_seq %llu next_seg_seq %llu\n"
" next_node_id %llu next_compact_id %llu\n"
" total_blocks %llu free_blocks %llu alloc_cursor %llu\n"
" quorum_fenced_term %llu quorum_server_term %llu unmount_barrier %llu\n"
" quorum_count %u server_addr %s\n"
" btree ring: first_blkno %llu nr_blocks %llu next_block %llu "
"next_seq %llu\n"
" lock_clients root: height %u blkno %llu seq %llu mig_len %u\n"
@@ -582,6 +607,11 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
le64_to_cpu(super->total_blocks),
le64_to_cpu(super->free_blocks),
le64_to_cpu(super->alloc_cursor),
le64_to_cpu(super->quorum_fenced_term),
le64_to_cpu(super->quorum_server_term),
le64_to_cpu(super->unmount_barrier),
super->quorum_count,
server_addr,
le64_to_cpu(super->bring.first_blkno),
le64_to_cpu(super->bring.nr_blocks),
le64_to_cpu(super->bring.next_block),
@@ -615,21 +645,7 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
}
printf("\n");
printf(" quorum_config:\n gen: %llu\n",
le64_to_cpu(super->quorum_config.gen));
for (i = 0; i < array_size(super->quorum_config.slots); i++) {
slot = &super->quorum_config.slots[i];
if (slot->flags == 0)
continue;
in.s_addr = htonl(le32_to_cpu(slot->addr.addr));
printf(" [%2u]: name %s priority %u addr %s:%u flags ",
i, slot->name, slot->vote_priority, inet_ntoa(in),
le16_to_cpu(slot->addr.port));
print_slot_flags(slot->flags);
printf("\n");
}
free(server_addr);
}
static int print_volume(int fd)