diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 24cf93fe..fd79053d 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -26,6 +26,7 @@ scoutfs-y += \ net.o \ options.o \ per_task.o \ + quorum.o \ scoutfs_trace.o \ seg.o \ server.o \ diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 82bead6b..da993bc8 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -111,6 +111,18 @@ EXPAND_COUNTER(net_recv_invalid_message) \ EXPAND_COUNTER(net_recv_messages) \ EXPAND_COUNTER(net_unknown_request) \ + EXPAND_COUNTER(quorum_elected) \ + EXPAND_COUNTER(quorum_election_error) \ + EXPAND_COUNTER(quorum_fenced) \ + EXPAND_COUNTER(quorum_found_leader) \ + EXPAND_COUNTER(quorum_no_leader) \ + EXPAND_COUNTER(quorum_read_block) \ + EXPAND_COUNTER(quorum_read_block_error) \ + EXPAND_COUNTER(quorum_read_invalid_block) \ + EXPAND_COUNTER(quorum_read_invalid_config) \ + EXPAND_COUNTER(quorum_waited) \ + EXPAND_COUNTER(quorum_write_block) \ + EXPAND_COUNTER(quorum_write_block_error) \ EXPAND_COUNTER(seg_alloc) \ EXPAND_COUNTER(seg_csum_error) \ EXPAND_COUNTER(seg_free) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index f577b4bf..6d1bd222 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -38,6 +38,14 @@ */ #define SCOUTFS_SUPER_BLKNO ((64ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) +/* + * A reasonably large region of aligned quorum blocks follow the super + * block. + */ +#define SCOUTFS_QUORUM_BLKNO ((128ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_QUORUM_BLOCKS ((128ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS + /* * Base types used by other structures. */ @@ -366,6 +374,65 @@ struct scoutfs_xattr { #define SCOUTFS_UUID_BYTES 16 #define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */ +/* + * During each quorum voting interval the fabric has to process 2 reads + * and a write for each voting mount. The only reason we limit the + * number of active quorum mounts is to limit the number of IOs per + * interval. We use a pretty conservative interval given that IOs will + * generally be faster than our constant and we'll have fewer active + * than the max. + */ +#define SCOUTFS_QUORUM_MAX_ACTIVE 7 +#define SCOUTFS_QUORUM_IO_LATENCY_MS 10 +#define SCOUTFS_QUORUM_INTERVAL_MS \ + (SCOUTFS_QUORUM_MAX_ACTIVE * 3 * SCOUTFS_QUORUM_IO_LATENCY_MS) + +/* + * Each mount that is found in the quorum config in the super block can + * write to quorum blocks indicating which mount they vote for as + * the leader. + * + * @config_gen: references the config gen in the super block + * @write_nr: incremented for every write, only 0 when never written + * @elected_nr: incremented when elected, 0 otherwise + * @vote_slot: the active config slot that the writer is voting for + */ +struct scoutfs_quorum_block { + __le64 fsid; + __le64 blkno; + __le64 config_gen; + __le64 write_nr; + __le64 elected_nr; + __le32 crc; + __u8 vote_slot; +} __packed; + +#define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS + +/* + * Each quorum voter is described by a slot which corresponds to the + * block that the voter will write to. + * + * The stale flag is used to support config migration. A new + * configuration is written in free slots and the old configuration is + * marked stale. Stale slots can only be reclaimed once we have + * evidence that the named mount won't try and write to it by seeing it + * write to other slots or connect with the new gen. + */ +struct scoutfs_quorum_config { + __le64 gen; + struct scoutfs_quorum_slot { + __u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; + struct scoutfs_inet_addr addr; + __u8 vote_priority; + __u8 flags; + } __packed slots[SCOUTFS_QUORUM_MAX_SLOTS]; +} __packed; + +#define SCOUTFS_QUORUM_SLOT_ACTIVE (1 << 0) +#define SCOUTFS_QUORUM_SLOT_STALE (1 << 1) +#define SCOUTFS_QUORUM_SLOT_FLAGS_UNKNOWN (U8_MAX << 2) + struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; @@ -383,6 +450,7 @@ struct scoutfs_super_block { struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; struct scoutfs_inet_addr server_addr; + struct scoutfs_quorum_config quorum_config; } __packed; #define SCOUTFS_ROOT_INO 1 diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c new file mode 100644 index 00000000..112e2da9 --- /dev/null +++ b/kmod/src/quorum.c @@ -0,0 +1,718 @@ +/* + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "msg.h" +#include "counters.h" +#include "quorum.h" +#include "server.h" +#include "net.h" +#include "scoutfs_trace.h" + +/* + * scoutfs mounts use a region of statically allocated blocks in the + * shared metadata device to elect a leader mount who runs the server + * that the rest of the mounts of the filesystem connect to. + * + * Mounts that should participate in the election are configured in an + * array in the super block. Their position in the array determines the + * preallocated block that they'll be writing to. Mounts that aren't + * participating in the election only read the blocks to discover the + * outcome of the election. + * + * During the election each participating mount reads all the quorum + * blocks that all the mounts wrote, sees which are active and chooses + * which to vote for, and writes a new version of their block that + * includes their vote. Mounts vote for the mount with the highest + * priority in the config that is seen actively writing voting blocks + * over time. + * + * Once a mount receives a majority of votes from its peers then it + * writes its block with an indication that it has been elected. Only + * after reading that block, and seeing no other blocks that indicate + * more recently elected leaders, will it consider itself elected and + * try to fence any other previously elected leaders before starting the + * server. This ensures that racing elected leaders will always result + * in fencing all but the most recent. + * + * XXX: + * - actually fence + * - add temporary priority for choosing a specific mount as a leader + * - add config rotation (write new config, reclaim stale slots) + */ + +static void addr_to_sin(struct sockaddr_in *sin, struct scoutfs_inet_addr *addr) +{ + sin->sin_family = AF_INET; + sin->sin_addr.s_addr = cpu_to_be32(le32_to_cpu(addr->addr)); + sin->sin_port = cpu_to_be16(le16_to_cpu(addr->port)); +} + +/* active slots are sorted to the front for validation */ +static int cmp_slot_active(const struct scoutfs_quorum_slot *a, + const struct scoutfs_quorum_slot *b) +{ + int a_active = !!(a->flags & SCOUTFS_QUORUM_SLOT_ACTIVE); + int b_active = !!(b->flags & SCOUTFS_QUORUM_SLOT_ACTIVE); + + return b_active - a_active; +} + +/* slot validation has ensured that the names are null terminated */ +static int cmp_slot_names(const void *A, const void *B) +{ + const struct scoutfs_quorum_slot *a = A; + const struct scoutfs_quorum_slot *b = B; + + return cmp_slot_active(a, b) ?: + strcmp(a->name, b->name); +} + +static int cmp_slot_addrs(const void *A, const void *B) +{ + const struct scoutfs_quorum_slot *a = A; + const struct scoutfs_quorum_slot *b = B; + + return cmp_slot_active(a, b) ?: + memcmp(&a->addr, &b->addr, sizeof(a->addr)); +} + +static void swap_slots(void *A, void *B, int size) +{ + struct scoutfs_quorum_slot *a = A; + struct scoutfs_quorum_slot *b = B; + + swap(*a, *b); +} + +/* + * We'll set the callers our_slot to the slot that contains the their. + * If the name isn't found then it'll be set to -1. + */ +static int read_quorum_config(struct super_block *sb, + struct scoutfs_super_block *super, + char *our_name, int *our_slot_ret, + int *nr_active_ret) +{ + struct scoutfs_quorum_slot *sorted = NULL; + struct scoutfs_quorum_slot *slot; + struct scoutfs_quorum_config *conf; + struct sockaddr_in sin; + int nr_active = 0; + int our_slot = -1; + int ret; + int i; + + sorted = kcalloc(SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), GFP_NOFS); + if (sorted == NULL) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_read_super(sb, super); + if (ret) + goto out; + conf = &super->quorum_config; + + ret = -EINVAL; + + if (conf->gen == 0) { + scoutfs_err(sb, "invalid zero quorum config gen"); + goto out; + } + + for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { + slot = &conf->slots[i]; + + if (slot->flags & SCOUTFS_QUORUM_SLOT_FLAGS_UNKNOWN) { + scoutfs_err(sb, "quorum slot ind %u unknown flags 0x%02x", + i, slot->flags); + goto out; + } + + if ((slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE) && + (slot->flags & SCOUTFS_QUORUM_SLOT_STALE)) { + scoutfs_err(sb, "quorum slot ind %u is both active and stale", + i); + goto out; + } + + if (!(slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE)) + continue; + + nr_active++; + + if (slot->name[0] == '\0') { + scoutfs_err(sb, "quorum slot ind %u name is null", i); + goto out; + } + + if (slot->name[SCOUTFS_UNIQUE_NAME_MAX_BYTES - 1] != '\0') { + scoutfs_err(sb, "quorum slot ind %u name isn't null terminated", + i); + goto out; + } + + if (our_name && strcmp(our_name, slot->name) == 0) + our_slot = i; + + addr_to_sin(&sin, &slot->addr); + + if (ipv4_is_multicast(sin.sin_addr.s_addr) || + ipv4_is_lbcast(sin.sin_addr.s_addr) || + ipv4_is_zeronet(sin.sin_addr.s_addr) || + ipv4_is_local_multicast(sin.sin_addr.s_addr) || + ntohs(sin.sin_port) == 0 || + ntohs(sin.sin_port) == U16_MAX) { + scoutfs_err(sb, "quorum slot ind %u has invalid addr %pIS:%u", + i, &sin, ntohs(sin.sin_port)); + goto out; + } + } + + if (nr_active == 0) { + scoutfs_err(sb, "quorum config has no active slots"); + goto out; + } + + if (nr_active > SCOUTFS_QUORUM_MAX_ACTIVE) { + scoutfs_err(sb, "quorum config has %u active slots, can have at most %u ", + nr_active, SCOUTFS_QUORUM_MAX_ACTIVE); + goto out; + } + + memcpy(sorted, conf->slots, + SCOUTFS_QUORUM_MAX_SLOTS * sizeof(sorted[0])); + + sort(sorted, SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), + cmp_slot_names, swap_slots); + + for (i = 1; i < nr_active; i++) { + if (strcmp(sorted[i].name, sorted[i - 1].name) == 0) { + scoutfs_err(sb, "multiple quorum slots have the same name '%s'", + sorted[i].name); + goto out; + } + } + + sort(sorted, SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), + cmp_slot_addrs, swap_slots); + + for (i = 1; i < nr_active; i++) { + if (memcmp(&sorted[i].addr, &sorted[i - 1].addr, + sizeof(sorted[i].addr)) == 0) { + addr_to_sin(&sin, &sorted[i].addr); + scoutfs_err(sb, "multiple quorum slots have the same address %pIS:%u", + &sin, ntohs(sin.sin_port)); + goto out; + } + } + + ret = 0; + if (our_slot_ret) + *our_slot_ret = our_slot; + if (nr_active_ret) + *nr_active_ret = nr_active; +out: + if (ret) + scoutfs_inc_counter(sb, quorum_read_invalid_config); + kfree(sorted); + return ret; +} + +/* + * The caller is about to read the current version of a set of quorum + * blocks. We invalidate all the quorum blocks in the cache and + * populate the cache with all the blocks with one large contiguous + * read. The caller then uses simple sync bh methods to access + * whichever blocks it needs. I'm not a huge fan of the plug but I + * couldn't get the individual readahead requests merged without it. + */ +static void readahead_quorum_blocks(struct super_block *sb) +{ + struct buffer_head *bh; + struct blk_plug plug; + int i; + + blk_start_plug(&plug); + + for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { + bh = sb_getblk(sb, SCOUTFS_QUORUM_BLKNO + i); + if (!bh) + continue; + + lock_buffer(bh); + clear_buffer_uptodate(bh); + unlock_buffer(bh); + + ll_rw_block(READA | REQ_META | REQ_PRIO, 1, &bh); + brelse(bh); + } + + blk_finish_plug(&plug); +} + +/* + * Callers don't mind us clobbering the crc temporarily. + */ +static __le32 quorum_block_crc(struct scoutfs_quorum_block *blk) +{ + __le32 calc_crc; + __le32 blk_crc; + + blk_crc = blk->crc; + blk->crc = 0; + calc_crc = cpu_to_le32(crc32c(~0, blk, sizeof(*blk))); + blk->crc = blk_crc; + + return calc_crc; +} + +static bool invalid_quorum_block(struct scoutfs_super_block *super, + struct buffer_head *bh, + struct scoutfs_quorum_block *blk) +{ + return quorum_block_crc(blk) != blk->crc || + blk->fsid != super->hdr.fsid || + le64_to_cpu(blk->blkno) != bh->b_blocknr || + blk->vote_slot >= SCOUTFS_QUORUM_MAX_SLOTS; +} + +/* + * Give the caller the most recently updated version of the quorum + * block. Returns 0 and fills the callers block struct on success. + * Returns -ENOENT and zeros the caller's block if we couldn't read a + * valid block. We don't consider the config gen, that's up to the + * caller. + */ +static int read_quorum_block(struct super_block *sb, + struct scoutfs_super_block *super, int slot, + struct scoutfs_quorum_block *blk_ret) +{ + struct scoutfs_quorum_block *blk; + struct buffer_head *bh; + int ret; + + /* code strongly assumes that slots and blocks are directly mapped */ + BUILD_BUG_ON(SCOUTFS_QUORUM_BLOCKS != SCOUTFS_QUORUM_MAX_SLOTS); + + ret = -ENOENT; + + bh = sb_bread(sb, SCOUTFS_QUORUM_BLKNO + slot); + if (!bh) { + scoutfs_inc_counter(sb, quorum_read_block_error); + goto out; + } + blk = (void *)(bh->b_data); + + /* ignore unwritten blocks */ + if (blk->write_nr == 0) + goto out; + + trace_scoutfs_quorum_read_block(sb, bh->b_blocknr, blk); + + if (invalid_quorum_block(super, bh, blk)) { + scoutfs_inc_counter(sb, quorum_read_invalid_block); + goto out; + } + + *blk_ret = *blk; + scoutfs_inc_counter(sb, quorum_read_block); + ret = 0; +out: + if (ret < 0) + memset(blk_ret, 0, sizeof(struct scoutfs_quorum_block)); + brelse(bh); + return ret; +} + +/* + * Iterate over config slots from the given index and return the first + * slot that has any of the given flags set. + */ +static inline int first_slot_flags(struct scoutfs_quorum_config *conf, + int i, u8 flags) +{ + for (; i < ARRAY_SIZE(conf->slots); i++) { + if (conf->slots[i].flags & flags) + break; + } + return i; +} + +/* + * Execute the loop body with the read block for each slot that's + * configured and active. If we can't read the block for whatever + * reason then the loop will execute with the blk struct zeroed. + */ +#define for_each_active_block(sb, super, conf, hists, hi, blk, slot, i) \ + for (i = first_slot_flags(conf, 0, SCOUTFS_QUORUM_SLOT_ACTIVE); \ + (i < ARRAY_SIZE(conf->slots)) && \ + (slot = &conf->slots[i], \ + hi = &hists[i], \ + read_quorum_block(sb, super, i, blk), 1); \ + i = first_slot_flags(conf, i + 1, SCOUTFS_QUORUM_SLOT_ACTIVE)) + +/* + * Iterate over every possible block, regardless of config. A lot of these + * will be zero. + */ +#define for_each_block(sb, super, i, blk) \ + for (i = 0; \ + (i < SCOUTFS_QUORUM_BLOCKS) && \ + (read_quorum_block(sb, super, i, blk), 1); \ + i++) + +/* + * Synchronously write a single quorum block. The caller has provided + * the meaningful fields for the write. We fill in the rest that are + * consistent for every write and zero the rest of the block. + */ +static int write_quorum_block(struct super_block *sb, __le64 fsid, + __le64 config_gen, u8 our_slot, __le64 write_nr, + u64 elected_nr, u8 vote_slot) +{ + struct scoutfs_quorum_block *blk; + struct buffer_head *bh; + int ret; + + BUILD_BUG_ON(sizeof(struct scoutfs_quorum_block) > SCOUTFS_BLOCK_SIZE); + + if (WARN_ON_ONCE(our_slot >= SCOUTFS_QUORUM_MAX_SLOTS) || + WARN_ON_ONCE(vote_slot >= SCOUTFS_QUORUM_MAX_SLOTS)) + return -EINVAL; + + bh = sb_getblk(sb, SCOUTFS_QUORUM_BLKNO + our_slot); + if (bh == NULL) { + ret = -EIO; + goto out; + } + blk = (void *)bh->b_data; + + blk->fsid = fsid; + blk->blkno = cpu_to_le64(bh->b_blocknr); + blk->config_gen = config_gen; + blk->write_nr = write_nr; + blk->elected_nr = cpu_to_le64(elected_nr); + blk->vote_slot = vote_slot; + + blk->crc = quorum_block_crc(blk); + + lock_buffer(bh); + set_buffer_mapped(bh); + bh->b_end_io = end_buffer_write_sync; + get_bh(bh); + submit_bh(WRITE_SYNC | REQ_META | REQ_PRIO, bh); + + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) + ret = -EIO; + else + ret = 0; + + if (ret == 0) { + trace_scoutfs_quorum_write_block(sb, bh->b_blocknr, blk); + scoutfs_inc_counter(sb, quorum_write_block); + } +out: + if (ret) + scoutfs_inc_counter(sb, quorum_write_block_error); + brelse(bh); + return ret; +} + +/* + * The caller read their quorum block which indicated that they were + * elected. We have to fence all other previously elected leaders so + * that we're running the only instance of the server. + * + * Time can pass between all phases of this: reading that we're elected, + * fencing, and writing the quorum block that clears the elected flag of + * those we fenced. + * + * This is always safe because we either have exclusive access to the + * device having fenced others or someone else would have fenced us + * before they write. + */ +static int fence_other_elected(struct super_block *sb, + struct scoutfs_super_block *super, + int our_slot, u64 elected_nr) +{ + struct scoutfs_quorum_config *conf = &super->quorum_config; + struct scoutfs_quorum_block blk; + int ret; + int i; + + for_each_block(sb, super, i, &blk) { + if (i != our_slot && + le64_to_cpu(blk.elected_nr) > 0 && + le64_to_cpu(blk.elected_nr) <= elected_nr) { + scoutfs_err(sb, "would have fenced"); + scoutfs_inc_counter(sb, quorum_fenced); + + ret = write_quorum_block(sb, super->hdr.fsid, + conf->gen, i, blk.write_nr, + 0, i); + if (ret) + break; + } + } + + return ret; +} + +struct quorum_block_history { + __le64 write_nr; + u8 writing; +}; + +/* + * The caller couldn't connect to a server. Read the quorum blocks + * until we see an elected leader and give their address to the caller. + * If we're configured as part of the quorum then we participate in the + * electing by writing our vote to our quorum block. + * + * Voting members read the blocks at regular intervals and update their + * quorum block with their vote for the elected leader. new leader. + * When a mount receives enough votes it marks its vote in the block as + * elected, fences other elected leaders, and returns to the caller who + * starts up the server for others to connect to. + * + * The calling client may have never seen a server before, or could have + * failed to connect to a valid server, or might have tried to connect + * to a dead server. They pass in an existing elected_nr if they want + * us to ignore old servers and they pass in a timeout so that they can + * return to retrying to connect to whatever address we find. + * + * When we return success we update the caller's elected info with the + * most recent elected leader we found, which may well be long gone. We + * return -ENOENT if we didn't find any elected leaders. + */ +int scoutfs_quorum_election(struct super_block *sb, char *our_name, + u64 old_elected_nr, ktime_t timeout_abs, + struct scoutfs_quorum_elected_info *qei) +{ + struct scoutfs_super_block *super = NULL; + struct scoutfs_quorum_config *conf; + struct scoutfs_quorum_slot *slot; + struct scoutfs_quorum_block blk; + struct quorum_block_history *hist; + struct quorum_block_history *hi; + ktime_t expires; + ktime_t now; + __le64 write_nr = 0; + u64 elected_nr = 0; + int vote_streak = 0; + int vote_slot; + int our_slot; + int vote_prio; + int nr_active; + int nr_votes; + int majority; + int ret; + int i; + + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + hist = kcalloc(SCOUTFS_QUORUM_MAX_SLOTS, sizeof(hist[0]), GFP_NOFS); + if (!super || !hist) { + ret = -ENOMEM; + goto out; + } + + for (;;) { + now = ktime_get(); + expires = ktime_add_ms(now, SCOUTFS_QUORUM_INTERVAL_MS); + + ret = read_quorum_config(sb, super, our_name, &our_slot, + &nr_active); + if (ret) + goto out; + conf = &super->quorum_config; + + /* allow a single vote majority when 1 or 2 active */ + if (nr_active <= 2) + majority = 1; + else if (nr_active & 1) + majority = (nr_active + 1) / 2; + else + majority = (nr_active / 2) + 1; + + readahead_quorum_blocks(sb); + + /* default to voting for ourselves, but at min prio */ + vote_slot = our_slot; + vote_prio = -1; + memset(qei, 0, sizeof(*qei)); + + for_each_active_block(sb, super, conf, hist, hi, &blk, slot, i){ + /* determine which mounts are writing */ + if (blk.config_gen == conf->gen && + blk.write_nr != 0 && + blk.write_nr != hi->write_nr) + hi->writing = min(hi->writing + 1, 2); + else + hi->writing = 0; + hi->write_nr = blk.write_nr; + + /* vote for first highest priority writing block */ + if (hi->writing >= 2 && + slot->vote_priority > vote_prio) { + vote_slot = i; + vote_prio = slot->vote_priority; + } + + /* find the most recently elected leader */ + if ((blk.config_gen == conf->gen) && + (le64_to_cpu(blk.elected_nr) > qei->elected_nr)){ + addr_to_sin(&qei->sin, &slot->addr); + qei->config_gen = blk.config_gen; + qei->write_nr = blk.write_nr; + qei->elected_nr = le64_to_cpu(blk.elected_nr); + qei->config_slot = i; + } + } + + /* + * After writing a block indicating that we were elected + * we make sure that we can read it and that we're still + * the most recent elected leader. If we are then we + * try to fence. If we can't read it, or we're not the + * most recent, or we couldn't fence, then we fall back + * to participating in the election. + */ + if (elected_nr != 0) { + if (qei->write_nr == write_nr && + qei->elected_nr == elected_nr && + qei->config_slot == our_slot) { + ret = fence_other_elected(sb, super, our_slot, + elected_nr); + if (ret == 0) { + qei->run_server = true; + goto out; + } + + memset(qei, 0, sizeof(*qei)); + } + + vote_streak = 0; + } + + /* return if we found a new leader or ran out of time */ + if (qei->elected_nr > old_elected_nr || + ktime_after(now, timeout_abs)) { + if (qei->elected_nr > 0) { + scoutfs_inc_counter(sb, quorum_found_leader); + ret = 0; + } else { + scoutfs_inc_counter(sb, quorum_no_leader); + ret = -ENOENT; + } + goto out; + } + + /* wait for the next cycle if we're not in the voting config */ + if (our_slot < 0) + continue; + + nr_votes = 0; + write_nr = cpu_to_le64(1); + elected_nr = 0; + + for_each_active_block(sb, super, conf, hist, hi, &blk, slot, i){ + /* count our votes (maybe including from us) */ + if (hi->writing >= 2 && blk.vote_slot == our_slot) + nr_votes++; + + /* sample existing fields for our write */ + if (i == our_slot) { + write_nr = blk.write_nr; + le64_add_cpu(&write_nr, 1); + } + elected_nr = max(elected_nr, + le64_to_cpu(blk.elected_nr)); + } + + + /* elected after sufficient cycles with a majority vote */ + if (nr_votes >= majority) + vote_streak = min(vote_streak + 1, 2); + else + vote_streak = 0; + + if (vote_streak >= 2) + elected_nr++; + else + elected_nr = 0; + + write_quorum_block(sb, super->hdr.fsid, conf->gen, our_slot, + write_nr, elected_nr, vote_slot); + + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_hrtimeout(&expires, HRTIMER_MODE_ABS); + scoutfs_inc_counter(sb, quorum_waited); + } + +out: + kfree(super); + kfree(hist); + + if (ret) { + memset(qei, 0, sizeof(*qei)); + scoutfs_inc_counter(sb, quorum_election_error); + } + + return ret; +} + +/* + * The calling server is shutting down and has finished modifying + * persistent state. We clear elected_nr from our quorum block so that + * mounts won't try to connect and so that the next next leader won't + * try to fence. + * + * By definition nothing has written to the slot since we wrote our + * elected_nr and the slot could not have been reclaimed. To reclaim + * the slot would have required proving that we were gone or fencing + * us. + * + * If this fails then the mount is in trouble because it'll probably be + * fenced by the next elected leader. + * + * XXX I think there's an interesting race here. If the server is + * running in an old config then the server's slot can be reclaimed if + * the server sees a connection from the current gen. If the server is + * taking a client connection as an indication that the slot won't be + * written then the client needs to shut down the server before trying + * to connect with a new gen. + */ +int scoutfs_quorum_clear_elected(struct super_block *sb, + struct scoutfs_quorum_elected_info *qei) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, + qei->config_slot, qei->write_nr, 0, + qei->config_slot); +} diff --git a/kmod/src/quorum.h b/kmod/src/quorum.h new file mode 100644 index 00000000..82e6708e --- /dev/null +++ b/kmod/src/quorum.h @@ -0,0 +1,19 @@ +#ifndef _SCOUTFS_QUORUM_H_ +#define _SCOUTFS_QUORUM_H_ + +struct scoutfs_quorum_elected_info { + struct sockaddr_in sin; + __le64 config_gen; + __le64 write_nr; + u64 elected_nr; + unsigned int config_slot; + bool run_server; +}; + +int scoutfs_quorum_election(struct super_block *sb, char *our_name, + u64 old_elected_nr, ktime_t timeout_abs, + struct scoutfs_quorum_elected_info *qei); +int scoutfs_quorum_clear_elected(struct super_block *sb, + struct scoutfs_quorum_elected_info *qei); + +#endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 132c94f1..3afe9059 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2423,6 +2423,50 @@ DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_down, TP_ARGS(sb, node_id, nr_clients) ); +DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, + TP_PROTO(struct super_block *sb, u64 io_blkno, + struct scoutfs_quorum_block *blk), + + TP_ARGS(sb, io_blkno, blk), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, io_blkno) + __field(__u64, hdr_blkno) + __field(__u64, config_gen) + __field(__u64, write_nr) + __field(__u64, elected_nr) + __field(__u32, crc) + __field(__u8, vote_slot) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->io_blkno = io_blkno; + __entry->hdr_blkno = le64_to_cpu(blk->blkno); + __entry->config_gen = le64_to_cpu(blk->config_gen); + __entry->write_nr = le64_to_cpu(blk->write_nr); + __entry->elected_nr = le64_to_cpu(blk->elected_nr); + __entry->crc = le32_to_cpu(blk->crc); + __entry->vote_slot = blk->vote_slot; + ), + + TP_printk("fsid "FSID_FMT" io_blkno %llu hdr_blkno %llu config_gen %llu write_nr %llu elected_nr %llu crc 0x%08x vote_slot %u", + __entry->fsid, __entry->io_blkno, __entry->hdr_blkno, + __entry->config_gen, __entry->write_nr, __entry->elected_nr, + __entry->crc, __entry->vote_slot) +); +DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_read_block, + TP_PROTO(struct super_block *sb, u64 io_blkno, + struct scoutfs_quorum_block *blk), + TP_ARGS(sb, io_blkno, blk) +); +DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_write_block, + TP_PROTO(struct super_block *sb, u64 io_blkno, + struct scoutfs_quorum_block *blk), + TP_ARGS(sb, io_blkno, blk) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */