diff --git a/kmod/src/client.c b/kmod/src/client.c index 8d3bcee6..8ef58589 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -53,10 +53,7 @@ struct client_info { atomic_t shutting_down; struct workqueue_struct *workq; - struct work_struct connect_work; - - struct scoutfs_quorum_elected_info qei; - u64 old_elected_nr; + struct delayed_work connect_dwork; u64 server_term; u64 greeting_umb; @@ -373,117 +370,108 @@ out: } /* - * If the previous election told us to start the server then stop it and - * clear the indication that we were elected. We get the current - * version of the election info from the server because they might have - * modified it while they were running. the old election info. + * This work is responsible for maintaining a connection from the client + * to the server. It's queued on mount and disconnect and we requeue + * the work if the work fails and we're not shutting down. * - * If we're not fast enough to clear the election from the quorum block - * then the next server might fence us. Should be very unlikely as - * election requires multiple RMW cycles. - */ -static void stop_our_server(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei) -{ - if (qei->run_server) { - scoutfs_server_stop(sb, qei); - scoutfs_quorum_clear_elected(sb, qei); - memset(qei, 0, sizeof(*qei)); - } -} - -/* - * This work is responsible for managing leader elections, running the - * server, and connecting clients to the server. - * - * In the typical case a mount reads the quorum blocks and finds the + * In the typical case a mount reads the super blocks and finds the * address of the currently running server and connects to it. + * Non-voting clients who can't connect will keep trying alternating + * reading the address and getting connect timeouts. * - * More rarely clients who aren't connected and are configured to - * participate in quorum need to elect the new leader. The elected info - * filled by quorum tells us if we were elected to run the server. + * Voting mounts will try to elect a leader if they can't connect to the + * server. When a quorum can't connect and are able to elect a leader + * then a new server is started. The new server will write its address + * in the super and everyone will be able to connect. * - * This leads to the possibility that the mount who is running the - * server had its mount disconnect. This is only weirdly different from - * other clients disconnecting and trying to reconnect because of the - * way quorum slots are reconfigured and reclaimed. If we connect to a - * server with the new quorum config then we can't have any old servers - * running in the stale old quorum slot. The simplest way to do this is - * to *always* stop the server if we're running it and we got - * disconnected. It's a big hammer, but it's reliable, and arguably if - * *we* couldn't' use *our* server then something bad is happening and - * someone else should be the server. - * - * This only executes on mount, error, or as a connection disconnects - * and there's only ever one executing. + * There's a tricky bit of coordination required to safely unmount. + * Clients need to tell the server that they won't be coming back with a + * farewell request. Once a client receives its farewell response it + * can exit. But a majority of clients need to stick around to elect a + * server to process all their farewell requests. This is coordinated + * by having the greeting tell the server that a client is a voter. The + * server then holds on to farewell requests from voters until only + * requests from the final quorum remain. These farewell responses are + * only sent after updating an unmount barrier in the super to indicate + * to the final quorum that they can safely exit without having received + * a farewell response over the network. */ static void scoutfs_client_connect_worker(struct work_struct *work) { struct client_info *client = container_of(work, struct client_info, - connect_work); + connect_dwork.work); struct super_block *sb = client->sb; - struct scoutfs_quorum_elected_info *qei = &client->qei; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_super_block *super = NULL; struct mount_options *opts = &sbi->opts; + const bool am_voter = opts->server_addr.sin_addr.s_addr != 0; struct scoutfs_net_greeting greet; + struct sockaddr_in sin; ktime_t timeout_abs; + u64 elected_term; int ret; - /* don't try quorum and connecting while our mount runs a server */ - stop_our_server(sb, qei); + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (!super) { + ret = -ENOMEM; + goto out; + } - timeout_abs = ktime_add_ms(ktime_get(), CLIENT_QUORUM_TIMEOUT_MS); - - ret = scoutfs_quorum_election(sb, opts->uniq_name, - client->old_elected_nr, - timeout_abs, client->sending_farewell, - client->greeting_umb, qei); + ret = scoutfs_read_super(sb, super); if (ret) goto out; - /* we saw that the server wrote a new unmount barrier */ - if (client->sending_farewell && qei->elected_nr == 0 && - qei->unmount_barrier > client->greeting_umb) { + /* can safely unmount if we see that server processed our farewell */ + if (am_voter && client->sending_farewell && + (le64_to_cpu(super->unmount_barrier) > client->greeting_umb)) { client->farewell_error = 0; complete(&client->farewell_comp); ret = 0; goto out; } - if (qei->run_server) { - ret = scoutfs_server_start(sb, &qei->sin, qei->elected_nr, qei); - if (ret) { - /* forget that we tried to start the server */ - memset(qei, 0, sizeof(*qei)); + /* try to connect to the super's server address */ + scoutfs_addr_to_sin(&sin, &super->server_addr); + if (sin.sin_addr.s_addr != 0 && sin.sin_port != 0) + ret = scoutfs_net_connect(sb, client->conn, &sin, + CLIENT_CONNECT_TIMEOUT_MS); + else + ret = -ENOTCONN; + + /* voters try to elect a leader if they couldn't connect */ + if (ret < 0) { + /* non-voters will keep retrying */ + if (!am_voter) goto out; - } - } - /* always give the server some time before connecting */ - msleep(CLIENT_CONNECT_DELAY_MS); + /* make sure local server isn't writing super during votes */ + scoutfs_server_stop(sb); - ret = scoutfs_net_connect(sb, client->conn, &qei->sin, - CLIENT_CONNECT_TIMEOUT_MS); - if (ret) { - /* we couldn't connect, try electing a new server */ - client->old_elected_nr = qei->elected_nr; + timeout_abs = ktime_add_ms(ktime_get(), + CLIENT_QUORUM_TIMEOUT_MS); + + ret = scoutfs_quorum_election(sb, timeout_abs, + le64_to_cpu(super->quorum_server_term), + &elected_term); + /* start the server if we were asked to */ + if (elected_term > 0) + ret = scoutfs_server_start(sb, &opts->server_addr, + elected_term); + ret = -ENOTCONN; goto out; } - /* trust this server again if it's still around after we disconnect */ - client->old_elected_nr = 0; - /* send a greeting to verify endpoints of each connection */ - memcpy(greet.name, opts->uniq_name, sizeof(greet.name)); greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; greet.server_term = cpu_to_le64(client->server_term); - greet.unmount_barrier = 0; + greet.unmount_barrier = cpu_to_le64(client->greeting_umb); greet.node_id = cpu_to_le64(sbi->node_id); greet.flags = 0; if (client->sending_farewell) greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL); + if (am_voter) + greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_VOTER); ret = scoutfs_net_submit_request(sb, client->conn, SCOUTFS_NET_CMD_GREETING, @@ -492,8 +480,12 @@ static void scoutfs_client_connect_worker(struct work_struct *work) if (ret) scoutfs_net_shutdown(sb, client->conn); out: + kfree(super); + + /* always have a small delay before retrying to avoid storms */ if (ret && !atomic_read(&client->shutting_down)) - queue_work(client->workq, &client->connect_work); + queue_delayed_work(client->workq, &client->connect_dwork, + msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS)); } /* @@ -566,7 +558,7 @@ static void client_notify_down(struct super_block *sb, struct client_info *client = SCOUTFS_SB(sb)->client_info; if (!atomic_read(&client->shutting_down)) - queue_work(client->workq, &client->connect_work); + queue_delayed_work(client->workq, &client->connect_dwork, 0); } /* @@ -597,7 +589,8 @@ int scoutfs_client_setup(struct super_block *sb) client->sb = sb; init_completion(&client->node_id_comp); atomic_set(&client->shutting_down, 0); - INIT_WORK(&client->connect_work, scoutfs_client_connect_worker); + INIT_DELAYED_WORK(&client->connect_dwork, + scoutfs_client_connect_worker); init_completion(&client->farewell_comp); client->conn = scoutfs_net_alloc_conn(sb, NULL, client_notify_down, 0, @@ -613,7 +606,7 @@ int scoutfs_client_setup(struct super_block *sb) goto out; } - queue_work(client->workq, &client->connect_work); + queue_delayed_work(client->workq, &client->connect_dwork, 0); ret = 0; out: @@ -693,16 +686,13 @@ void scoutfs_client_destroy(struct super_block *sb) atomic_set(&client->shutting_down, 1); /* make sure worker isn't using the conn */ - cancel_work_sync(&client->connect_work); + cancel_delayed_work_sync(&client->connect_dwork); /* make racing conn use explode */ conn = client->conn; client->conn = NULL; scoutfs_net_free_conn(sb, conn); - /* stop running the server if we were, harmless otherwise */ - stop_our_server(sb, &client->qei); - if (client->workq) destroy_workqueue(client->workq); kfree(client); diff --git a/kmod/src/counters.h b/kmod/src/counters.h index a35df40b..60ea6ee9 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -117,18 +117,19 @@ EXPAND_COUNTER(net_recv_invalid_message) \ EXPAND_COUNTER(net_recv_messages) \ EXPAND_COUNTER(net_unknown_request) \ - EXPAND_COUNTER(quorum_elected) \ - EXPAND_COUNTER(quorum_election_error) \ - EXPAND_COUNTER(quorum_fenced) \ - EXPAND_COUNTER(quorum_found_leader) \ - EXPAND_COUNTER(quorum_no_leader) \ + EXPAND_COUNTER(quorum_cycle) \ + EXPAND_COUNTER(quorum_elected_leader) \ + EXPAND_COUNTER(quorum_election_timeout) \ + EXPAND_COUNTER(quorum_failure) \ + EXPAND_COUNTER(quorum_new_leader) \ EXPAND_COUNTER(quorum_read_block) \ EXPAND_COUNTER(quorum_read_block_error) \ EXPAND_COUNTER(quorum_read_invalid_block) \ - EXPAND_COUNTER(quorum_read_invalid_config) \ - EXPAND_COUNTER(quorum_waited) \ + EXPAND_COUNTER(quorum_saw_super_leader) \ + EXPAND_COUNTER(quorum_timedout) \ EXPAND_COUNTER(quorum_write_block) \ EXPAND_COUNTER(quorum_write_block_error) \ + EXPAND_COUNTER(quorum_fenced) \ EXPAND_COUNTER(seg_alloc) \ EXPAND_COUNTER(seg_csum_error) \ EXPAND_COUNTER(seg_free) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index 68396c19..c2f2fa15 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -42,11 +42,14 @@ /* * A reasonably large region of aligned quorum blocks follow the super - * block. + * block. Each voting cycle reads the entire region so we don't want it + * to be too enormous. 256K seems like a reasonably chunky single IO. + * The number of blocks in the region also determines the number of + * mounts that have a reasonable probability of not overwriting each + * other's random block locations. */ -#define SCOUTFS_QUORUM_BLKNO ((128ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) -#define SCOUTFS_QUORUM_BLOCKS ((128ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) -#define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS +#define SCOUTFS_QUORUM_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_QUORUM_BLOCKS ((256ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */ @@ -304,9 +307,10 @@ struct scoutfs_mounted_client_btree_key { } __packed; struct scoutfs_mounted_client_btree_val { - __u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; + __u8 flags; } __packed; +#define SCOUTFS_MOUNTED_CLIENT_VOTER (1 << 0) /* * The max number of links defines the max number of entries that we can @@ -421,70 +425,47 @@ struct scoutfs_xattr { #define SCOUTFS_UUID_BYTES 16 /* - * During each quorum voting interval the fabric has to process 2 reads - * and a write for each voting mount. The only reason we limit the - * number of active quorum mounts is to limit the number of IOs per - * interval. We use a pretty conservative interval given that IOs will - * generally be faster than our constant and we'll have fewer active - * than the max. + * Mounts read all the quorum blocks and write to one random quorum + * block during a cycle. The min cycle time limits the per-mount iop + * load during elections. The random cycle delay makes it less likely + * that mounts will read and write at the same time and miss each + * other's writes. An election only completes if a quorum of mounts + * vote for a leader before any of their elections timeout. This is + * made less likely by the probability that mounts will overwrite each + * others random block locations. The max quorum count limits that + * probability. 9 mounts only have a 55% chance of writing to unique 4k + * blocks in a 256k region. The election timeout is set to include + * enough cycles to usually complete the election. Once a leader is + * elected it spends a number of cycles writing out blocks with itself + * logged as a leader. This reduces the possibility that servers + * will have their log entries overwritten and not be fenced. */ -#define SCOUTFS_QUORUM_MAX_ACTIVE 7 -#define SCOUTFS_QUORUM_IO_LATENCY_MS 10 -#define SCOUTFS_QUORUM_INTERVAL_MS \ - (SCOUTFS_QUORUM_MAX_ACTIVE * 3 * SCOUTFS_QUORUM_IO_LATENCY_MS) +#define SCOUTFS_QUORUM_MAX_COUNT 9 +#define SCOUTFS_QUORUM_CYCLE_LO_MS 10 +#define SCOUTFS_QUORUM_CYCLE_HI_MS 20 +#define SCOUTFS_QUORUM_TERM_LO_MS 250 +#define SCOUTFS_QUORUM_TERM_HI_MS 500 +#define SCOUTFS_QUORUM_ELECTED_LOG_CYCLES 10 -/* - * Each mount that is found in the quorum config in the super block can - * write to quorum blocks indicating which mount they vote for as - * the leader. - * - * @config_gen: references the config gen in the super block - * @write_nr: incremented for every write, only 0 when never written - * @elected_nr: incremented when elected, 0 otherwise - * @unmount_barrier: incremented by servers when all members have unmounted - * @vote_slot: the active config slot that the writer is voting for - */ struct scoutfs_quorum_block { __le64 fsid; __le64 blkno; - __le64 config_gen; + __le64 term; __le64 write_nr; - __le64 elected_nr; - __le64 unmount_barrier; + __le64 voter_rid; + __le64 vote_for_rid; __le32 crc; - __u8 vote_slot; - __u8 flags; -} __packed; - -#define SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED (1 << 0) -#define SCOUTFS_QUORUM_BLOCK_FLAG_LISTENING (1 << 1) -#define SCOUTFS_QUORUM_BLOCK_FLAGS_UNKNOWN (U8_MAX << 2) - -#define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS - -/* - * Each quorum voter is described by a slot which corresponds to the - * block that the voter will write to. - * - * The stale flag is used to support config migration. A new - * configuration is written in free slots and the old configuration is - * marked stale. Stale slots can only be reclaimed once we have - * evidence that the named mount won't try and write to it by seeing it - * write to other slots or connect with the new gen. - */ -struct scoutfs_quorum_config { - __le64 gen; - struct scoutfs_quorum_slot { - __u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; + __u8 log_nr; + struct scoutfs_quorum_log { + __le64 term; + __le64 rid; struct scoutfs_inet_addr addr; - __u8 vote_priority; - __u8 flags; - } __packed slots[SCOUTFS_QUORUM_MAX_SLOTS]; + } __packed log[0]; } __packed; -#define SCOUTFS_QUORUM_SLOT_ACTIVE (1 << 0) -#define SCOUTFS_QUORUM_SLOT_STALE (1 << 1) -#define SCOUTFS_QUORUM_SLOT_FLAGS_UNKNOWN (U8_MAX << 2) +#define SCOUTFS_QUORUM_LOG_MAX \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_quorum_block)) / \ + sizeof(struct scoutfs_quorum_log)) struct scoutfs_super_block { struct scoutfs_block_header hdr; @@ -500,9 +481,13 @@ struct scoutfs_super_block { __le64 next_seg_seq; __le64 next_node_id; __le64 next_compact_id; + __le64 quorum_fenced_term; + __le64 quorum_server_term; + __le64 unmount_barrier; + __u8 quorum_count; + struct scoutfs_inet_addr server_addr; struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; - struct scoutfs_quorum_config quorum_config; struct scoutfs_btree_root lock_clients; struct scoutfs_btree_root trans_seqs; struct scoutfs_btree_root mounted_clients; @@ -624,8 +609,6 @@ enum { * Greetings verify identity of communicating nodes. The sender sends * their credentials and the receiver verifies them. * - * @name: The client sends its unique name to the server. - * * @server_term: The raft term that elected the server. Initially 0 * from the client, sent by the server, then sent by the client as it * tries to reconnect. Used to identify a client reconnecting to a @@ -634,7 +617,7 @@ enum { * @unmount_barrier: Incremented every time the remaining majority of * quorum members all agree to leave. The server tells a quorum member * the value that it's connecting under so that if the client sees the - * value increase in a quorum block it knows that the server has + * value increase in the super block then it knows that the server has * processed its farewell and can safely unmount. * * @node_id: The id of the client. Initially 0 from the client, @@ -643,7 +626,6 @@ enum { * state must be dealt with. */ struct scoutfs_net_greeting { - __u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; __le64 fsid; __le64 format_hash; __le64 server_term; @@ -653,7 +635,8 @@ struct scoutfs_net_greeting { } __packed; #define SCOUTFS_NET_GREETING_FLAG_FAREWELL (1 << 0) -#define SCOUTFS_NET_GREETING_FLAG_INVALID (~(__u64)0 << 1) +#define SCOUTFS_NET_GREETING_FLAG_VOTER (1 << 1) +#define SCOUTFS_NET_GREETING_FLAG_INVALID (~(__u64)0 << 2) /* * This header precedes and describes all network messages sent over diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index 7664aef6..a3e7e6d2 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -33,50 +33,66 @@ #include "scoutfs_trace.h" /* - * scoutfs mounts use a region of statically allocated blocks in the - * shared metadata device to elect a leader mount who runs the server - * that the rest of the mounts of the filesystem connect to. + * scoutfs mounts communicate through a region of preallocated blocks to + * elect a leader who starts the server. Mounts which have been + * configured with a server address and which can't connect to a server + * attempt to form a quorum to elect a new leader who starts a new + * server. * - * Mounts that should participate in the election are configured in an - * array in the super block. Their position in the array determines the - * preallocated block that they'll be writing to. Mounts that aren't - * participating in the election only read the blocks to discover the - * outcome of the election. + * The mounts participating in the election use a variant of the raft + * election protocol to establish quorum and elect a leader. We use + * block reads and writes instead of network messages. Mounts read all + * the blocks looking for messages to receive. Mounts write their vote + * to a random block in the region to send a message to all other + * mounts. Unlikely collisions are analogous to lossy networks losing + * messages and are handled by the protocol. * - * During the election each participating mount reads all the quorum - * blocks that all the mounts wrote, sees which are active and chooses - * which to vote for, and writes a new version of their block that - * includes their vote. Mounts vote for the mount with the highest - * priority in the config that is seen actively writing voting blocks - * over time. + * We allow a "majority" of 1 voter when there are less than three + * possible voters. This lets a simple network establish quorum. If + * the raft quorum timeouts align to leaders could both elect themselves + * and race to fence each other. In the worst case they could continue + * to do this indefinitely but it's unlikely as it would require a + * sequence of identical random raft timeouts. * - * Once a mount receives a majority of votes from its peers then it - * writes its block with an indication that it has been elected. Only - * after reading that block, and seeing no other blocks that indicate - * more recently elected leaders, will it consider itself elected and - * try to fence any other previously elected leaders before starting the - * server. This ensures that racing elected leaders will always result - * in fencing all but the most recent. + * One of the reasons we use block reads and writes as the quorum + * communication medium is that it lets us leave behind a shared + * persistent log of previous election results. This then lets a newly + * elected leader fence all previously elected leaders that haven't + * shutdown so that they can safely assume exclusive access to the + * shared device. Every written block includes a log of election + * results. Every voter merges the log from every block it reads the + * block it writes. A leader doesn't attempt to fence until it's spent + * a few cycles writing blocks with itself as the log entry. This gives + * other voters time to migrate the log entry through the blocks. * - * Once the elected leader verifies its written elected block it tries - * to start up the server. Once it's listening it writes another quorum - * block that indicates that it's listening. Once mounts see that - * they'll try to connect. If the server takes too long to write its - * listening flag the mounts may decide that the leader has died and try - * to elect a new leader. + * Once a leader is elected it fences any previously elected leaders + * still present in the log it merged while reading all the voting + * blocks. Once they've fenced they update the super block record of + * the latest term that has been fenced. This trims the log over time + * and keeps from attempting to fence the same mounts multiple times. + * As the server later shuts down it writes its term into the super to + * stop it from being fenced. * - * XXX: - * - actually fence - * - add temporary priority for choosing a specific mount as a leader - * - add config rotation (write new config, reclaim stale slots) + * The final complication comes during unmount. Clients exit after the + * server responds to their farewell request. But a majority of clients + * need to be present to elect a server to process farewell requests. + * The server knows which clients will attempt to vote for quorum and + * only responds to their farewell requests once they're no longer + * needed to elect a server -- either there's still quorum remaining of + * other mounts or the only mounts remaining are all quorum voters that + * have sent farewell requests. Before sending these final responses + * the server updates an unmount_barrier field in the super. If clients + * that are waiting for a farewell response see the unmount barrier + * increment they know that their farewell has been processed and they + * can assume a successful farewell response and exit cleanly. + * + * XXX: - actually fence */ struct quorum_info { struct scoutfs_sysfs_attrs ssa; bool is_leader; - struct sockaddr_in conf_addr; - u16 conf_port; }; #define DECLARE_QUORUM_INFO(sb, name) \ @@ -84,199 +100,21 @@ struct quorum_info { #define DECLARE_QUORUM_INFO_KOBJ(kobj, name) \ DECLARE_QUORUM_INFO(SCOUTFS_SYSFS_ATTRS_SB(kobj), name) -static void addr_to_sin(struct sockaddr_in *sin, struct scoutfs_inet_addr *addr) -{ - sin->sin_family = AF_INET; - sin->sin_addr.s_addr = cpu_to_be32(le32_to_cpu(addr->addr)); - sin->sin_port = cpu_to_be16(le16_to_cpu(addr->port)); -} - -/* active slots are sorted to the front for validation */ -static int cmp_slot_active(const struct scoutfs_quorum_slot *a, - const struct scoutfs_quorum_slot *b) -{ - int a_active = !!(a->flags & SCOUTFS_QUORUM_SLOT_ACTIVE); - int b_active = !!(b->flags & SCOUTFS_QUORUM_SLOT_ACTIVE); - - return b_active - a_active; -} - -/* slot validation has ensured that the names are null terminated */ -static int cmp_slot_names(const void *A, const void *B) -{ - const struct scoutfs_quorum_slot *a = A; - const struct scoutfs_quorum_slot *b = B; - - return cmp_slot_active(a, b) ?: - strcmp(a->name, b->name); -} - -static int cmp_slot_addrs(const void *A, const void *B) -{ - const struct scoutfs_quorum_slot *a = A; - const struct scoutfs_quorum_slot *b = B; - - return cmp_slot_active(a, b) ?: - memcmp(&a->addr, &b->addr, sizeof(a->addr)); -} - -static void swap_slots(void *A, void *B, int size) -{ - struct scoutfs_quorum_slot *a = A; - struct scoutfs_quorum_slot *b = B; - - swap(*a, *b); -} - /* - * We'll set the callers our_slot to the slot that contains the their. - * If the name isn't found then it'll be set to -1. + * Return an absolute ktime timeout expires value in the future after a + * random duration between hi and lo where both limits are possible. */ -static int read_quorum_config(struct super_block *sb, - struct scoutfs_super_block *super, - char *our_name, int *our_slot_ret, - int *nr_active_ret) +static ktime_t random_to(u32 lo, u32 hi) { - struct scoutfs_quorum_slot *sorted = NULL; - struct scoutfs_quorum_slot *slot; - struct scoutfs_quorum_config *conf; - struct sockaddr_in sin; - int nr_active = 0; - int our_slot = -1; - int ret; - int i; - - sorted = kcalloc(SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), GFP_NOFS); - if (sorted == NULL) { - ret = -ENOMEM; - goto out; - } - - ret = scoutfs_read_super(sb, super); - if (ret) - goto out; - conf = &super->quorum_config; - - ret = -EINVAL; - - if (conf->gen == 0) { - scoutfs_err(sb, "invalid zero quorum config gen"); - goto out; - } - - for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - slot = &conf->slots[i]; - - if (slot->flags & SCOUTFS_QUORUM_SLOT_FLAGS_UNKNOWN) { - scoutfs_err(sb, "quorum slot ind %u unknown flags 0x%02x", - i, slot->flags); - goto out; - } - - if ((slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE) && - (slot->flags & SCOUTFS_QUORUM_SLOT_STALE)) { - scoutfs_err(sb, "quorum slot ind %u is both active and stale", - i); - goto out; - } - - if (!(slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE)) - continue; - - nr_active++; - - if (slot->name[0] == '\0') { - scoutfs_err(sb, "quorum slot ind %u name is null", i); - goto out; - } - - if (slot->name[SCOUTFS_UNIQUE_NAME_MAX_BYTES - 1] != '\0') { - scoutfs_err(sb, "quorum slot ind %u name isn't null terminated", - i); - goto out; - } - - if (our_name && strcmp(our_name, slot->name) == 0) - our_slot = i; - - addr_to_sin(&sin, &slot->addr); - - if (ipv4_is_multicast(sin.sin_addr.s_addr) || - ipv4_is_lbcast(sin.sin_addr.s_addr) || - ipv4_is_zeronet(sin.sin_addr.s_addr) || - ipv4_is_local_multicast(sin.sin_addr.s_addr) || - ntohs(sin.sin_port) == 0 || - ntohs(sin.sin_port) == U16_MAX) { - scoutfs_err(sb, "quorum slot ind %u has invalid addr %pIS:%u", - i, &sin, ntohs(sin.sin_port)); - goto out; - } - } - - if (nr_active == 0) { - scoutfs_err(sb, "quorum config has no active slots"); - goto out; - } - - if (nr_active > SCOUTFS_QUORUM_MAX_ACTIVE) { - scoutfs_err(sb, "quorum config has %u active slots, can have at most %u ", - nr_active, SCOUTFS_QUORUM_MAX_ACTIVE); - goto out; - } - - memcpy(sorted, conf->slots, - SCOUTFS_QUORUM_MAX_SLOTS * sizeof(sorted[0])); - - sort(sorted, SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), - cmp_slot_names, swap_slots); - - for (i = 1; i < nr_active; i++) { - if (strcmp(sorted[i].name, sorted[i - 1].name) == 0) { - scoutfs_err(sb, "multiple quorum slots have the same name '%s'", - sorted[i].name); - goto out; - } - } - - sort(sorted, SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), - cmp_slot_addrs, swap_slots); - - for (i = 1; i < nr_active; i++) { - if (memcmp(&sorted[i].addr, &sorted[i - 1].addr, - sizeof(sorted[i].addr)) == 0) { - addr_to_sin(&sin, &sorted[i].addr); - scoutfs_err(sb, "multiple quorum slots have the same address %pIS:%u", - &sin, ntohs(sin.sin_port)); - goto out; - } - } - - ret = 0; - if (our_slot_ret) - *our_slot_ret = our_slot; - if (nr_active_ret) - *nr_active_ret = nr_active; -out: - if (ret) - scoutfs_inc_counter(sb, quorum_read_invalid_config); - kfree(sorted); - return ret; + return ktime_add_ms(ktime_get(), lo + prandom_u32_max((hi + 1) - lo)); } -enum { - BH_ScoutfsTraced = BH_PrivateStart, -}; - -BUFFER_FNS(ScoutfsTraced, scoutfs_traced) /* has been traced */ -TAS_BUFFER_FNS(ScoutfsTraced, scoutfs_traced) - /* - * The caller is about to read the current version of a set of quorum - * blocks. We invalidate all the quorum blocks in the cache and - * populate the cache with all the blocks with one large contiguous - * read. The caller then uses simple sync bh methods to access - * whichever blocks it needs. I'm not a huge fan of the plug but I - * couldn't get the individual readahead requests merged without it. + * The caller is about to read all the quorum blocks. We invalidate any + * cached blocks and issue one large contiguous read to repopulate the + * cache. The caller then uses normal sb_bread to read each block. I'm + * not a huge fan of the plug but I couldn't get the individual + * readahead requests merged without it. */ static void readahead_quorum_blocks(struct super_block *sb) { @@ -293,7 +131,6 @@ static void readahead_quorum_blocks(struct super_block *sb) lock_buffer(bh); clear_buffer_uptodate(bh); - clear_buffer_scoutfs_traced(bh); unlock_buffer(bh); ll_rw_block(READA | REQ_META | REQ_PRIO, 1, &bh); @@ -303,6 +140,25 @@ static void readahead_quorum_blocks(struct super_block *sb) blk_finish_plug(&plug); } +struct quorum_block_head { + struct list_head head; + union { + struct scoutfs_quorum_block blk; + u8 bytes[SCOUTFS_BLOCK_SIZE]; + }; +}; + +static void free_quorum_blocks(struct list_head *blocks) +{ + struct quorum_block_head *qbh; + struct quorum_block_head *tmp; + + list_for_each_entry_safe(qbh, tmp, blocks, head) { + list_del_init(&qbh->head); + kfree(qbh); + } +} + /* * Callers don't mind us clobbering the crc temporarily. */ @@ -319,139 +175,149 @@ static __le32 quorum_block_crc(struct scoutfs_quorum_block *blk) return calc_crc; } -static bool invalid_quorum_block(struct scoutfs_super_block *super, - struct buffer_head *bh, +static size_t quorum_block_bytes(struct scoutfs_quorum_block *blk) +{ + return offsetof(struct scoutfs_quorum_block, + log[blk->log_nr]); +} + +static bool invalid_quorum_block(struct buffer_head *bh, struct scoutfs_quorum_block *blk) { - return quorum_block_crc(blk) != blk->crc || - blk->fsid != super->hdr.fsid || + return bh->b_size != SCOUTFS_BLOCK_SIZE || + sizeof(struct scoutfs_quorum_block) > SCOUTFS_BLOCK_SIZE || + quorum_block_crc(blk) != blk->crc || le64_to_cpu(blk->blkno) != bh->b_blocknr || - blk->vote_slot >= SCOUTFS_QUORUM_MAX_SLOTS || - (blk->flags & SCOUTFS_QUORUM_BLOCK_FLAGS_UNKNOWN); + blk->term == 0 || + blk->log_nr > SCOUTFS_QUORUM_LOG_MAX || + quorum_block_bytes(blk) > SCOUTFS_BLOCK_SIZE; +} + +/* true if a is stale and should be ignored */ +static bool stale_quorum_block(struct scoutfs_quorum_block *a, + struct scoutfs_quorum_block *b) +{ + if (le64_to_cpu(a->term) < le64_to_cpu(b->term)) + return true; + + if (le64_to_cpu(a->voter_rid) == le64_to_cpu(b->voter_rid) && + le64_to_cpu(a->write_nr) <= le64_to_cpu(b->write_nr)) + return true; + + return false; } /* - * Give the caller the most recently updated version of the quorum - * block. Returns 0 and fills the callers block struct on success. - * Returns -ENOENT and zeros the caller's block if we couldn't read a - * valid block. We don't consider the config gen, that's up to the - * caller. + * Get the most recent blocks from all the voters for the most recent term. + * We ignore any corrupt blocks, blocks not for our fsid, previous terms, + * and previous writes from a rid in the current term. */ -static int read_quorum_block(struct super_block *sb, - struct scoutfs_super_block *super, int slot, - struct scoutfs_quorum_block *blk_ret) +static int read_quorum_blocks(struct super_block *sb, struct list_head *blocks) { + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_quorum_block *blk; - struct buffer_head *bh; + struct quorum_block_head *qbh; + struct quorum_block_head *tmp; + struct buffer_head *bh = NULL; + LIST_HEAD(stale); int ret; + int i; - /* code strongly assumes that slots and blocks are directly mapped */ - BUILD_BUG_ON(SCOUTFS_QUORUM_BLOCKS != SCOUTFS_QUORUM_MAX_SLOTS); + readahead_quorum_blocks(sb); - ret = -ENOENT; + for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { + brelse(bh); + bh = sb_bread(sb, SCOUTFS_QUORUM_BLKNO + i); + if (!bh) { + scoutfs_inc_counter(sb, quorum_read_block_error); + ret = -EIO; + goto out; + } + blk = (void *)(bh->b_data); - bh = sb_bread(sb, SCOUTFS_QUORUM_BLKNO + slot); - if (!bh) { - scoutfs_inc_counter(sb, quorum_read_block_error); - goto out; - } - blk = (void *)(bh->b_data); + /* ignore unwritten blocks or blocks for other filesystems */ + if (blk->voter_rid == 0 || blk->fsid != super->hdr.fsid) + continue; - /* ignore unwritten blocks */ - if (blk->write_nr == 0) - goto out; + if (invalid_quorum_block(bh, blk)) { + scoutfs_inc_counter(sb, quorum_read_invalid_block); + continue; + } - if (!test_set_buffer_scoutfs_traced(bh)) - trace_scoutfs_quorum_read_block(sb, bh->b_blocknr, blk); + list_for_each_entry_safe(qbh, tmp, blocks, head) { + if (stale_quorum_block(blk, &qbh->blk)) { + blk = NULL; + break; + } - if (invalid_quorum_block(super, bh, blk)) { - scoutfs_inc_counter(sb, quorum_read_invalid_block); - goto out; + if (stale_quorum_block(&qbh->blk, blk)) + list_move(&qbh->head, &stale); + } + free_quorum_blocks(&stale); + + if (!blk) + continue; + + qbh = kmalloc(sizeof(struct quorum_block_head), + GFP_NOFS); + if (!qbh) { + ret = -ENOMEM; + goto out; + } + + memcpy(&qbh->blk, blk, quorum_block_bytes(blk)); + list_add_tail(&qbh->head, blocks); + } + + list_for_each_entry(qbh, blocks, head) { + trace_scoutfs_quorum_read_block(sb, &qbh->blk); + scoutfs_inc_counter(sb, quorum_read_block); } - *blk_ret = *blk; - scoutfs_inc_counter(sb, quorum_read_block); ret = 0; out: - if (ret < 0) - memset(blk_ret, 0, sizeof(struct scoutfs_quorum_block)); brelse(bh); + if (ret < 0) + free_quorum_blocks(blocks); return ret; } -/* - * Iterate over config slots from the given index and return the first - * slot that has any of the given flags set. - */ -static inline int first_slot_flags(struct scoutfs_quorum_config *conf, - int i, u8 flags) -{ - for (; i < ARRAY_SIZE(conf->slots); i++) { - if (conf->slots[i].flags & flags) - break; - } - return i; -} - -/* - * Execute the loop body with the read block for each slot that's - * configured and active. If we can't read the block for whatever - * reason then the loop will execute with the blk struct zeroed. - */ -#define for_each_active_block(sb, super, conf, hists, hi, blk, slot, i) \ - for (i = first_slot_flags(conf, 0, SCOUTFS_QUORUM_SLOT_ACTIVE); \ - (i < ARRAY_SIZE(conf->slots)) && \ - (slot = &conf->slots[i], \ - hi = &hists[i], \ - read_quorum_block(sb, super, i, blk), 1); \ - i = first_slot_flags(conf, i + 1, SCOUTFS_QUORUM_SLOT_ACTIVE)) - -/* - * Iterate over every possible block, regardless of config. A lot of these - * will be zero. - */ -#define for_each_block(sb, super, i, blk) \ - for (i = 0; \ - (i < SCOUTFS_QUORUM_BLOCKS) && \ - (read_quorum_block(sb, super, i, blk), 1); \ - i++) - /* * Synchronously write a single quorum block. The caller has provided - * the meaningful fields for the write. We fill in the rest that are - * consistent for every write and zero the rest of the block. + * the meaningful fields for the write. We fill in the fsid, blkno, and + * crc for every write and zero the rest of the block. */ -static int write_quorum_block(struct super_block *sb, __le64 fsid, - __le64 config_gen, u8 our_slot, __le64 write_nr, - u64 elected_nr, u64 unmount_barrier, - u8 vote_slot, u8 flags) +static int write_quorum_block(struct super_block *sb, + struct scoutfs_quorum_block *our_blk) { + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_quorum_block *blk; - struct buffer_head *bh; + struct buffer_head *bh = NULL; + size_t size; int ret; BUILD_BUG_ON(sizeof(struct scoutfs_quorum_block) > SCOUTFS_BLOCK_SIZE); - if (WARN_ON_ONCE(our_slot >= SCOUTFS_QUORUM_MAX_SLOTS) || - WARN_ON_ONCE(vote_slot >= SCOUTFS_QUORUM_MAX_SLOTS)) - return -EINVAL; - - bh = sb_getblk(sb, SCOUTFS_QUORUM_BLKNO + our_slot); + bh = sb_getblk(sb, SCOUTFS_QUORUM_BLKNO + + prandom_u32_max(SCOUTFS_QUORUM_BLOCKS)); if (bh == NULL) { ret = -EIO; goto out; } + + size = quorum_block_bytes(our_blk); + if (WARN_ON_ONCE(size > SCOUTFS_BLOCK_SIZE || + size > bh->b_size)) { + ret = -EIO; + goto out; + } + blk = (void *)bh->b_data; + memset(blk, 0, bh->b_size); + memcpy(blk, our_blk, size); - blk->fsid = fsid; + blk->fsid = super->hdr.fsid; blk->blkno = cpu_to_le64(bh->b_blocknr); - blk->config_gen = config_gen; - blk->write_nr = write_nr; - blk->elected_nr = cpu_to_le64(elected_nr); - blk->unmount_barrier = cpu_to_le64(unmount_barrier); - blk->vote_slot = vote_slot; - blk->flags = flags; - blk->crc = quorum_block_crc(blk); lock_buffer(bh); @@ -467,7 +333,7 @@ static int write_quorum_block(struct super_block *sb, __le64 fsid, ret = 0; if (ret == 0) { - trace_scoutfs_quorum_write_block(sb, bh->b_blocknr, blk); + trace_scoutfs_quorum_write_block(sb, blk); scoutfs_inc_counter(sb, quorum_write_block); } out: @@ -478,401 +344,384 @@ out: } /* - * The caller read their quorum block which indicated that they were - * elected. We have to fence all other previously elected leaders so - * that we're running the only instance of the server. - * - * Time can pass between all phases of this: reading that we're elected, - * fencing, and writing the quorum block that clears the elected flag of - * those we fenced. - * - * This is always safe because we either have exclusive access to the - * device having fenced others or someone else would have fenced us - * before they write. + * Returns true if there's an entry for the given election. */ -static int fence_other_elected(struct super_block *sb, - struct scoutfs_super_block *super, - int our_slot, u64 elected_nr) +static bool log_contains(struct scoutfs_quorum_block *blk, u64 term, u64 rid) { - struct scoutfs_quorum_config *conf = &super->quorum_config; - struct scoutfs_quorum_block blk; - u8 flags; - int ret; int i; - for_each_block(sb, super, i, &blk) { - if (i != our_slot && - (blk.flags & SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED) && - le64_to_cpu(blk.elected_nr) <= elected_nr) { - scoutfs_err(sb, "would have fenced"); - scoutfs_inc_counter(sb, quorum_fenced); - - flags = blk.flags & ~SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED; - - ret = write_quorum_block(sb, super->hdr.fsid, - conf->gen, i, blk.write_nr, - le64_to_cpu(blk.elected_nr), - le64_to_cpu(blk.unmount_barrier), i, - flags); - if (ret) - break; - } - } - - return ret; -} - -struct quorum_block_history { - __le64 write_nr; - u8 writing; -}; - -/* - * The caller couldn't connect to a server. Read the quorum blocks - * until we see an elected leader and give their address to the caller. - * If we're configured as part of the quorum then we participate in the - * electing by writing our vote to our quorum block. - * - * Voting members read the blocks at regular intervals and update their - * quorum block with their vote for the elected leader. new leader. - * When a mount receives enough votes it marks its vote in the block as - * elected, fences other elected leaders, and returns to the caller who - * starts up the server for others to connect to. - * - * The calling client may have never seen a server before, or could have - * failed to connect to a valid server, or might have tried to connect - * to a dead server. They pass in an existing elected_nr if they want - * us to ignore old servers and they pass in a timeout so that they can - * return to retrying to connect to whatever address we find. - * - * When we return success we update the caller's elected info with the - * most recent elected leader we found, which may well be long gone. We - * return -ENOENT if we didn't find any elected leaders. - * - * If we return success because we saw a larger unmount barrier we set - * elected_nr to 0 and fill the unmount_barrier. - */ -int scoutfs_quorum_election(struct super_block *sb, char *our_name, - u64 old_elected_nr, ktime_t timeout_abs, - bool unmounting, u64 our_umb, - struct scoutfs_quorum_elected_info *qei) -{ - DECLARE_QUORUM_INFO(sb, qinf); - struct scoutfs_super_block *super = NULL; - struct scoutfs_quorum_config *conf; - struct scoutfs_quorum_slot *slot; - struct scoutfs_quorum_block blk; - struct quorum_block_history *hist; - struct quorum_block_history *hi; - ktime_t expires; - ktime_t now; - __le64 write_nr = 0; - u64 elected_nr = 0; - u64 unmount_barrier = 0; - u8 flags = 0; - int vote_streak = 0; - int vote_slot; - int our_slot; - int vote_prio; - int nr_active; - int nr_votes; - int majority; - int ret; - int i; - - super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); - hist = kcalloc(SCOUTFS_QUORUM_MAX_SLOTS, sizeof(hist[0]), GFP_NOFS); - if (!super || !hist) { - ret = -ENOMEM; - goto out; - } - - for (;;) { - now = ktime_get(); - expires = ktime_add_ms(now, SCOUTFS_QUORUM_INTERVAL_MS); - - ret = read_quorum_config(sb, super, our_name, &our_slot, - &nr_active); - if (ret) - goto out; - conf = &super->quorum_config; - - /* update sysfs with most recently seen config */ - if (our_slot >= 0) { - slot = &conf->slots[our_slot]; - addr_to_sin(&qinf->conf_addr, &slot->addr); - qinf->conf_port = le16_to_cpu(slot->addr.port); - } else { - memset(&qinf->conf_addr, 0, sizeof(qinf->conf_addr)); - qinf->conf_port = 0; - } - - majority = scoutfs_quorum_majority(sb, conf); - - readahead_quorum_blocks(sb); - - /* default to voting for ourselves, but at min prio */ - vote_slot = our_slot; - vote_prio = -1; - memset(qei, 0, sizeof(*qei)); - - for_each_active_block(sb, super, conf, hist, hi, &blk, slot, i){ - /* determine which mounts are writing */ - if (blk.config_gen == conf->gen && - blk.write_nr != 0 && - blk.write_nr != hi->write_nr) - hi->writing = min(hi->writing + 1, 2); - else - hi->writing = 0; - hi->write_nr = blk.write_nr; - - /* vote for first highest priority writing block */ - if (hi->writing >= 2 && - slot->vote_priority > vote_prio) { - vote_slot = i; - vote_prio = slot->vote_priority; - } - - /* find the most recently elected leader */ - if ((blk.config_gen == conf->gen) && - (blk.flags & SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED) && - (le64_to_cpu(blk.elected_nr) > qei->elected_nr)){ - addr_to_sin(&qei->sin, &slot->addr); - qei->config_gen = blk.config_gen; - qei->write_nr = blk.write_nr; - qei->elected_nr = le64_to_cpu(blk.elected_nr); - qei->unmount_barrier = - le64_to_cpu(blk.unmount_barrier); - qei->config_slot = i; - qei->flags = blk.flags; - } - } - - /* - * After writing a block indicating that we were elected - * we make sure that we can read it and that we're still - * the most recent elected leader. If we are then we - * try to fence. If we can't read it, or we're not the - * most recent, or we couldn't fence, then we fall back - * to participating in the election. - */ - if (flags & SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED) { - if (qei->write_nr == write_nr && - qei->elected_nr == elected_nr && - qei->config_slot == our_slot) { - ret = fence_other_elected(sb, super, our_slot, - elected_nr); - if (ret == 0) { - qei->run_server = true; - qinf->is_leader = true; - goto out; - } - - memset(qei, 0, sizeof(*qei)); - } - - vote_streak = 0; - } - - /* return if we found a new listening leader or timed out */ - if (((qei->elected_nr > old_elected_nr) && - (qei->flags & SCOUTFS_QUORUM_BLOCK_FLAG_LISTENING)) || - ktime_after(now, timeout_abs)) { - if (qei->elected_nr > 0) { - scoutfs_inc_counter(sb, quorum_found_leader); - ret = 0; - } else { - scoutfs_inc_counter(sb, quorum_no_leader); - ret = -ENOENT; - } - goto out; - } - - /* wait for the next cycle if we're not in the voting config */ - if (our_slot < 0) - continue; - - nr_votes = 0; - write_nr = cpu_to_le64(1); - elected_nr = 0; - unmount_barrier = 0; - flags = 0; - - for_each_active_block(sb, super, conf, hist, hi, &blk, slot, i){ - /* count our votes (maybe including from us) */ - if (hi->writing >= 2 && blk.vote_slot == our_slot) - nr_votes++; - - /* can finish unmounting if members all left */ - if (unmounting && - le64_to_cpu(blk.unmount_barrier) > our_umb) { - qei->elected_nr = 0; - qei->unmount_barrier = - le64_to_cpu(blk.unmount_barrier); - ret = 0; - goto out; - } - - /* sample existing fields for our write */ - if (i == our_slot) { - write_nr = blk.write_nr; - le64_add_cpu(&write_nr, 1); - } - elected_nr = max(elected_nr, - le64_to_cpu(blk.elected_nr)); - unmount_barrier = max(unmount_barrier, - le64_to_cpu(blk.unmount_barrier)); - } - - - /* elected after sufficient cycles with a majority vote */ - if (nr_votes >= majority) - vote_streak = min(vote_streak + 1, 2); - else - vote_streak = 0; - - if (vote_streak >= 2) { - flags |= SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED; - elected_nr++; - } - - write_quorum_block(sb, super->hdr.fsid, conf->gen, our_slot, - write_nr, elected_nr, unmount_barrier, - vote_slot, flags); - - set_current_state(TASK_UNINTERRUPTIBLE); - schedule_hrtimeout(&expires, HRTIMER_MODE_ABS); - scoutfs_inc_counter(sb, quorum_waited); - } - -out: - kfree(super); - kfree(hist); - - if (ret) { - memset(qei, 0, sizeof(*qei)); - scoutfs_inc_counter(sb, quorum_election_error); - } - - return ret; -} - -/* - * The calling server has successfully started and is listening for - * collections. It writes a new block to communicate to the other - * mounts that they should now try to connect. We do increase the write_nr - * here to still indicate that we're alive. - */ -int scoutfs_quorum_set_listening(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - - qei->flags |= SCOUTFS_QUORUM_BLOCK_FLAG_LISTENING; - le64_add_cpu(&qei->write_nr, 1); - - return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, - qei->config_slot, qei->write_nr, - qei->elected_nr, qei->unmount_barrier, - qei->config_slot, qei->flags); -} - -/* - * The calling server is shutting down and has finished modifying - * persistent state. We clear the elected flag from our quorum block so - * that mounts won't try to connect and so that the next next leader - * won't try to fence. - * - * By definition nothing has written to the slot since we wrote our - * elected quorum block and the slot could not have been reclaimed. To - * reclaim the slot would have required proving that we were gone or - * fencing us. - * - * If this fails then the mount is in trouble because it'll probably be - * fenced by the next elected leader. - * - * XXX I think there's an interesting race here. If the server is - * running in an old config then the server's slot can be reclaimed if - * the server sees a connection from the current gen. If the server is - * taking a client connection as an indication that the slot won't be - * written then the client needs to shut down the server before trying - * to connect with a new gen. - */ -int scoutfs_quorum_clear_elected(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - DECLARE_QUORUM_INFO(sb, qinf); - - qei->flags &= ~SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED; - qinf->is_leader = false; - - return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, - qei->config_slot, qei->write_nr, - qei->elected_nr, qei->unmount_barrier, - qei->config_slot, qei->flags); -} - -int scoutfs_quorum_update_barrier(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei, - u64 unmount_barrier) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - - qei->unmount_barrier = unmount_barrier; - - return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, - qei->config_slot, qei->write_nr, - qei->elected_nr, qei->unmount_barrier, - qei->config_slot, qei->flags); -} - -/* - * If there's only one or two active slots then a single vote is sufficient - * for a majority. - */ -int scoutfs_quorum_majority(struct super_block *sb, - struct scoutfs_quorum_config *conf) -{ - struct scoutfs_quorum_slot *slot; - int nr_active = 0; - int majority; - int i; - - for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - slot = &conf->slots[i]; - - if (slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE) - nr_active++; - } - - if (nr_active <= 2) - majority = 1; - else if (nr_active & 1) - majority = (nr_active + 1) / 2; - else - majority = (nr_active / 2) + 1; - - return majority; -} - -bool scoutfs_quorum_voting_member(struct super_block *sb, - struct scoutfs_quorum_config *conf, - char *name) -{ - struct scoutfs_quorum_slot *slot; - int i; - - for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - slot = &conf->slots[i]; - - if (strcmp(slot->name, name) == 0) + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].term) == term && + le64_to_cpu(blk->log[i].rid) == rid) return true; } return false; } +/* add an entry to the log, returning error if it's full */ +static int log_add(struct scoutfs_quorum_block *blk, u64 term, u64 rid, + struct scoutfs_inet_addr *addr) +{ + int i; + + if (log_contains(blk, term, rid)) + return 0; + + if (blk->log_nr == SCOUTFS_QUORUM_LOG_MAX) + return -ENOSPC; + + i = blk->log_nr++; + blk->log[i].term = cpu_to_le64(term); + blk->log[i].rid = cpu_to_le64(rid); + blk->log[i].addr = *addr; + + return 0; +} + +/* migrate live log entries between blocks, returning err if full */ +static int log_merge(struct scoutfs_quorum_block *our_blk, + struct scoutfs_quorum_block *blk, + u64 fenced_term) +{ + int ret; + int i; + + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].term) > fenced_term) { + ret = log_add(our_blk, le64_to_cpu(blk->log[i].term), + le64_to_cpu(blk->log[i].rid), + &blk->log[i].addr); + if (ret < 0) + return ret; + } + } + + return 0; +} + +/* Remove old log entries for a voter before a given term. */ +static void log_purge(struct scoutfs_quorum_block *blk, u64 term, u64 rid) +{ + int i; + + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].term) < term && + le64_to_cpu(blk->log[i].rid) == rid) { + if (i != blk->log_nr - 1) + swap(blk->log[i], blk->log[blk->log_nr - 1]); + blk->log_nr--; + i--; /* continue from swapped in entry */ + } + } +} + + +/* + * The caller received a majority of votes and has been elected. Before + * assuming exclusive write access to the device we fence the winners of + * any previous elections still present in the log. Once they're fenced + * we re-read the super and update the fenced_term to indicate that + * those previous elections can be ignored and purged from the log. + * + * We can be attempting this concurrently with both previous and future + * elected leaders. The leader with the greatest elected term will win + * and fence all previous elected leaders. + * + * We clobber the caller's block as we go to not fence rids multiple times. + */ +static int fence_previous(struct super_block *sb, + struct scoutfs_quorum_block *blk, + u64 our_rid, u64 fenced_term, u64 term) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct sockaddr_in their_sin; + int ret; + int i; + + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].rid) != our_rid && + le64_to_cpu(blk->log[i].term) > fenced_term && + le64_to_cpu(blk->log[i].term) < term) { + + scoutfs_inc_counter(sb, quorum_fenced); + scoutfs_addr_to_sin(&their_sin, &blk->log[i].addr); + scoutfs_err(sb, "fencing "SCSBF" at "SIN_FMT, + SCSB_LEFR_ARGS(super->hdr.fsid, + blk->log[i].rid), + SIN_ARG(&their_sin)); + + log_purge(blk, term, le64_to_cpu(blk->log[i].rid)); + i = -1; /* start over */ + } + } + + /* update fenced term now that we have exclusive access */ + ret = 0; + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (super) { + ret = scoutfs_read_super(sb, super); + if (ret == 0) { + super->quorum_fenced_term = cpu_to_le64(term - 1); + ret = scoutfs_write_super(sb, super); + + } + kfree(super); + } else { + ret = -ENOMEM; + } + + if (ret != 0) { + scoutfs_err(sb, "failed to update fenced_term in super, this mount will probably be fenced"); + } + + return ret; +} + + + +/* + * The calling voting mount couldn't connect to a server. Participate + * in a raft election to chose a mount to start a new server. If a + * majority of other mounts join us then one of us will be elected and + * our caller will start the server. + * + * Voting members read the blocks at regular intervals. If they see a + * new election they vote for that candidate for the remainder of the + * election. If the election timeout expires they will start a new + * election and vote for themselves. Eventually a sufficient majority + * sees a new election and all vote in the majority for that candidate. + * + * The calling client may have just failed to connect to an elected + * address in the super block. We assume that server is dead and ignore + * it when trying to elect a new leader. But we eventually return with + * a timeout because the server could actually be fine and the client + * could have had communication to the server restored. + * + * We return success if we see a new server elected. If we are elected + * we set the caller's elected_term so they know to start the server. + */ +int scoutfs_quorum_election(struct super_block *sb, ktime_t timeout_abs, + u64 prev_term, u64 *elected_term) +{ + DECLARE_QUORUM_INFO(sb, qinf); + struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = NULL; + struct scoutfs_quorum_block *our_blk = NULL; + struct scoutfs_quorum_block *blk; + struct quorum_block_head *qbh; + struct scoutfs_inet_addr addr; + enum { VOTER, CANDIDATE }; + ktime_t cycle_to; + ktime_t term_to; + LIST_HEAD(blocks); + u64 vote_for_write_nr; + u64 vote_for_rid; + u64 write_nr; + u64 term; + int log_cycles = 0; + int votes; + int role; + int ret; + + *elected_term = 0; + + trace_scoutfs_quorum_election(sb, prev_term); + + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + our_blk = kmalloc(SCOUTFS_BLOCK_SIZE, GFP_NOFS); + if (!super || !our_blk) { + ret = -ENOMEM; + goto out; + } + + /* start out as a passive voter */ + role = VOTER; + term = 0; + write_nr = 0; + vote_for_rid = 0; + vote_for_write_nr = 0; + + /* we'll become a candidate if we don't see another candidate */ + term_to = random_to(SCOUTFS_QUORUM_TERM_LO_MS, + SCOUTFS_QUORUM_TERM_HI_MS); + + for (;;) { + memset(our_blk, 0, SCOUTFS_BLOCK_SIZE); + + scoutfs_inc_counter(sb, quorum_cycle); + + ret = scoutfs_read_super(sb, super); + if (ret) + goto out; + + /* done if we see evidence of a new server */ + if (le64_to_cpu(super->quorum_server_term) > prev_term) { + scoutfs_inc_counter(sb, quorum_saw_super_leader); + ret = 0; + goto out; + } + + /* done if we couldn't elect anyone */ + if (ktime_after(ktime_get(), timeout_abs)) { + scoutfs_inc_counter(sb, quorum_timedout); + ret = -ETIMEDOUT; + goto out; + } + + /* become a candidate if the election times out */ + if (ktime_after(ktime_get(), term_to)) { + scoutfs_inc_counter(sb, quorum_election_timeout); + term_to = random_to(SCOUTFS_QUORUM_TERM_LO_MS, + SCOUTFS_QUORUM_TERM_HI_MS); + role = CANDIDATE; + term++; + vote_for_rid = sbi->rid; + log_cycles = 0; + } + + free_quorum_blocks(&blocks); + ret = read_quorum_blocks(sb, &blocks); + if (ret < 0) + goto out; + + votes = 0; + + list_for_each_entry(qbh, &blocks, head) { + blk = &qbh->blk; + + /* + * Become a voter for a candidate the first time + * we see a new term. + * + * And also if we're a candidate and see a + * higher rid candidate in our term. This + * minimizes instability when two quorums are + * possible and race to elect two leaders. This + * is only barely reasonable when accepting the + * risk of instability in two mount + * configurations. + */ + if ((le64_to_cpu(blk->term) > term) || + (role == CANDIDATE && + le64_to_cpu(blk->term) == term && + blk->voter_rid == blk->vote_for_rid && + le64_to_cpu(blk->voter_rid) > sbi->rid)) { + role = VOTER; + term = le64_to_cpu(blk->term); + vote_for_rid = le64_to_cpu(blk->vote_for_rid); + vote_for_write_nr = 0; + votes = 0; + log_cycles = 0; + } + + /* candidate writes suppress voter election timers */ + if (role == VOTER && + blk->voter_rid == blk->vote_for_rid && + le64_to_cpu(blk->write_nr) > vote_for_write_nr) { + term_to = random_to(SCOUTFS_QUORUM_TERM_LO_MS, + SCOUTFS_QUORUM_TERM_HI_MS); + vote_for_write_nr = le64_to_cpu(blk->write_nr); + } + + /* count our votes */ + if (role == CANDIDATE && + le64_to_cpu(blk->vote_for_rid) == sbi->rid) { + votes++; + } + + /* try to write greater write_nr */ + write_nr = max(write_nr, le64_to_cpu(blk->write_nr)); + } + + trace_scoutfs_quorum_election_vote(sb, role, term, + vote_for_rid, votes, + log_cycles, + super->quorum_count); + + /* first merge logs from all votes this term */ + list_for_each_entry(qbh, &blocks, head) { + blk = &qbh->blk; + + ret = log_merge(our_blk, blk, + le64_to_cpu(super->quorum_fenced_term)); + if (ret < 0) + goto out; + } + + /* remove logs for voters that can't be servers */ + list_for_each_entry(qbh, &blocks, head) { + blk = &qbh->blk; + + if (blk->voter_rid != blk->vote_for_rid) + log_purge(our_blk, le64_to_cpu(blk->term), + le64_to_cpu(blk->voter_rid)); + } + + /* add ourselves to the log when we see vote quorum */ + if (role == CANDIDATE && votes >= super->quorum_count) { + scoutfs_addr_from_sin(&addr, &opts->server_addr); + ret = log_add(our_blk, term, vote_for_rid, &addr); + if (ret < 0) + goto out; + log_cycles++; /* will be written *this* cycle */ + } + + /* elected candidates can proceed after their log cycles */ + if (role == CANDIDATE && + log_cycles > SCOUTFS_QUORUM_ELECTED_LOG_CYCLES) { + /* our_blk is clobbered */ + ret = fence_previous(sb, our_blk, sbi->rid, + le64_to_cpu(super->quorum_fenced_term), + term); + if (ret < 0) + goto out; + scoutfs_inc_counter(sb, quorum_elected_leader); + qinf->is_leader = true; + *elected_term = term; + goto out; + } + + /* write our block every cycle */ + if (term > 0) { + our_blk->term = cpu_to_le64(term); + write_nr++; + our_blk->write_nr = cpu_to_le64(write_nr); + our_blk->voter_rid = cpu_to_le64(sbi->rid); + our_blk->vote_for_rid = cpu_to_le64(vote_for_rid); + + ret = write_quorum_block(sb, our_blk); + if (ret < 0) + goto out; + } + + /* add a small random delay to each cycle */ + cycle_to = random_to(SCOUTFS_QUORUM_CYCLE_LO_MS, + SCOUTFS_QUORUM_CYCLE_HI_MS); + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_hrtimeout(&cycle_to, HRTIMER_MODE_ABS); + } + +out: + free_quorum_blocks(&blocks); + kfree(super); + kfree(our_blk); + + trace_scoutfs_quorum_election_ret(sb, ret, *elected_term); + if (ret) + scoutfs_inc_counter(sb, quorum_failure); + + return ret; +} + +void scoutfs_quorum_clear_leader(struct super_block *sb) +{ + DECLARE_QUORUM_INFO(sb, qinf); + + qinf->is_leader = false; +} + static ssize_t is_leader_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -882,28 +731,8 @@ static ssize_t is_leader_show(struct kobject *kobj, } SCOUTFS_ATTR_RO(is_leader); -static ssize_t ipv4_addr_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - DECLARE_QUORUM_INFO_KOBJ(kobj, qinf); - - return snprintf(buf, PAGE_SIZE, "%pIS", &qinf->conf_addr); -} -SCOUTFS_ATTR_RO(ipv4_addr); - -static ssize_t ipv4_port_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - DECLARE_QUORUM_INFO_KOBJ(kobj, qinf); - - return snprintf(buf, PAGE_SIZE, "%u", qinf->conf_port); -} -SCOUTFS_ATTR_RO(ipv4_port); - static struct attribute *quorum_attrs[] = { SCOUTFS_ATTR_PTR(is_leader), - SCOUTFS_ATTR_PTR(ipv4_addr), - SCOUTFS_ATTR_PTR(ipv4_port), NULL, }; diff --git a/kmod/src/quorum.h b/kmod/src/quorum.h index cea55525..96eac0e4 100644 --- a/kmod/src/quorum.h +++ b/kmod/src/quorum.h @@ -1,33 +1,9 @@ #ifndef _SCOUTFS_QUORUM_H_ #define _SCOUTFS_QUORUM_H_ -struct scoutfs_quorum_elected_info { - struct sockaddr_in sin; - __le64 config_gen; - __le64 write_nr; - u64 elected_nr; - u64 unmount_barrier; - unsigned int config_slot; - bool run_server; - u8 flags; -}; - -int scoutfs_quorum_election(struct super_block *sb, char *our_name, - u64 old_elected_nr, ktime_t timeout_abs, - bool unmounting, u64 our_umb, - struct scoutfs_quorum_elected_info *qei); -int scoutfs_quorum_set_listening(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei); -int scoutfs_quorum_clear_elected(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei); -int scoutfs_quorum_update_barrier(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei, - u64 unmount_barrier); -int scoutfs_quorum_majority(struct super_block *sb, - struct scoutfs_quorum_config *conf); -bool scoutfs_quorum_voting_member(struct super_block *sb, - struct scoutfs_quorum_config *conf, - char *name); +int scoutfs_quorum_election(struct super_block *sb, ktime_t timeout_abs, + u64 prev_term, u64 *elected_term); +void scoutfs_quorum_clear_leader(struct super_block *sb); int scoutfs_quorum_setup(struct super_block *sb); void scoutfs_quorum_destroy(struct super_block *sb); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 20fc1282..d87dcb2c 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2544,53 +2544,118 @@ TRACE_EVENT(scoutfs_lock_message, __entry->old_mode, __entry->new_mode) ); -DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, - TP_PROTO(struct super_block *sb, u64 io_blkno, - struct scoutfs_quorum_block *blk), - TP_ARGS(sb, io_blkno, blk), +TRACE_EVENT(scoutfs_quorum_election, + TP_PROTO(struct super_block *sb, u64 prev_term), + + TP_ARGS(sb, prev_term), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__u64, io_blkno) - __field(__u64, hdr_blkno) - __field(__u64, config_gen) - __field(__u64, write_nr) - __field(__u64, elected_nr) - __field(__u64, unmount_barrier) - __field(__u32, crc) - __field(__u8, vote_slot) - __field(__u8, flags) + __field(__u64, prev_term) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->io_blkno = io_blkno; - __entry->hdr_blkno = le64_to_cpu(blk->blkno); - __entry->config_gen = le64_to_cpu(blk->config_gen); - __entry->write_nr = le64_to_cpu(blk->write_nr); - __entry->elected_nr = le64_to_cpu(blk->elected_nr); - __entry->unmount_barrier = le64_to_cpu(blk->unmount_barrier); - __entry->crc = le32_to_cpu(blk->crc); - __entry->vote_slot = blk->vote_slot; - __entry->flags = blk->flags; + __entry->prev_term = prev_term; ), - TP_printk(SCSBF" io_blkno %llu hdr_blkno %llu config_gen %llu write_nr %llu elected_nr %llu umb %llu crc 0x%08x vote_slot %u flags %02x", - SCSB_TRACE_ARGS, __entry->io_blkno, __entry->hdr_blkno, - __entry->config_gen, __entry->write_nr, __entry->elected_nr, - __entry->unmount_barrier, __entry->crc, __entry->vote_slot, - __entry->flags) + TP_printk(SCSBF" prev_term %llu", + SCSB_TRACE_ARGS, __entry->prev_term) +); + +TRACE_EVENT(scoutfs_quorum_election_ret, + TP_PROTO(struct super_block *sb, int ret, u64 elected_term), + + TP_ARGS(sb, ret, elected_term), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, ret) + __field(__u64, elected_term) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ret = ret; + __entry->elected_term = elected_term; + ), + + TP_printk(SCSBF" ret %d elected_term %llu", + SCSB_TRACE_ARGS, __entry->ret, __entry->elected_term) +); + +TRACE_EVENT(scoutfs_quorum_election_vote, + TP_PROTO(struct super_block *sb, int role, u64 term, u64 vote_for_rid, + int votes, int log_cycles, int quorum_count), + + TP_ARGS(sb, role, term, vote_for_rid, votes, log_cycles, quorum_count), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, role) + __field(__u64, term) + __field(__u64, vote_for_rid) + __field(int, votes) + __field(int, log_cycles) + __field(int, quorum_count) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->role = role; + __entry->term = term; + __entry->vote_for_rid = vote_for_rid; + __entry->votes = votes; + __entry->log_cycles = log_cycles; + __entry->quorum_count = quorum_count; + ), + + TP_printk(SCSBF" role %d term %llu vote_for_rid %016llx votes %d log_cycles %d quorum_count %d", + SCSB_TRACE_ARGS, __entry->role, __entry->term, + __entry->vote_for_rid, __entry->votes, __entry->log_cycles, + __entry->quorum_count) +); + +DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, + TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk), + + TP_ARGS(sb, blk), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(__u64, term) + __field(__u64, write_nr) + __field(__u64, voter_rid) + __field(__u64, vote_for_rid) + __field(__u32, crc) + __field(__u8, log_nr) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = le64_to_cpu(blk->blkno); + __entry->term = le64_to_cpu(blk->term); + __entry->write_nr = le64_to_cpu(blk->write_nr); + __entry->voter_rid = le64_to_cpu(blk->voter_rid); + __entry->vote_for_rid = le64_to_cpu(blk->vote_for_rid); + __entry->crc = le32_to_cpu(blk->crc); + __entry->log_nr = blk->log_nr; + ), + + TP_printk(SCSBF" blkno %llu term %llu write_nr %llu voter_rid %016llx vote_for_rid %016llx crc 0x%08x log_nr %u", + SCSB_TRACE_ARGS, __entry->blkno, __entry->term, + __entry->write_nr, __entry->voter_rid, __entry->vote_for_rid, + __entry->crc, __entry->log_nr) ); DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_read_block, - TP_PROTO(struct super_block *sb, u64 io_blkno, - struct scoutfs_quorum_block *blk), - TP_ARGS(sb, io_blkno, blk) + TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk), + TP_ARGS(sb, blk) ); DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_write_block, - TP_PROTO(struct super_block *sb, u64 io_blkno, - struct scoutfs_quorum_block *blk), - TP_ARGS(sb, io_blkno, blk) + TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk), + TP_ARGS(sb, blk) ); /* diff --git a/kmod/src/server.c b/kmod/src/server.c index d077d858..291c7df5 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -42,8 +42,8 @@ * connection and accepts connections from all the other mounts acting * as clients. * - * The server is started when raft elects the mount as the leader. If - * it sees errors it shuts down the server in the hopes that another + * The server is started by the mount that is elected leader by quorum. + * If it sees errors it shuts down the server in the hopes that another * mount will become the leader and have less trouble. */ @@ -61,8 +61,6 @@ struct server_info { u64 term; struct scoutfs_net_connection *conn; - struct scoutfs_quorum_elected_info qei; - /* request processing coordinates committing manifest and alloc */ struct rw_semaphore commit_rwsem; struct llist_head commit_waiters; @@ -1185,14 +1183,16 @@ int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, } static int insert_mounted_client(struct super_block *sb, u64 node_id, - char *name) + u64 gr_flags) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_mounted_client_btree_key mck; struct scoutfs_mounted_client_btree_val mcv; mck.node_id = cpu_to_be64(node_id); - strncpy(mcv.name, name, sizeof(mcv.name)); + mcv.flags = 0; + if (gr_flags & SCOUTFS_NET_GREETING_FLAG_VOTER) + mcv.flags |= SCOUTFS_MOUNTED_CLIENT_VOTER; return scoutfs_btree_insert(sb, &super->mounted_clients, &mck, sizeof(mck), &mcv, sizeof(mcv)); @@ -1260,6 +1260,7 @@ static int server_greeting(struct super_block *sb, DECLARE_SERVER_INFO(sb, server); struct commit_waiter cw; __le64 node_id = 0; + __le64 umb = 0; bool sent_node_id; bool first_contact; bool farewell; @@ -1293,10 +1294,12 @@ static int server_greeting(struct super_block *sb, spin_lock(&server->lock); node_id = super->next_node_id; le64_add_cpu(&super->next_node_id, 1); + umb = super->unmount_barrier; spin_unlock(&server->lock); mutex_lock(&server->farewell_mutex); - ret = insert_mounted_client(sb, le64_to_cpu(node_id), gr->name); + ret = insert_mounted_client(sb, le64_to_cpu(node_id), + le64_to_cpu(gr->flags)); mutex_unlock(&server->farewell_mutex); if (ret == 0) @@ -1308,6 +1311,7 @@ static int server_greeting(struct super_block *sb, } } else { node_id = gr->node_id; + umb = gr->unmount_barrier; } send_err: @@ -1315,11 +1319,10 @@ send_err: if (err) node_id = 0; - memset(greet.name, 0, sizeof(greet.name)); greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; greet.server_term = cpu_to_le64(server->term); - greet.unmount_barrier = cpu_to_le64(server->qei.unmount_barrier); + greet.unmount_barrier = umb; greet.node_id = node_id; greet.flags = 0; @@ -1379,31 +1382,20 @@ static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) /* * This work processes farewell requests asynchronously. Requests from - * voting quorum members can be held until they're no longer needed to - * vote for quorum and elect a server to process farewell requests. - * - * This will hold farewell requests from voting clients until either it - * isn't needed for quorum because a majority remains without it, or it - * won't be needed for quorum because all the remaining mounted clients - * are voting and waiting for farewell. + * voting clients can be held until only the final quorum remains and + * they've all sent farewell requests. * * When we remove the last mounted client record for the last voting - * client then we increase the unmount_barrier and write it to the - * server's quorum block. If voting clients don't get their farewell - * response they'll attempt to form quorum again to start the server for - * their farewell response but will find the increased umount_barrier. - * The'll know that their farewell has been processed and they can exit - * without forming quorum. + * client then we increase the unmount_barrier and write it to the super + * block. If voting clients don't get their farewell response they'll + * see the greater umount_barrier in the super and will know that their + * farewell has been processed and that they can exit. * * Responses that are waiting for clients who aren't voting are * immediately sent. Clients that don't have a mounted client record * have already had their farewell processed by another server and can * proceed. * - * This can trust the quorum config found in the super that was read - * when the server started. Only the current server can rewrite the - * working config. - * * Farewell responses are unique in that sending them causes the server * to shutdown the connection to the client next time the socket * disconnects. If the socket is destroyed before the client gets the @@ -1420,7 +1412,6 @@ static void farewell_worker(struct work_struct *work) farewell_work); struct super_block *sb = server->sb; struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_quorum_config *conf = &super->quorum_config; struct scoutfs_mounted_client_btree_key mck; struct scoutfs_mounted_client_btree_val *mcv; struct farewell_request *tmp; @@ -1429,7 +1420,6 @@ static void farewell_worker(struct work_struct *work) struct commit_waiter cw; unsigned int nr_unmounting = 0; unsigned int nr_mounted = 0; - unsigned int majority; LIST_HEAD(reqs); LIST_HEAD(send); bool deleted = false; @@ -1437,8 +1427,6 @@ static void farewell_worker(struct work_struct *work) bool more_reqs; int ret; - majority = scoutfs_quorum_majority(sb, conf); - /* grab all the requests that are waiting */ mutex_lock(&server->farewell_mutex); list_splice_init(&server->farewell_requests, &reqs); @@ -1463,7 +1451,7 @@ static void farewell_worker(struct work_struct *work) } mcv = iref.val; - voting = scoutfs_quorum_voting_member(sb, conf, mcv->name); + voting = (mcv->flags & SCOUTFS_MOUNTED_CLIENT_VOTER) != 0; scoutfs_btree_put_iref(&iref); if (!voting) { @@ -1492,7 +1480,7 @@ static void farewell_worker(struct work_struct *work) memcpy(&mck, iref.key, sizeof(mck)); mcv = iref.val; - if (scoutfs_quorum_voting_member(sb, conf, mcv->name)) + if (mcv->flags & SCOUTFS_MOUNTED_CLIENT_VOTER) nr_mounted++; scoutfs_btree_put_iref(&iref); @@ -1503,7 +1491,8 @@ static void farewell_worker(struct work_struct *work) /* send as many responses as we can to maintain quorum */ while ((fw = list_first_entry_or_null(&reqs, struct farewell_request, entry)) && - (nr_mounted > majority || nr_unmounting >= nr_mounted)) { + (nr_mounted > super->quorum_count || + nr_unmounting >= nr_mounted)) { list_move_tail(&fw->entry, &send); nr_mounted--; @@ -1529,10 +1518,13 @@ static void farewell_worker(struct work_struct *work) goto out; } - /* update the unmount barrier the first time we delete all mounted */ + /* update the unmount barrier if we deleted all voting clients */ if (deleted && nr_mounted == 0) { - ret = scoutfs_quorum_update_barrier(sb, &server->qei, - server->qei.unmount_barrier + 1); + down_read(&server->commit_rwsem); + le64_add_cpu(&super->unmount_barrier, 1); + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + ret = wait_for_commit(&cw); if (ret) goto out; } @@ -2290,9 +2282,14 @@ static void scoutfs_server_worker(struct work_struct *work) struct sockaddr_in sin; LIST_HEAD(conn_list); int ret; + int err; trace_scoutfs_server_work_enter(sb, 0, 0); + sin = server->listen_sin; + + scoutfs_info(sb, "server setting up at "SIN_FMT, SIN_ARG(&sin)); + conn = scoutfs_net_alloc_conn(sb, server_notify_up, server_notify_down, sizeof(struct server_client_info), server_req_funcs, "server"); @@ -2301,8 +2298,6 @@ static void scoutfs_server_worker(struct work_struct *work) goto out; } - sin = server->listen_sin; - ret = scoutfs_net_bind(sb, conn, &sin); if (ret) { scoutfs_err(sb, "server failed to bind to "SIN_FMT", err %d%s", @@ -2312,37 +2307,44 @@ static void scoutfs_server_worker(struct work_struct *work) goto out; } - ret = scoutfs_read_super(sb, super); if (ret) goto out; /* start up the server subsystems before accepting */ - ret = scoutfs_btree_setup(sb) ?: + ret = scoutfs_read_super(sb, super) ?: + scoutfs_btree_setup(sb) ?: scoutfs_manifest_setup(sb) ?: scoutfs_lock_server_setup(sb); if (ret) goto shutdown; - complete(&server->start_comp); + /* + * Write our address in the super before it's possible for net + * processing to start writing the super as part of + * transactions. In theory clients could be trying to connect + * to our address without having seen it in the super (maybe + * they saw it a long time ago). + */ + scoutfs_addr_from_sin(&super->server_addr, &sin); + super->quorum_server_term = cpu_to_le64(server->term); + ret = scoutfs_write_super(sb, super); + if (ret < 0) + goto shutdown; server->stable_manifest_root = super->manifest.root; - scoutfs_info(sb, "server started on "SIN_FMT, SIN_ARG(&sin)); - /* start accepting connections and processing work */ server->conn = conn; scoutfs_net_listen(sb, conn); - ret = scoutfs_quorum_set_listening(sb, &server->qei); + scoutfs_info(sb, "server ready at "SIN_FMT, SIN_ARG(&sin)); + complete(&server->start_comp); - if (ret == 0) { - /* wait_event/wake_up provide barriers */ - wait_event_interruptible(server->waitq, server->shutting_down); - } - - scoutfs_info(sb, "server shutting down on "SIN_FMT, SIN_ARG(&sin)); + /* wait_event/wake_up provide barriers */ + wait_event_interruptible(server->waitq, server->shutting_down); shutdown: + scoutfs_info(sb, "server shutting down at "SIN_FMT, SIN_ARG(&sin)); /* wait for request processing */ scoutfs_net_shutdown(sb, conn); /* drain compact work queued by responses */ @@ -2357,17 +2359,41 @@ shutdown: scoutfs_lock_server_destroy(sb); out: + scoutfs_quorum_clear_leader(sb); scoutfs_net_free_conn(sb, conn); + scoutfs_info(sb, "server stopped at "SIN_FMT, SIN_ARG(&sin)); trace_scoutfs_server_work_exit(sb, 0, ret); + /* + * Always try to clear our presence in the super so that we're + * not fenced. We do this last because other mounts will try to + * reach quorum the moment they see zero here. The later we do + * this the longer we have to finish shutdown while clients + * timeout. + */ + err = scoutfs_read_super(sb, super); + if (err == 0) { + super->quorum_fenced_term = cpu_to_le64(server->term); + memset(&super->server_addr, 0, sizeof(super->server_addr)); + err = scoutfs_write_super(sb, super); + } + if (err < 0) { + scoutfs_err(sb, "failed to clear election term %llu at "SIN_FMT", this mount could be fenced", + server->term, SIN_ARG(&sin)); + } + server->err = ret; complete(&server->start_comp); } -/* XXX can we call start multiple times? */ +/* + * Wait for the server to successfully start. If this returns error then + * the super block's fence_term has been set to the new server's term so + * that it won't be fenced. + */ int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, - u64 term, struct scoutfs_quorum_elected_info *qei) + u64 term) { DECLARE_SERVER_INFO(sb, server); @@ -2375,7 +2401,6 @@ int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, server->shutting_down = false; server->listen_sin = *sin; server->term = term; - server->qei = *qei; init_completion(&server->start_comp); queue_work(server->wq, &server->work); @@ -2398,8 +2423,7 @@ void scoutfs_server_abort(struct super_block *sb) * Once the server is stopped we give the caller our election info * which might have been modified while we were running. */ -void scoutfs_server_stop(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei) +void scoutfs_server_stop(struct super_block *sb) { DECLARE_SERVER_INFO(sb, server); @@ -2407,8 +2431,6 @@ void scoutfs_server_stop(struct super_block *sb, /* XXX not sure both are needed */ cancel_work_sync(&server->work); cancel_work_sync(&server->commit_work); - - *qei = server->qei; } int scoutfs_server_setup(struct super_block *sb) diff --git a/kmod/src/server.h b/kmod/src/server.h index fee6ac0e..32c6ccea 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -74,10 +74,9 @@ int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, struct sockaddr_in; struct scoutfs_quorum_elected_info; int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, - u64 term, struct scoutfs_quorum_elected_info *qei); + u64 term); void scoutfs_server_abort(struct super_block *sb); -void scoutfs_server_stop(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei); +void scoutfs_server_stop(struct super_block *sb); int scoutfs_server_setup(struct super_block *sb); void scoutfs_server_destroy(struct super_block *sb); diff --git a/kmod/src/super.c b/kmod/src/super.c index e65f0c2a..1ac127ff 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -200,7 +200,6 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_shutdown_trans(sb); scoutfs_client_destroy(sb); - scoutfs_quorum_destroy(sb); scoutfs_inode_destroy(sb); /* the server locks the listen address and compacts */ @@ -210,6 +209,9 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_seg_destroy(sb); scoutfs_lock_destroy(sb); + /* server clears quorum leader flag during shutdown */ + scoutfs_quorum_destroy(sb); + scoutfs_item_destroy(sb); scoutfs_destroy_triggers(sb); scoutfs_options_destroy(sb); @@ -319,6 +321,16 @@ int scoutfs_read_super(struct super_block *sb, goto out; } + /* XXX do we want more rigorous invalid super checking? */ + + if (super->quorum_count == 0 || + super->quorum_count > SCOUTFS_QUORUM_MAX_COUNT) { + scoutfs_err(sb, "super block has invalid quorum count %u, must be > 0 and <= %u", + super->quorum_count, SCOUTFS_QUORUM_MAX_COUNT); + ret = -EINVAL; + goto out; + } + *super_res = *super; ret = 0; out: