From 5b258cee3b6d94b6ffaf6b856635b857e333accc Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 3 Jul 2019 17:04:09 -0700 Subject: [PATCH] scoutfs: refine quorum voting The current quorum voting implementatoin had some rough edges that increased the complexity of the system and introduced undesirable failure modes. We can keep the same basic pattern but move functionality around a few places, and rethink the quorum voting, to end up with a meaningfully simpler system. The motivation for this work was to remove the need to provide a uniq_name option for every mount instance. The first big change is to remove the idea of static configuration slots for mounts. This removes the use of uniq_name. Mounts now simply have a server_addr mount option instead of using their uniq_name to find their address in the configuration. The server can't check the configuration to see if a given connected client's name is found in the quorum config. Clients can set a flag in their sent greeting which indicates that they're a voter. This removes the uniq_name from the greeting and mounted client records. Without a static configuration mounts no longer have dedicated block locations to write to. We increase the size of the region of quorum blocks and have voters simply write to a random block. Overwriting vote blocks is OK because we move from heartbeating design patterns to a protocol strongly based on raft's election. We're using quorum blocks to communicate votes instead of network messages and overwriting blocks is analagous to lossy networks droping vote messages in the raft election protocol. We were using the dedicated per-mount quorum blocks to track mounts that had been elected and needed to be fenced. We no longer have that storage so instead we add the idea of an election log that is stored in every voting block. Readers merge the logs from all the blocks they read and write the resulting merged log in their block. With no static quorum configuration we no longer have to worry about the complexity of changing the slot configurations while they're in use. The only persistent configuration is the number of votes a candidate needs to be elected by a quorum. It was a mistake to use quorum voting blocks to communicate state between the server and the quorum voters. We can easily move the unmount_barrier, server address, and fencing state from the quorum blocks into the super block. The server no longer needs the quorum election info struct to be able to later write its quorum block. It instead writes a few fields in the super. There's only one place where clients need to look to find out who they should connect to or if they can finish unmount. Signed-off-by: Zach Brown --- kmod/src/client.c | 158 +++-- kmod/src/counters.h | 15 +- kmod/src/format.h | 113 ++-- kmod/src/quorum.c | 1269 +++++++++++++++++--------------------- kmod/src/quorum.h | 30 +- kmod/src/scoutfs_trace.h | 131 +++- kmod/src/server.c | 136 ++-- kmod/src/server.h | 5 +- kmod/src/super.c | 14 +- 9 files changed, 874 insertions(+), 997 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 8d3bcee6..8ef58589 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -53,10 +53,7 @@ struct client_info { atomic_t shutting_down; struct workqueue_struct *workq; - struct work_struct connect_work; - - struct scoutfs_quorum_elected_info qei; - u64 old_elected_nr; + struct delayed_work connect_dwork; u64 server_term; u64 greeting_umb; @@ -373,117 +370,108 @@ out: } /* - * If the previous election told us to start the server then stop it and - * clear the indication that we were elected. We get the current - * version of the election info from the server because they might have - * modified it while they were running. the old election info. + * This work is responsible for maintaining a connection from the client + * to the server. It's queued on mount and disconnect and we requeue + * the work if the work fails and we're not shutting down. * - * If we're not fast enough to clear the election from the quorum block - * then the next server might fence us. Should be very unlikely as - * election requires multiple RMW cycles. - */ -static void stop_our_server(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei) -{ - if (qei->run_server) { - scoutfs_server_stop(sb, qei); - scoutfs_quorum_clear_elected(sb, qei); - memset(qei, 0, sizeof(*qei)); - } -} - -/* - * This work is responsible for managing leader elections, running the - * server, and connecting clients to the server. - * - * In the typical case a mount reads the quorum blocks and finds the + * In the typical case a mount reads the super blocks and finds the * address of the currently running server and connects to it. + * Non-voting clients who can't connect will keep trying alternating + * reading the address and getting connect timeouts. * - * More rarely clients who aren't connected and are configured to - * participate in quorum need to elect the new leader. The elected info - * filled by quorum tells us if we were elected to run the server. + * Voting mounts will try to elect a leader if they can't connect to the + * server. When a quorum can't connect and are able to elect a leader + * then a new server is started. The new server will write its address + * in the super and everyone will be able to connect. * - * This leads to the possibility that the mount who is running the - * server had its mount disconnect. This is only weirdly different from - * other clients disconnecting and trying to reconnect because of the - * way quorum slots are reconfigured and reclaimed. If we connect to a - * server with the new quorum config then we can't have any old servers - * running in the stale old quorum slot. The simplest way to do this is - * to *always* stop the server if we're running it and we got - * disconnected. It's a big hammer, but it's reliable, and arguably if - * *we* couldn't' use *our* server then something bad is happening and - * someone else should be the server. - * - * This only executes on mount, error, or as a connection disconnects - * and there's only ever one executing. + * There's a tricky bit of coordination required to safely unmount. + * Clients need to tell the server that they won't be coming back with a + * farewell request. Once a client receives its farewell response it + * can exit. But a majority of clients need to stick around to elect a + * server to process all their farewell requests. This is coordinated + * by having the greeting tell the server that a client is a voter. The + * server then holds on to farewell requests from voters until only + * requests from the final quorum remain. These farewell responses are + * only sent after updating an unmount barrier in the super to indicate + * to the final quorum that they can safely exit without having received + * a farewell response over the network. */ static void scoutfs_client_connect_worker(struct work_struct *work) { struct client_info *client = container_of(work, struct client_info, - connect_work); + connect_dwork.work); struct super_block *sb = client->sb; - struct scoutfs_quorum_elected_info *qei = &client->qei; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_super_block *super = NULL; struct mount_options *opts = &sbi->opts; + const bool am_voter = opts->server_addr.sin_addr.s_addr != 0; struct scoutfs_net_greeting greet; + struct sockaddr_in sin; ktime_t timeout_abs; + u64 elected_term; int ret; - /* don't try quorum and connecting while our mount runs a server */ - stop_our_server(sb, qei); + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (!super) { + ret = -ENOMEM; + goto out; + } - timeout_abs = ktime_add_ms(ktime_get(), CLIENT_QUORUM_TIMEOUT_MS); - - ret = scoutfs_quorum_election(sb, opts->uniq_name, - client->old_elected_nr, - timeout_abs, client->sending_farewell, - client->greeting_umb, qei); + ret = scoutfs_read_super(sb, super); if (ret) goto out; - /* we saw that the server wrote a new unmount barrier */ - if (client->sending_farewell && qei->elected_nr == 0 && - qei->unmount_barrier > client->greeting_umb) { + /* can safely unmount if we see that server processed our farewell */ + if (am_voter && client->sending_farewell && + (le64_to_cpu(super->unmount_barrier) > client->greeting_umb)) { client->farewell_error = 0; complete(&client->farewell_comp); ret = 0; goto out; } - if (qei->run_server) { - ret = scoutfs_server_start(sb, &qei->sin, qei->elected_nr, qei); - if (ret) { - /* forget that we tried to start the server */ - memset(qei, 0, sizeof(*qei)); + /* try to connect to the super's server address */ + scoutfs_addr_to_sin(&sin, &super->server_addr); + if (sin.sin_addr.s_addr != 0 && sin.sin_port != 0) + ret = scoutfs_net_connect(sb, client->conn, &sin, + CLIENT_CONNECT_TIMEOUT_MS); + else + ret = -ENOTCONN; + + /* voters try to elect a leader if they couldn't connect */ + if (ret < 0) { + /* non-voters will keep retrying */ + if (!am_voter) goto out; - } - } - /* always give the server some time before connecting */ - msleep(CLIENT_CONNECT_DELAY_MS); + /* make sure local server isn't writing super during votes */ + scoutfs_server_stop(sb); - ret = scoutfs_net_connect(sb, client->conn, &qei->sin, - CLIENT_CONNECT_TIMEOUT_MS); - if (ret) { - /* we couldn't connect, try electing a new server */ - client->old_elected_nr = qei->elected_nr; + timeout_abs = ktime_add_ms(ktime_get(), + CLIENT_QUORUM_TIMEOUT_MS); + + ret = scoutfs_quorum_election(sb, timeout_abs, + le64_to_cpu(super->quorum_server_term), + &elected_term); + /* start the server if we were asked to */ + if (elected_term > 0) + ret = scoutfs_server_start(sb, &opts->server_addr, + elected_term); + ret = -ENOTCONN; goto out; } - /* trust this server again if it's still around after we disconnect */ - client->old_elected_nr = 0; - /* send a greeting to verify endpoints of each connection */ - memcpy(greet.name, opts->uniq_name, sizeof(greet.name)); greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; greet.server_term = cpu_to_le64(client->server_term); - greet.unmount_barrier = 0; + greet.unmount_barrier = cpu_to_le64(client->greeting_umb); greet.node_id = cpu_to_le64(sbi->node_id); greet.flags = 0; if (client->sending_farewell) greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL); + if (am_voter) + greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_VOTER); ret = scoutfs_net_submit_request(sb, client->conn, SCOUTFS_NET_CMD_GREETING, @@ -492,8 +480,12 @@ static void scoutfs_client_connect_worker(struct work_struct *work) if (ret) scoutfs_net_shutdown(sb, client->conn); out: + kfree(super); + + /* always have a small delay before retrying to avoid storms */ if (ret && !atomic_read(&client->shutting_down)) - queue_work(client->workq, &client->connect_work); + queue_delayed_work(client->workq, &client->connect_dwork, + msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS)); } /* @@ -566,7 +558,7 @@ static void client_notify_down(struct super_block *sb, struct client_info *client = SCOUTFS_SB(sb)->client_info; if (!atomic_read(&client->shutting_down)) - queue_work(client->workq, &client->connect_work); + queue_delayed_work(client->workq, &client->connect_dwork, 0); } /* @@ -597,7 +589,8 @@ int scoutfs_client_setup(struct super_block *sb) client->sb = sb; init_completion(&client->node_id_comp); atomic_set(&client->shutting_down, 0); - INIT_WORK(&client->connect_work, scoutfs_client_connect_worker); + INIT_DELAYED_WORK(&client->connect_dwork, + scoutfs_client_connect_worker); init_completion(&client->farewell_comp); client->conn = scoutfs_net_alloc_conn(sb, NULL, client_notify_down, 0, @@ -613,7 +606,7 @@ int scoutfs_client_setup(struct super_block *sb) goto out; } - queue_work(client->workq, &client->connect_work); + queue_delayed_work(client->workq, &client->connect_dwork, 0); ret = 0; out: @@ -693,16 +686,13 @@ void scoutfs_client_destroy(struct super_block *sb) atomic_set(&client->shutting_down, 1); /* make sure worker isn't using the conn */ - cancel_work_sync(&client->connect_work); + cancel_delayed_work_sync(&client->connect_dwork); /* make racing conn use explode */ conn = client->conn; client->conn = NULL; scoutfs_net_free_conn(sb, conn); - /* stop running the server if we were, harmless otherwise */ - stop_our_server(sb, &client->qei); - if (client->workq) destroy_workqueue(client->workq); kfree(client); diff --git a/kmod/src/counters.h b/kmod/src/counters.h index a35df40b..60ea6ee9 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -117,18 +117,19 @@ EXPAND_COUNTER(net_recv_invalid_message) \ EXPAND_COUNTER(net_recv_messages) \ EXPAND_COUNTER(net_unknown_request) \ - EXPAND_COUNTER(quorum_elected) \ - EXPAND_COUNTER(quorum_election_error) \ - EXPAND_COUNTER(quorum_fenced) \ - EXPAND_COUNTER(quorum_found_leader) \ - EXPAND_COUNTER(quorum_no_leader) \ + EXPAND_COUNTER(quorum_cycle) \ + EXPAND_COUNTER(quorum_elected_leader) \ + EXPAND_COUNTER(quorum_election_timeout) \ + EXPAND_COUNTER(quorum_failure) \ + EXPAND_COUNTER(quorum_new_leader) \ EXPAND_COUNTER(quorum_read_block) \ EXPAND_COUNTER(quorum_read_block_error) \ EXPAND_COUNTER(quorum_read_invalid_block) \ - EXPAND_COUNTER(quorum_read_invalid_config) \ - EXPAND_COUNTER(quorum_waited) \ + EXPAND_COUNTER(quorum_saw_super_leader) \ + EXPAND_COUNTER(quorum_timedout) \ EXPAND_COUNTER(quorum_write_block) \ EXPAND_COUNTER(quorum_write_block_error) \ + EXPAND_COUNTER(quorum_fenced) \ EXPAND_COUNTER(seg_alloc) \ EXPAND_COUNTER(seg_csum_error) \ EXPAND_COUNTER(seg_free) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index 68396c19..c2f2fa15 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -42,11 +42,14 @@ /* * A reasonably large region of aligned quorum blocks follow the super - * block. + * block. Each voting cycle reads the entire region so we don't want it + * to be too enormous. 256K seems like a reasonably chunky single IO. + * The number of blocks in the region also determines the number of + * mounts that have a reasonable probability of not overwriting each + * other's random block locations. */ -#define SCOUTFS_QUORUM_BLKNO ((128ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) -#define SCOUTFS_QUORUM_BLOCKS ((128ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) -#define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS +#define SCOUTFS_QUORUM_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_QUORUM_BLOCKS ((256ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */ @@ -304,9 +307,10 @@ struct scoutfs_mounted_client_btree_key { } __packed; struct scoutfs_mounted_client_btree_val { - __u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; + __u8 flags; } __packed; +#define SCOUTFS_MOUNTED_CLIENT_VOTER (1 << 0) /* * The max number of links defines the max number of entries that we can @@ -421,70 +425,47 @@ struct scoutfs_xattr { #define SCOUTFS_UUID_BYTES 16 /* - * During each quorum voting interval the fabric has to process 2 reads - * and a write for each voting mount. The only reason we limit the - * number of active quorum mounts is to limit the number of IOs per - * interval. We use a pretty conservative interval given that IOs will - * generally be faster than our constant and we'll have fewer active - * than the max. + * Mounts read all the quorum blocks and write to one random quorum + * block during a cycle. The min cycle time limits the per-mount iop + * load during elections. The random cycle delay makes it less likely + * that mounts will read and write at the same time and miss each + * other's writes. An election only completes if a quorum of mounts + * vote for a leader before any of their elections timeout. This is + * made less likely by the probability that mounts will overwrite each + * others random block locations. The max quorum count limits that + * probability. 9 mounts only have a 55% chance of writing to unique 4k + * blocks in a 256k region. The election timeout is set to include + * enough cycles to usually complete the election. Once a leader is + * elected it spends a number of cycles writing out blocks with itself + * logged as a leader. This reduces the possibility that servers + * will have their log entries overwritten and not be fenced. */ -#define SCOUTFS_QUORUM_MAX_ACTIVE 7 -#define SCOUTFS_QUORUM_IO_LATENCY_MS 10 -#define SCOUTFS_QUORUM_INTERVAL_MS \ - (SCOUTFS_QUORUM_MAX_ACTIVE * 3 * SCOUTFS_QUORUM_IO_LATENCY_MS) +#define SCOUTFS_QUORUM_MAX_COUNT 9 +#define SCOUTFS_QUORUM_CYCLE_LO_MS 10 +#define SCOUTFS_QUORUM_CYCLE_HI_MS 20 +#define SCOUTFS_QUORUM_TERM_LO_MS 250 +#define SCOUTFS_QUORUM_TERM_HI_MS 500 +#define SCOUTFS_QUORUM_ELECTED_LOG_CYCLES 10 -/* - * Each mount that is found in the quorum config in the super block can - * write to quorum blocks indicating which mount they vote for as - * the leader. - * - * @config_gen: references the config gen in the super block - * @write_nr: incremented for every write, only 0 when never written - * @elected_nr: incremented when elected, 0 otherwise - * @unmount_barrier: incremented by servers when all members have unmounted - * @vote_slot: the active config slot that the writer is voting for - */ struct scoutfs_quorum_block { __le64 fsid; __le64 blkno; - __le64 config_gen; + __le64 term; __le64 write_nr; - __le64 elected_nr; - __le64 unmount_barrier; + __le64 voter_rid; + __le64 vote_for_rid; __le32 crc; - __u8 vote_slot; - __u8 flags; -} __packed; - -#define SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED (1 << 0) -#define SCOUTFS_QUORUM_BLOCK_FLAG_LISTENING (1 << 1) -#define SCOUTFS_QUORUM_BLOCK_FLAGS_UNKNOWN (U8_MAX << 2) - -#define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS - -/* - * Each quorum voter is described by a slot which corresponds to the - * block that the voter will write to. - * - * The stale flag is used to support config migration. A new - * configuration is written in free slots and the old configuration is - * marked stale. Stale slots can only be reclaimed once we have - * evidence that the named mount won't try and write to it by seeing it - * write to other slots or connect with the new gen. - */ -struct scoutfs_quorum_config { - __le64 gen; - struct scoutfs_quorum_slot { - __u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; + __u8 log_nr; + struct scoutfs_quorum_log { + __le64 term; + __le64 rid; struct scoutfs_inet_addr addr; - __u8 vote_priority; - __u8 flags; - } __packed slots[SCOUTFS_QUORUM_MAX_SLOTS]; + } __packed log[0]; } __packed; -#define SCOUTFS_QUORUM_SLOT_ACTIVE (1 << 0) -#define SCOUTFS_QUORUM_SLOT_STALE (1 << 1) -#define SCOUTFS_QUORUM_SLOT_FLAGS_UNKNOWN (U8_MAX << 2) +#define SCOUTFS_QUORUM_LOG_MAX \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_quorum_block)) / \ + sizeof(struct scoutfs_quorum_log)) struct scoutfs_super_block { struct scoutfs_block_header hdr; @@ -500,9 +481,13 @@ struct scoutfs_super_block { __le64 next_seg_seq; __le64 next_node_id; __le64 next_compact_id; + __le64 quorum_fenced_term; + __le64 quorum_server_term; + __le64 unmount_barrier; + __u8 quorum_count; + struct scoutfs_inet_addr server_addr; struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; - struct scoutfs_quorum_config quorum_config; struct scoutfs_btree_root lock_clients; struct scoutfs_btree_root trans_seqs; struct scoutfs_btree_root mounted_clients; @@ -624,8 +609,6 @@ enum { * Greetings verify identity of communicating nodes. The sender sends * their credentials and the receiver verifies them. * - * @name: The client sends its unique name to the server. - * * @server_term: The raft term that elected the server. Initially 0 * from the client, sent by the server, then sent by the client as it * tries to reconnect. Used to identify a client reconnecting to a @@ -634,7 +617,7 @@ enum { * @unmount_barrier: Incremented every time the remaining majority of * quorum members all agree to leave. The server tells a quorum member * the value that it's connecting under so that if the client sees the - * value increase in a quorum block it knows that the server has + * value increase in the super block then it knows that the server has * processed its farewell and can safely unmount. * * @node_id: The id of the client. Initially 0 from the client, @@ -643,7 +626,6 @@ enum { * state must be dealt with. */ struct scoutfs_net_greeting { - __u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; __le64 fsid; __le64 format_hash; __le64 server_term; @@ -653,7 +635,8 @@ struct scoutfs_net_greeting { } __packed; #define SCOUTFS_NET_GREETING_FLAG_FAREWELL (1 << 0) -#define SCOUTFS_NET_GREETING_FLAG_INVALID (~(__u64)0 << 1) +#define SCOUTFS_NET_GREETING_FLAG_VOTER (1 << 1) +#define SCOUTFS_NET_GREETING_FLAG_INVALID (~(__u64)0 << 2) /* * This header precedes and describes all network messages sent over diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index 7664aef6..a3e7e6d2 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -33,50 +33,66 @@ #include "scoutfs_trace.h" /* - * scoutfs mounts use a region of statically allocated blocks in the - * shared metadata device to elect a leader mount who runs the server - * that the rest of the mounts of the filesystem connect to. + * scoutfs mounts communicate through a region of preallocated blocks to + * elect a leader who starts the server. Mounts which have been + * configured with a server address and which can't connect to a server + * attempt to form a quorum to elect a new leader who starts a new + * server. * - * Mounts that should participate in the election are configured in an - * array in the super block. Their position in the array determines the - * preallocated block that they'll be writing to. Mounts that aren't - * participating in the election only read the blocks to discover the - * outcome of the election. + * The mounts participating in the election use a variant of the raft + * election protocol to establish quorum and elect a leader. We use + * block reads and writes instead of network messages. Mounts read all + * the blocks looking for messages to receive. Mounts write their vote + * to a random block in the region to send a message to all other + * mounts. Unlikely collisions are analogous to lossy networks losing + * messages and are handled by the protocol. * - * During the election each participating mount reads all the quorum - * blocks that all the mounts wrote, sees which are active and chooses - * which to vote for, and writes a new version of their block that - * includes their vote. Mounts vote for the mount with the highest - * priority in the config that is seen actively writing voting blocks - * over time. + * We allow a "majority" of 1 voter when there are less than three + * possible voters. This lets a simple network establish quorum. If + * the raft quorum timeouts align to leaders could both elect themselves + * and race to fence each other. In the worst case they could continue + * to do this indefinitely but it's unlikely as it would require a + * sequence of identical random raft timeouts. * - * Once a mount receives a majority of votes from its peers then it - * writes its block with an indication that it has been elected. Only - * after reading that block, and seeing no other blocks that indicate - * more recently elected leaders, will it consider itself elected and - * try to fence any other previously elected leaders before starting the - * server. This ensures that racing elected leaders will always result - * in fencing all but the most recent. + * One of the reasons we use block reads and writes as the quorum + * communication medium is that it lets us leave behind a shared + * persistent log of previous election results. This then lets a newly + * elected leader fence all previously elected leaders that haven't + * shutdown so that they can safely assume exclusive access to the + * shared device. Every written block includes a log of election + * results. Every voter merges the log from every block it reads the + * block it writes. A leader doesn't attempt to fence until it's spent + * a few cycles writing blocks with itself as the log entry. This gives + * other voters time to migrate the log entry through the blocks. * - * Once the elected leader verifies its written elected block it tries - * to start up the server. Once it's listening it writes another quorum - * block that indicates that it's listening. Once mounts see that - * they'll try to connect. If the server takes too long to write its - * listening flag the mounts may decide that the leader has died and try - * to elect a new leader. + * Once a leader is elected it fences any previously elected leaders + * still present in the log it merged while reading all the voting + * blocks. Once they've fenced they update the super block record of + * the latest term that has been fenced. This trims the log over time + * and keeps from attempting to fence the same mounts multiple times. + * As the server later shuts down it writes its term into the super to + * stop it from being fenced. * - * XXX: - * - actually fence - * - add temporary priority for choosing a specific mount as a leader - * - add config rotation (write new config, reclaim stale slots) + * The final complication comes during unmount. Clients exit after the + * server responds to their farewell request. But a majority of clients + * need to be present to elect a server to process farewell requests. + * The server knows which clients will attempt to vote for quorum and + * only responds to their farewell requests once they're no longer + * needed to elect a server -- either there's still quorum remaining of + * other mounts or the only mounts remaining are all quorum voters that + * have sent farewell requests. Before sending these final responses + * the server updates an unmount_barrier field in the super. If clients + * that are waiting for a farewell response see the unmount barrier + * increment they know that their farewell has been processed and they + * can assume a successful farewell response and exit cleanly. + * + * XXX: - actually fence */ struct quorum_info { struct scoutfs_sysfs_attrs ssa; bool is_leader; - struct sockaddr_in conf_addr; - u16 conf_port; }; #define DECLARE_QUORUM_INFO(sb, name) \ @@ -84,199 +100,21 @@ struct quorum_info { #define DECLARE_QUORUM_INFO_KOBJ(kobj, name) \ DECLARE_QUORUM_INFO(SCOUTFS_SYSFS_ATTRS_SB(kobj), name) -static void addr_to_sin(struct sockaddr_in *sin, struct scoutfs_inet_addr *addr) -{ - sin->sin_family = AF_INET; - sin->sin_addr.s_addr = cpu_to_be32(le32_to_cpu(addr->addr)); - sin->sin_port = cpu_to_be16(le16_to_cpu(addr->port)); -} - -/* active slots are sorted to the front for validation */ -static int cmp_slot_active(const struct scoutfs_quorum_slot *a, - const struct scoutfs_quorum_slot *b) -{ - int a_active = !!(a->flags & SCOUTFS_QUORUM_SLOT_ACTIVE); - int b_active = !!(b->flags & SCOUTFS_QUORUM_SLOT_ACTIVE); - - return b_active - a_active; -} - -/* slot validation has ensured that the names are null terminated */ -static int cmp_slot_names(const void *A, const void *B) -{ - const struct scoutfs_quorum_slot *a = A; - const struct scoutfs_quorum_slot *b = B; - - return cmp_slot_active(a, b) ?: - strcmp(a->name, b->name); -} - -static int cmp_slot_addrs(const void *A, const void *B) -{ - const struct scoutfs_quorum_slot *a = A; - const struct scoutfs_quorum_slot *b = B; - - return cmp_slot_active(a, b) ?: - memcmp(&a->addr, &b->addr, sizeof(a->addr)); -} - -static void swap_slots(void *A, void *B, int size) -{ - struct scoutfs_quorum_slot *a = A; - struct scoutfs_quorum_slot *b = B; - - swap(*a, *b); -} - /* - * We'll set the callers our_slot to the slot that contains the their. - * If the name isn't found then it'll be set to -1. + * Return an absolute ktime timeout expires value in the future after a + * random duration between hi and lo where both limits are possible. */ -static int read_quorum_config(struct super_block *sb, - struct scoutfs_super_block *super, - char *our_name, int *our_slot_ret, - int *nr_active_ret) +static ktime_t random_to(u32 lo, u32 hi) { - struct scoutfs_quorum_slot *sorted = NULL; - struct scoutfs_quorum_slot *slot; - struct scoutfs_quorum_config *conf; - struct sockaddr_in sin; - int nr_active = 0; - int our_slot = -1; - int ret; - int i; - - sorted = kcalloc(SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), GFP_NOFS); - if (sorted == NULL) { - ret = -ENOMEM; - goto out; - } - - ret = scoutfs_read_super(sb, super); - if (ret) - goto out; - conf = &super->quorum_config; - - ret = -EINVAL; - - if (conf->gen == 0) { - scoutfs_err(sb, "invalid zero quorum config gen"); - goto out; - } - - for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - slot = &conf->slots[i]; - - if (slot->flags & SCOUTFS_QUORUM_SLOT_FLAGS_UNKNOWN) { - scoutfs_err(sb, "quorum slot ind %u unknown flags 0x%02x", - i, slot->flags); - goto out; - } - - if ((slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE) && - (slot->flags & SCOUTFS_QUORUM_SLOT_STALE)) { - scoutfs_err(sb, "quorum slot ind %u is both active and stale", - i); - goto out; - } - - if (!(slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE)) - continue; - - nr_active++; - - if (slot->name[0] == '\0') { - scoutfs_err(sb, "quorum slot ind %u name is null", i); - goto out; - } - - if (slot->name[SCOUTFS_UNIQUE_NAME_MAX_BYTES - 1] != '\0') { - scoutfs_err(sb, "quorum slot ind %u name isn't null terminated", - i); - goto out; - } - - if (our_name && strcmp(our_name, slot->name) == 0) - our_slot = i; - - addr_to_sin(&sin, &slot->addr); - - if (ipv4_is_multicast(sin.sin_addr.s_addr) || - ipv4_is_lbcast(sin.sin_addr.s_addr) || - ipv4_is_zeronet(sin.sin_addr.s_addr) || - ipv4_is_local_multicast(sin.sin_addr.s_addr) || - ntohs(sin.sin_port) == 0 || - ntohs(sin.sin_port) == U16_MAX) { - scoutfs_err(sb, "quorum slot ind %u has invalid addr %pIS:%u", - i, &sin, ntohs(sin.sin_port)); - goto out; - } - } - - if (nr_active == 0) { - scoutfs_err(sb, "quorum config has no active slots"); - goto out; - } - - if (nr_active > SCOUTFS_QUORUM_MAX_ACTIVE) { - scoutfs_err(sb, "quorum config has %u active slots, can have at most %u ", - nr_active, SCOUTFS_QUORUM_MAX_ACTIVE); - goto out; - } - - memcpy(sorted, conf->slots, - SCOUTFS_QUORUM_MAX_SLOTS * sizeof(sorted[0])); - - sort(sorted, SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), - cmp_slot_names, swap_slots); - - for (i = 1; i < nr_active; i++) { - if (strcmp(sorted[i].name, sorted[i - 1].name) == 0) { - scoutfs_err(sb, "multiple quorum slots have the same name '%s'", - sorted[i].name); - goto out; - } - } - - sort(sorted, SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), - cmp_slot_addrs, swap_slots); - - for (i = 1; i < nr_active; i++) { - if (memcmp(&sorted[i].addr, &sorted[i - 1].addr, - sizeof(sorted[i].addr)) == 0) { - addr_to_sin(&sin, &sorted[i].addr); - scoutfs_err(sb, "multiple quorum slots have the same address %pIS:%u", - &sin, ntohs(sin.sin_port)); - goto out; - } - } - - ret = 0; - if (our_slot_ret) - *our_slot_ret = our_slot; - if (nr_active_ret) - *nr_active_ret = nr_active; -out: - if (ret) - scoutfs_inc_counter(sb, quorum_read_invalid_config); - kfree(sorted); - return ret; + return ktime_add_ms(ktime_get(), lo + prandom_u32_max((hi + 1) - lo)); } -enum { - BH_ScoutfsTraced = BH_PrivateStart, -}; - -BUFFER_FNS(ScoutfsTraced, scoutfs_traced) /* has been traced */ -TAS_BUFFER_FNS(ScoutfsTraced, scoutfs_traced) - /* - * The caller is about to read the current version of a set of quorum - * blocks. We invalidate all the quorum blocks in the cache and - * populate the cache with all the blocks with one large contiguous - * read. The caller then uses simple sync bh methods to access - * whichever blocks it needs. I'm not a huge fan of the plug but I - * couldn't get the individual readahead requests merged without it. + * The caller is about to read all the quorum blocks. We invalidate any + * cached blocks and issue one large contiguous read to repopulate the + * cache. The caller then uses normal sb_bread to read each block. I'm + * not a huge fan of the plug but I couldn't get the individual + * readahead requests merged without it. */ static void readahead_quorum_blocks(struct super_block *sb) { @@ -293,7 +131,6 @@ static void readahead_quorum_blocks(struct super_block *sb) lock_buffer(bh); clear_buffer_uptodate(bh); - clear_buffer_scoutfs_traced(bh); unlock_buffer(bh); ll_rw_block(READA | REQ_META | REQ_PRIO, 1, &bh); @@ -303,6 +140,25 @@ static void readahead_quorum_blocks(struct super_block *sb) blk_finish_plug(&plug); } +struct quorum_block_head { + struct list_head head; + union { + struct scoutfs_quorum_block blk; + u8 bytes[SCOUTFS_BLOCK_SIZE]; + }; +}; + +static void free_quorum_blocks(struct list_head *blocks) +{ + struct quorum_block_head *qbh; + struct quorum_block_head *tmp; + + list_for_each_entry_safe(qbh, tmp, blocks, head) { + list_del_init(&qbh->head); + kfree(qbh); + } +} + /* * Callers don't mind us clobbering the crc temporarily. */ @@ -319,139 +175,149 @@ static __le32 quorum_block_crc(struct scoutfs_quorum_block *blk) return calc_crc; } -static bool invalid_quorum_block(struct scoutfs_super_block *super, - struct buffer_head *bh, +static size_t quorum_block_bytes(struct scoutfs_quorum_block *blk) +{ + return offsetof(struct scoutfs_quorum_block, + log[blk->log_nr]); +} + +static bool invalid_quorum_block(struct buffer_head *bh, struct scoutfs_quorum_block *blk) { - return quorum_block_crc(blk) != blk->crc || - blk->fsid != super->hdr.fsid || + return bh->b_size != SCOUTFS_BLOCK_SIZE || + sizeof(struct scoutfs_quorum_block) > SCOUTFS_BLOCK_SIZE || + quorum_block_crc(blk) != blk->crc || le64_to_cpu(blk->blkno) != bh->b_blocknr || - blk->vote_slot >= SCOUTFS_QUORUM_MAX_SLOTS || - (blk->flags & SCOUTFS_QUORUM_BLOCK_FLAGS_UNKNOWN); + blk->term == 0 || + blk->log_nr > SCOUTFS_QUORUM_LOG_MAX || + quorum_block_bytes(blk) > SCOUTFS_BLOCK_SIZE; +} + +/* true if a is stale and should be ignored */ +static bool stale_quorum_block(struct scoutfs_quorum_block *a, + struct scoutfs_quorum_block *b) +{ + if (le64_to_cpu(a->term) < le64_to_cpu(b->term)) + return true; + + if (le64_to_cpu(a->voter_rid) == le64_to_cpu(b->voter_rid) && + le64_to_cpu(a->write_nr) <= le64_to_cpu(b->write_nr)) + return true; + + return false; } /* - * Give the caller the most recently updated version of the quorum - * block. Returns 0 and fills the callers block struct on success. - * Returns -ENOENT and zeros the caller's block if we couldn't read a - * valid block. We don't consider the config gen, that's up to the - * caller. + * Get the most recent blocks from all the voters for the most recent term. + * We ignore any corrupt blocks, blocks not for our fsid, previous terms, + * and previous writes from a rid in the current term. */ -static int read_quorum_block(struct super_block *sb, - struct scoutfs_super_block *super, int slot, - struct scoutfs_quorum_block *blk_ret) +static int read_quorum_blocks(struct super_block *sb, struct list_head *blocks) { + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_quorum_block *blk; - struct buffer_head *bh; + struct quorum_block_head *qbh; + struct quorum_block_head *tmp; + struct buffer_head *bh = NULL; + LIST_HEAD(stale); int ret; + int i; - /* code strongly assumes that slots and blocks are directly mapped */ - BUILD_BUG_ON(SCOUTFS_QUORUM_BLOCKS != SCOUTFS_QUORUM_MAX_SLOTS); + readahead_quorum_blocks(sb); - ret = -ENOENT; + for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { + brelse(bh); + bh = sb_bread(sb, SCOUTFS_QUORUM_BLKNO + i); + if (!bh) { + scoutfs_inc_counter(sb, quorum_read_block_error); + ret = -EIO; + goto out; + } + blk = (void *)(bh->b_data); - bh = sb_bread(sb, SCOUTFS_QUORUM_BLKNO + slot); - if (!bh) { - scoutfs_inc_counter(sb, quorum_read_block_error); - goto out; - } - blk = (void *)(bh->b_data); + /* ignore unwritten blocks or blocks for other filesystems */ + if (blk->voter_rid == 0 || blk->fsid != super->hdr.fsid) + continue; - /* ignore unwritten blocks */ - if (blk->write_nr == 0) - goto out; + if (invalid_quorum_block(bh, blk)) { + scoutfs_inc_counter(sb, quorum_read_invalid_block); + continue; + } - if (!test_set_buffer_scoutfs_traced(bh)) - trace_scoutfs_quorum_read_block(sb, bh->b_blocknr, blk); + list_for_each_entry_safe(qbh, tmp, blocks, head) { + if (stale_quorum_block(blk, &qbh->blk)) { + blk = NULL; + break; + } - if (invalid_quorum_block(super, bh, blk)) { - scoutfs_inc_counter(sb, quorum_read_invalid_block); - goto out; + if (stale_quorum_block(&qbh->blk, blk)) + list_move(&qbh->head, &stale); + } + free_quorum_blocks(&stale); + + if (!blk) + continue; + + qbh = kmalloc(sizeof(struct quorum_block_head), + GFP_NOFS); + if (!qbh) { + ret = -ENOMEM; + goto out; + } + + memcpy(&qbh->blk, blk, quorum_block_bytes(blk)); + list_add_tail(&qbh->head, blocks); + } + + list_for_each_entry(qbh, blocks, head) { + trace_scoutfs_quorum_read_block(sb, &qbh->blk); + scoutfs_inc_counter(sb, quorum_read_block); } - *blk_ret = *blk; - scoutfs_inc_counter(sb, quorum_read_block); ret = 0; out: - if (ret < 0) - memset(blk_ret, 0, sizeof(struct scoutfs_quorum_block)); brelse(bh); + if (ret < 0) + free_quorum_blocks(blocks); return ret; } -/* - * Iterate over config slots from the given index and return the first - * slot that has any of the given flags set. - */ -static inline int first_slot_flags(struct scoutfs_quorum_config *conf, - int i, u8 flags) -{ - for (; i < ARRAY_SIZE(conf->slots); i++) { - if (conf->slots[i].flags & flags) - break; - } - return i; -} - -/* - * Execute the loop body with the read block for each slot that's - * configured and active. If we can't read the block for whatever - * reason then the loop will execute with the blk struct zeroed. - */ -#define for_each_active_block(sb, super, conf, hists, hi, blk, slot, i) \ - for (i = first_slot_flags(conf, 0, SCOUTFS_QUORUM_SLOT_ACTIVE); \ - (i < ARRAY_SIZE(conf->slots)) && \ - (slot = &conf->slots[i], \ - hi = &hists[i], \ - read_quorum_block(sb, super, i, blk), 1); \ - i = first_slot_flags(conf, i + 1, SCOUTFS_QUORUM_SLOT_ACTIVE)) - -/* - * Iterate over every possible block, regardless of config. A lot of these - * will be zero. - */ -#define for_each_block(sb, super, i, blk) \ - for (i = 0; \ - (i < SCOUTFS_QUORUM_BLOCKS) && \ - (read_quorum_block(sb, super, i, blk), 1); \ - i++) - /* * Synchronously write a single quorum block. The caller has provided - * the meaningful fields for the write. We fill in the rest that are - * consistent for every write and zero the rest of the block. + * the meaningful fields for the write. We fill in the fsid, blkno, and + * crc for every write and zero the rest of the block. */ -static int write_quorum_block(struct super_block *sb, __le64 fsid, - __le64 config_gen, u8 our_slot, __le64 write_nr, - u64 elected_nr, u64 unmount_barrier, - u8 vote_slot, u8 flags) +static int write_quorum_block(struct super_block *sb, + struct scoutfs_quorum_block *our_blk) { + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_quorum_block *blk; - struct buffer_head *bh; + struct buffer_head *bh = NULL; + size_t size; int ret; BUILD_BUG_ON(sizeof(struct scoutfs_quorum_block) > SCOUTFS_BLOCK_SIZE); - if (WARN_ON_ONCE(our_slot >= SCOUTFS_QUORUM_MAX_SLOTS) || - WARN_ON_ONCE(vote_slot >= SCOUTFS_QUORUM_MAX_SLOTS)) - return -EINVAL; - - bh = sb_getblk(sb, SCOUTFS_QUORUM_BLKNO + our_slot); + bh = sb_getblk(sb, SCOUTFS_QUORUM_BLKNO + + prandom_u32_max(SCOUTFS_QUORUM_BLOCKS)); if (bh == NULL) { ret = -EIO; goto out; } + + size = quorum_block_bytes(our_blk); + if (WARN_ON_ONCE(size > SCOUTFS_BLOCK_SIZE || + size > bh->b_size)) { + ret = -EIO; + goto out; + } + blk = (void *)bh->b_data; + memset(blk, 0, bh->b_size); + memcpy(blk, our_blk, size); - blk->fsid = fsid; + blk->fsid = super->hdr.fsid; blk->blkno = cpu_to_le64(bh->b_blocknr); - blk->config_gen = config_gen; - blk->write_nr = write_nr; - blk->elected_nr = cpu_to_le64(elected_nr); - blk->unmount_barrier = cpu_to_le64(unmount_barrier); - blk->vote_slot = vote_slot; - blk->flags = flags; - blk->crc = quorum_block_crc(blk); lock_buffer(bh); @@ -467,7 +333,7 @@ static int write_quorum_block(struct super_block *sb, __le64 fsid, ret = 0; if (ret == 0) { - trace_scoutfs_quorum_write_block(sb, bh->b_blocknr, blk); + trace_scoutfs_quorum_write_block(sb, blk); scoutfs_inc_counter(sb, quorum_write_block); } out: @@ -478,401 +344,384 @@ out: } /* - * The caller read their quorum block which indicated that they were - * elected. We have to fence all other previously elected leaders so - * that we're running the only instance of the server. - * - * Time can pass between all phases of this: reading that we're elected, - * fencing, and writing the quorum block that clears the elected flag of - * those we fenced. - * - * This is always safe because we either have exclusive access to the - * device having fenced others or someone else would have fenced us - * before they write. + * Returns true if there's an entry for the given election. */ -static int fence_other_elected(struct super_block *sb, - struct scoutfs_super_block *super, - int our_slot, u64 elected_nr) +static bool log_contains(struct scoutfs_quorum_block *blk, u64 term, u64 rid) { - struct scoutfs_quorum_config *conf = &super->quorum_config; - struct scoutfs_quorum_block blk; - u8 flags; - int ret; int i; - for_each_block(sb, super, i, &blk) { - if (i != our_slot && - (blk.flags & SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED) && - le64_to_cpu(blk.elected_nr) <= elected_nr) { - scoutfs_err(sb, "would have fenced"); - scoutfs_inc_counter(sb, quorum_fenced); - - flags = blk.flags & ~SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED; - - ret = write_quorum_block(sb, super->hdr.fsid, - conf->gen, i, blk.write_nr, - le64_to_cpu(blk.elected_nr), - le64_to_cpu(blk.unmount_barrier), i, - flags); - if (ret) - break; - } - } - - return ret; -} - -struct quorum_block_history { - __le64 write_nr; - u8 writing; -}; - -/* - * The caller couldn't connect to a server. Read the quorum blocks - * until we see an elected leader and give their address to the caller. - * If we're configured as part of the quorum then we participate in the - * electing by writing our vote to our quorum block. - * - * Voting members read the blocks at regular intervals and update their - * quorum block with their vote for the elected leader. new leader. - * When a mount receives enough votes it marks its vote in the block as - * elected, fences other elected leaders, and returns to the caller who - * starts up the server for others to connect to. - * - * The calling client may have never seen a server before, or could have - * failed to connect to a valid server, or might have tried to connect - * to a dead server. They pass in an existing elected_nr if they want - * us to ignore old servers and they pass in a timeout so that they can - * return to retrying to connect to whatever address we find. - * - * When we return success we update the caller's elected info with the - * most recent elected leader we found, which may well be long gone. We - * return -ENOENT if we didn't find any elected leaders. - * - * If we return success because we saw a larger unmount barrier we set - * elected_nr to 0 and fill the unmount_barrier. - */ -int scoutfs_quorum_election(struct super_block *sb, char *our_name, - u64 old_elected_nr, ktime_t timeout_abs, - bool unmounting, u64 our_umb, - struct scoutfs_quorum_elected_info *qei) -{ - DECLARE_QUORUM_INFO(sb, qinf); - struct scoutfs_super_block *super = NULL; - struct scoutfs_quorum_config *conf; - struct scoutfs_quorum_slot *slot; - struct scoutfs_quorum_block blk; - struct quorum_block_history *hist; - struct quorum_block_history *hi; - ktime_t expires; - ktime_t now; - __le64 write_nr = 0; - u64 elected_nr = 0; - u64 unmount_barrier = 0; - u8 flags = 0; - int vote_streak = 0; - int vote_slot; - int our_slot; - int vote_prio; - int nr_active; - int nr_votes; - int majority; - int ret; - int i; - - super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); - hist = kcalloc(SCOUTFS_QUORUM_MAX_SLOTS, sizeof(hist[0]), GFP_NOFS); - if (!super || !hist) { - ret = -ENOMEM; - goto out; - } - - for (;;) { - now = ktime_get(); - expires = ktime_add_ms(now, SCOUTFS_QUORUM_INTERVAL_MS); - - ret = read_quorum_config(sb, super, our_name, &our_slot, - &nr_active); - if (ret) - goto out; - conf = &super->quorum_config; - - /* update sysfs with most recently seen config */ - if (our_slot >= 0) { - slot = &conf->slots[our_slot]; - addr_to_sin(&qinf->conf_addr, &slot->addr); - qinf->conf_port = le16_to_cpu(slot->addr.port); - } else { - memset(&qinf->conf_addr, 0, sizeof(qinf->conf_addr)); - qinf->conf_port = 0; - } - - majority = scoutfs_quorum_majority(sb, conf); - - readahead_quorum_blocks(sb); - - /* default to voting for ourselves, but at min prio */ - vote_slot = our_slot; - vote_prio = -1; - memset(qei, 0, sizeof(*qei)); - - for_each_active_block(sb, super, conf, hist, hi, &blk, slot, i){ - /* determine which mounts are writing */ - if (blk.config_gen == conf->gen && - blk.write_nr != 0 && - blk.write_nr != hi->write_nr) - hi->writing = min(hi->writing + 1, 2); - else - hi->writing = 0; - hi->write_nr = blk.write_nr; - - /* vote for first highest priority writing block */ - if (hi->writing >= 2 && - slot->vote_priority > vote_prio) { - vote_slot = i; - vote_prio = slot->vote_priority; - } - - /* find the most recently elected leader */ - if ((blk.config_gen == conf->gen) && - (blk.flags & SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED) && - (le64_to_cpu(blk.elected_nr) > qei->elected_nr)){ - addr_to_sin(&qei->sin, &slot->addr); - qei->config_gen = blk.config_gen; - qei->write_nr = blk.write_nr; - qei->elected_nr = le64_to_cpu(blk.elected_nr); - qei->unmount_barrier = - le64_to_cpu(blk.unmount_barrier); - qei->config_slot = i; - qei->flags = blk.flags; - } - } - - /* - * After writing a block indicating that we were elected - * we make sure that we can read it and that we're still - * the most recent elected leader. If we are then we - * try to fence. If we can't read it, or we're not the - * most recent, or we couldn't fence, then we fall back - * to participating in the election. - */ - if (flags & SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED) { - if (qei->write_nr == write_nr && - qei->elected_nr == elected_nr && - qei->config_slot == our_slot) { - ret = fence_other_elected(sb, super, our_slot, - elected_nr); - if (ret == 0) { - qei->run_server = true; - qinf->is_leader = true; - goto out; - } - - memset(qei, 0, sizeof(*qei)); - } - - vote_streak = 0; - } - - /* return if we found a new listening leader or timed out */ - if (((qei->elected_nr > old_elected_nr) && - (qei->flags & SCOUTFS_QUORUM_BLOCK_FLAG_LISTENING)) || - ktime_after(now, timeout_abs)) { - if (qei->elected_nr > 0) { - scoutfs_inc_counter(sb, quorum_found_leader); - ret = 0; - } else { - scoutfs_inc_counter(sb, quorum_no_leader); - ret = -ENOENT; - } - goto out; - } - - /* wait for the next cycle if we're not in the voting config */ - if (our_slot < 0) - continue; - - nr_votes = 0; - write_nr = cpu_to_le64(1); - elected_nr = 0; - unmount_barrier = 0; - flags = 0; - - for_each_active_block(sb, super, conf, hist, hi, &blk, slot, i){ - /* count our votes (maybe including from us) */ - if (hi->writing >= 2 && blk.vote_slot == our_slot) - nr_votes++; - - /* can finish unmounting if members all left */ - if (unmounting && - le64_to_cpu(blk.unmount_barrier) > our_umb) { - qei->elected_nr = 0; - qei->unmount_barrier = - le64_to_cpu(blk.unmount_barrier); - ret = 0; - goto out; - } - - /* sample existing fields for our write */ - if (i == our_slot) { - write_nr = blk.write_nr; - le64_add_cpu(&write_nr, 1); - } - elected_nr = max(elected_nr, - le64_to_cpu(blk.elected_nr)); - unmount_barrier = max(unmount_barrier, - le64_to_cpu(blk.unmount_barrier)); - } - - - /* elected after sufficient cycles with a majority vote */ - if (nr_votes >= majority) - vote_streak = min(vote_streak + 1, 2); - else - vote_streak = 0; - - if (vote_streak >= 2) { - flags |= SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED; - elected_nr++; - } - - write_quorum_block(sb, super->hdr.fsid, conf->gen, our_slot, - write_nr, elected_nr, unmount_barrier, - vote_slot, flags); - - set_current_state(TASK_UNINTERRUPTIBLE); - schedule_hrtimeout(&expires, HRTIMER_MODE_ABS); - scoutfs_inc_counter(sb, quorum_waited); - } - -out: - kfree(super); - kfree(hist); - - if (ret) { - memset(qei, 0, sizeof(*qei)); - scoutfs_inc_counter(sb, quorum_election_error); - } - - return ret; -} - -/* - * The calling server has successfully started and is listening for - * collections. It writes a new block to communicate to the other - * mounts that they should now try to connect. We do increase the write_nr - * here to still indicate that we're alive. - */ -int scoutfs_quorum_set_listening(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - - qei->flags |= SCOUTFS_QUORUM_BLOCK_FLAG_LISTENING; - le64_add_cpu(&qei->write_nr, 1); - - return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, - qei->config_slot, qei->write_nr, - qei->elected_nr, qei->unmount_barrier, - qei->config_slot, qei->flags); -} - -/* - * The calling server is shutting down and has finished modifying - * persistent state. We clear the elected flag from our quorum block so - * that mounts won't try to connect and so that the next next leader - * won't try to fence. - * - * By definition nothing has written to the slot since we wrote our - * elected quorum block and the slot could not have been reclaimed. To - * reclaim the slot would have required proving that we were gone or - * fencing us. - * - * If this fails then the mount is in trouble because it'll probably be - * fenced by the next elected leader. - * - * XXX I think there's an interesting race here. If the server is - * running in an old config then the server's slot can be reclaimed if - * the server sees a connection from the current gen. If the server is - * taking a client connection as an indication that the slot won't be - * written then the client needs to shut down the server before trying - * to connect with a new gen. - */ -int scoutfs_quorum_clear_elected(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - DECLARE_QUORUM_INFO(sb, qinf); - - qei->flags &= ~SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED; - qinf->is_leader = false; - - return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, - qei->config_slot, qei->write_nr, - qei->elected_nr, qei->unmount_barrier, - qei->config_slot, qei->flags); -} - -int scoutfs_quorum_update_barrier(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei, - u64 unmount_barrier) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - - qei->unmount_barrier = unmount_barrier; - - return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, - qei->config_slot, qei->write_nr, - qei->elected_nr, qei->unmount_barrier, - qei->config_slot, qei->flags); -} - -/* - * If there's only one or two active slots then a single vote is sufficient - * for a majority. - */ -int scoutfs_quorum_majority(struct super_block *sb, - struct scoutfs_quorum_config *conf) -{ - struct scoutfs_quorum_slot *slot; - int nr_active = 0; - int majority; - int i; - - for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - slot = &conf->slots[i]; - - if (slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE) - nr_active++; - } - - if (nr_active <= 2) - majority = 1; - else if (nr_active & 1) - majority = (nr_active + 1) / 2; - else - majority = (nr_active / 2) + 1; - - return majority; -} - -bool scoutfs_quorum_voting_member(struct super_block *sb, - struct scoutfs_quorum_config *conf, - char *name) -{ - struct scoutfs_quorum_slot *slot; - int i; - - for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - slot = &conf->slots[i]; - - if (strcmp(slot->name, name) == 0) + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].term) == term && + le64_to_cpu(blk->log[i].rid) == rid) return true; } return false; } +/* add an entry to the log, returning error if it's full */ +static int log_add(struct scoutfs_quorum_block *blk, u64 term, u64 rid, + struct scoutfs_inet_addr *addr) +{ + int i; + + if (log_contains(blk, term, rid)) + return 0; + + if (blk->log_nr == SCOUTFS_QUORUM_LOG_MAX) + return -ENOSPC; + + i = blk->log_nr++; + blk->log[i].term = cpu_to_le64(term); + blk->log[i].rid = cpu_to_le64(rid); + blk->log[i].addr = *addr; + + return 0; +} + +/* migrate live log entries between blocks, returning err if full */ +static int log_merge(struct scoutfs_quorum_block *our_blk, + struct scoutfs_quorum_block *blk, + u64 fenced_term) +{ + int ret; + int i; + + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].term) > fenced_term) { + ret = log_add(our_blk, le64_to_cpu(blk->log[i].term), + le64_to_cpu(blk->log[i].rid), + &blk->log[i].addr); + if (ret < 0) + return ret; + } + } + + return 0; +} + +/* Remove old log entries for a voter before a given term. */ +static void log_purge(struct scoutfs_quorum_block *blk, u64 term, u64 rid) +{ + int i; + + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].term) < term && + le64_to_cpu(blk->log[i].rid) == rid) { + if (i != blk->log_nr - 1) + swap(blk->log[i], blk->log[blk->log_nr - 1]); + blk->log_nr--; + i--; /* continue from swapped in entry */ + } + } +} + + +/* + * The caller received a majority of votes and has been elected. Before + * assuming exclusive write access to the device we fence the winners of + * any previous elections still present in the log. Once they're fenced + * we re-read the super and update the fenced_term to indicate that + * those previous elections can be ignored and purged from the log. + * + * We can be attempting this concurrently with both previous and future + * elected leaders. The leader with the greatest elected term will win + * and fence all previous elected leaders. + * + * We clobber the caller's block as we go to not fence rids multiple times. + */ +static int fence_previous(struct super_block *sb, + struct scoutfs_quorum_block *blk, + u64 our_rid, u64 fenced_term, u64 term) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct sockaddr_in their_sin; + int ret; + int i; + + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].rid) != our_rid && + le64_to_cpu(blk->log[i].term) > fenced_term && + le64_to_cpu(blk->log[i].term) < term) { + + scoutfs_inc_counter(sb, quorum_fenced); + scoutfs_addr_to_sin(&their_sin, &blk->log[i].addr); + scoutfs_err(sb, "fencing "SCSBF" at "SIN_FMT, + SCSB_LEFR_ARGS(super->hdr.fsid, + blk->log[i].rid), + SIN_ARG(&their_sin)); + + log_purge(blk, term, le64_to_cpu(blk->log[i].rid)); + i = -1; /* start over */ + } + } + + /* update fenced term now that we have exclusive access */ + ret = 0; + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (super) { + ret = scoutfs_read_super(sb, super); + if (ret == 0) { + super->quorum_fenced_term = cpu_to_le64(term - 1); + ret = scoutfs_write_super(sb, super); + + } + kfree(super); + } else { + ret = -ENOMEM; + } + + if (ret != 0) { + scoutfs_err(sb, "failed to update fenced_term in super, this mount will probably be fenced"); + } + + return ret; +} + + + +/* + * The calling voting mount couldn't connect to a server. Participate + * in a raft election to chose a mount to start a new server. If a + * majority of other mounts join us then one of us will be elected and + * our caller will start the server. + * + * Voting members read the blocks at regular intervals. If they see a + * new election they vote for that candidate for the remainder of the + * election. If the election timeout expires they will start a new + * election and vote for themselves. Eventually a sufficient majority + * sees a new election and all vote in the majority for that candidate. + * + * The calling client may have just failed to connect to an elected + * address in the super block. We assume that server is dead and ignore + * it when trying to elect a new leader. But we eventually return with + * a timeout because the server could actually be fine and the client + * could have had communication to the server restored. + * + * We return success if we see a new server elected. If we are elected + * we set the caller's elected_term so they know to start the server. + */ +int scoutfs_quorum_election(struct super_block *sb, ktime_t timeout_abs, + u64 prev_term, u64 *elected_term) +{ + DECLARE_QUORUM_INFO(sb, qinf); + struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = NULL; + struct scoutfs_quorum_block *our_blk = NULL; + struct scoutfs_quorum_block *blk; + struct quorum_block_head *qbh; + struct scoutfs_inet_addr addr; + enum { VOTER, CANDIDATE }; + ktime_t cycle_to; + ktime_t term_to; + LIST_HEAD(blocks); + u64 vote_for_write_nr; + u64 vote_for_rid; + u64 write_nr; + u64 term; + int log_cycles = 0; + int votes; + int role; + int ret; + + *elected_term = 0; + + trace_scoutfs_quorum_election(sb, prev_term); + + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + our_blk = kmalloc(SCOUTFS_BLOCK_SIZE, GFP_NOFS); + if (!super || !our_blk) { + ret = -ENOMEM; + goto out; + } + + /* start out as a passive voter */ + role = VOTER; + term = 0; + write_nr = 0; + vote_for_rid = 0; + vote_for_write_nr = 0; + + /* we'll become a candidate if we don't see another candidate */ + term_to = random_to(SCOUTFS_QUORUM_TERM_LO_MS, + SCOUTFS_QUORUM_TERM_HI_MS); + + for (;;) { + memset(our_blk, 0, SCOUTFS_BLOCK_SIZE); + + scoutfs_inc_counter(sb, quorum_cycle); + + ret = scoutfs_read_super(sb, super); + if (ret) + goto out; + + /* done if we see evidence of a new server */ + if (le64_to_cpu(super->quorum_server_term) > prev_term) { + scoutfs_inc_counter(sb, quorum_saw_super_leader); + ret = 0; + goto out; + } + + /* done if we couldn't elect anyone */ + if (ktime_after(ktime_get(), timeout_abs)) { + scoutfs_inc_counter(sb, quorum_timedout); + ret = -ETIMEDOUT; + goto out; + } + + /* become a candidate if the election times out */ + if (ktime_after(ktime_get(), term_to)) { + scoutfs_inc_counter(sb, quorum_election_timeout); + term_to = random_to(SCOUTFS_QUORUM_TERM_LO_MS, + SCOUTFS_QUORUM_TERM_HI_MS); + role = CANDIDATE; + term++; + vote_for_rid = sbi->rid; + log_cycles = 0; + } + + free_quorum_blocks(&blocks); + ret = read_quorum_blocks(sb, &blocks); + if (ret < 0) + goto out; + + votes = 0; + + list_for_each_entry(qbh, &blocks, head) { + blk = &qbh->blk; + + /* + * Become a voter for a candidate the first time + * we see a new term. + * + * And also if we're a candidate and see a + * higher rid candidate in our term. This + * minimizes instability when two quorums are + * possible and race to elect two leaders. This + * is only barely reasonable when accepting the + * risk of instability in two mount + * configurations. + */ + if ((le64_to_cpu(blk->term) > term) || + (role == CANDIDATE && + le64_to_cpu(blk->term) == term && + blk->voter_rid == blk->vote_for_rid && + le64_to_cpu(blk->voter_rid) > sbi->rid)) { + role = VOTER; + term = le64_to_cpu(blk->term); + vote_for_rid = le64_to_cpu(blk->vote_for_rid); + vote_for_write_nr = 0; + votes = 0; + log_cycles = 0; + } + + /* candidate writes suppress voter election timers */ + if (role == VOTER && + blk->voter_rid == blk->vote_for_rid && + le64_to_cpu(blk->write_nr) > vote_for_write_nr) { + term_to = random_to(SCOUTFS_QUORUM_TERM_LO_MS, + SCOUTFS_QUORUM_TERM_HI_MS); + vote_for_write_nr = le64_to_cpu(blk->write_nr); + } + + /* count our votes */ + if (role == CANDIDATE && + le64_to_cpu(blk->vote_for_rid) == sbi->rid) { + votes++; + } + + /* try to write greater write_nr */ + write_nr = max(write_nr, le64_to_cpu(blk->write_nr)); + } + + trace_scoutfs_quorum_election_vote(sb, role, term, + vote_for_rid, votes, + log_cycles, + super->quorum_count); + + /* first merge logs from all votes this term */ + list_for_each_entry(qbh, &blocks, head) { + blk = &qbh->blk; + + ret = log_merge(our_blk, blk, + le64_to_cpu(super->quorum_fenced_term)); + if (ret < 0) + goto out; + } + + /* remove logs for voters that can't be servers */ + list_for_each_entry(qbh, &blocks, head) { + blk = &qbh->blk; + + if (blk->voter_rid != blk->vote_for_rid) + log_purge(our_blk, le64_to_cpu(blk->term), + le64_to_cpu(blk->voter_rid)); + } + + /* add ourselves to the log when we see vote quorum */ + if (role == CANDIDATE && votes >= super->quorum_count) { + scoutfs_addr_from_sin(&addr, &opts->server_addr); + ret = log_add(our_blk, term, vote_for_rid, &addr); + if (ret < 0) + goto out; + log_cycles++; /* will be written *this* cycle */ + } + + /* elected candidates can proceed after their log cycles */ + if (role == CANDIDATE && + log_cycles > SCOUTFS_QUORUM_ELECTED_LOG_CYCLES) { + /* our_blk is clobbered */ + ret = fence_previous(sb, our_blk, sbi->rid, + le64_to_cpu(super->quorum_fenced_term), + term); + if (ret < 0) + goto out; + scoutfs_inc_counter(sb, quorum_elected_leader); + qinf->is_leader = true; + *elected_term = term; + goto out; + } + + /* write our block every cycle */ + if (term > 0) { + our_blk->term = cpu_to_le64(term); + write_nr++; + our_blk->write_nr = cpu_to_le64(write_nr); + our_blk->voter_rid = cpu_to_le64(sbi->rid); + our_blk->vote_for_rid = cpu_to_le64(vote_for_rid); + + ret = write_quorum_block(sb, our_blk); + if (ret < 0) + goto out; + } + + /* add a small random delay to each cycle */ + cycle_to = random_to(SCOUTFS_QUORUM_CYCLE_LO_MS, + SCOUTFS_QUORUM_CYCLE_HI_MS); + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_hrtimeout(&cycle_to, HRTIMER_MODE_ABS); + } + +out: + free_quorum_blocks(&blocks); + kfree(super); + kfree(our_blk); + + trace_scoutfs_quorum_election_ret(sb, ret, *elected_term); + if (ret) + scoutfs_inc_counter(sb, quorum_failure); + + return ret; +} + +void scoutfs_quorum_clear_leader(struct super_block *sb) +{ + DECLARE_QUORUM_INFO(sb, qinf); + + qinf->is_leader = false; +} + static ssize_t is_leader_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -882,28 +731,8 @@ static ssize_t is_leader_show(struct kobject *kobj, } SCOUTFS_ATTR_RO(is_leader); -static ssize_t ipv4_addr_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - DECLARE_QUORUM_INFO_KOBJ(kobj, qinf); - - return snprintf(buf, PAGE_SIZE, "%pIS", &qinf->conf_addr); -} -SCOUTFS_ATTR_RO(ipv4_addr); - -static ssize_t ipv4_port_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - DECLARE_QUORUM_INFO_KOBJ(kobj, qinf); - - return snprintf(buf, PAGE_SIZE, "%u", qinf->conf_port); -} -SCOUTFS_ATTR_RO(ipv4_port); - static struct attribute *quorum_attrs[] = { SCOUTFS_ATTR_PTR(is_leader), - SCOUTFS_ATTR_PTR(ipv4_addr), - SCOUTFS_ATTR_PTR(ipv4_port), NULL, }; diff --git a/kmod/src/quorum.h b/kmod/src/quorum.h index cea55525..96eac0e4 100644 --- a/kmod/src/quorum.h +++ b/kmod/src/quorum.h @@ -1,33 +1,9 @@ #ifndef _SCOUTFS_QUORUM_H_ #define _SCOUTFS_QUORUM_H_ -struct scoutfs_quorum_elected_info { - struct sockaddr_in sin; - __le64 config_gen; - __le64 write_nr; - u64 elected_nr; - u64 unmount_barrier; - unsigned int config_slot; - bool run_server; - u8 flags; -}; - -int scoutfs_quorum_election(struct super_block *sb, char *our_name, - u64 old_elected_nr, ktime_t timeout_abs, - bool unmounting, u64 our_umb, - struct scoutfs_quorum_elected_info *qei); -int scoutfs_quorum_set_listening(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei); -int scoutfs_quorum_clear_elected(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei); -int scoutfs_quorum_update_barrier(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei, - u64 unmount_barrier); -int scoutfs_quorum_majority(struct super_block *sb, - struct scoutfs_quorum_config *conf); -bool scoutfs_quorum_voting_member(struct super_block *sb, - struct scoutfs_quorum_config *conf, - char *name); +int scoutfs_quorum_election(struct super_block *sb, ktime_t timeout_abs, + u64 prev_term, u64 *elected_term); +void scoutfs_quorum_clear_leader(struct super_block *sb); int scoutfs_quorum_setup(struct super_block *sb); void scoutfs_quorum_destroy(struct super_block *sb); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 20fc1282..d87dcb2c 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2544,53 +2544,118 @@ TRACE_EVENT(scoutfs_lock_message, __entry->old_mode, __entry->new_mode) ); -DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, - TP_PROTO(struct super_block *sb, u64 io_blkno, - struct scoutfs_quorum_block *blk), - TP_ARGS(sb, io_blkno, blk), +TRACE_EVENT(scoutfs_quorum_election, + TP_PROTO(struct super_block *sb, u64 prev_term), + + TP_ARGS(sb, prev_term), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__u64, io_blkno) - __field(__u64, hdr_blkno) - __field(__u64, config_gen) - __field(__u64, write_nr) - __field(__u64, elected_nr) - __field(__u64, unmount_barrier) - __field(__u32, crc) - __field(__u8, vote_slot) - __field(__u8, flags) + __field(__u64, prev_term) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->io_blkno = io_blkno; - __entry->hdr_blkno = le64_to_cpu(blk->blkno); - __entry->config_gen = le64_to_cpu(blk->config_gen); - __entry->write_nr = le64_to_cpu(blk->write_nr); - __entry->elected_nr = le64_to_cpu(blk->elected_nr); - __entry->unmount_barrier = le64_to_cpu(blk->unmount_barrier); - __entry->crc = le32_to_cpu(blk->crc); - __entry->vote_slot = blk->vote_slot; - __entry->flags = blk->flags; + __entry->prev_term = prev_term; ), - TP_printk(SCSBF" io_blkno %llu hdr_blkno %llu config_gen %llu write_nr %llu elected_nr %llu umb %llu crc 0x%08x vote_slot %u flags %02x", - SCSB_TRACE_ARGS, __entry->io_blkno, __entry->hdr_blkno, - __entry->config_gen, __entry->write_nr, __entry->elected_nr, - __entry->unmount_barrier, __entry->crc, __entry->vote_slot, - __entry->flags) + TP_printk(SCSBF" prev_term %llu", + SCSB_TRACE_ARGS, __entry->prev_term) +); + +TRACE_EVENT(scoutfs_quorum_election_ret, + TP_PROTO(struct super_block *sb, int ret, u64 elected_term), + + TP_ARGS(sb, ret, elected_term), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, ret) + __field(__u64, elected_term) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ret = ret; + __entry->elected_term = elected_term; + ), + + TP_printk(SCSBF" ret %d elected_term %llu", + SCSB_TRACE_ARGS, __entry->ret, __entry->elected_term) +); + +TRACE_EVENT(scoutfs_quorum_election_vote, + TP_PROTO(struct super_block *sb, int role, u64 term, u64 vote_for_rid, + int votes, int log_cycles, int quorum_count), + + TP_ARGS(sb, role, term, vote_for_rid, votes, log_cycles, quorum_count), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, role) + __field(__u64, term) + __field(__u64, vote_for_rid) + __field(int, votes) + __field(int, log_cycles) + __field(int, quorum_count) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->role = role; + __entry->term = term; + __entry->vote_for_rid = vote_for_rid; + __entry->votes = votes; + __entry->log_cycles = log_cycles; + __entry->quorum_count = quorum_count; + ), + + TP_printk(SCSBF" role %d term %llu vote_for_rid %016llx votes %d log_cycles %d quorum_count %d", + SCSB_TRACE_ARGS, __entry->role, __entry->term, + __entry->vote_for_rid, __entry->votes, __entry->log_cycles, + __entry->quorum_count) +); + +DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, + TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk), + + TP_ARGS(sb, blk), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(__u64, term) + __field(__u64, write_nr) + __field(__u64, voter_rid) + __field(__u64, vote_for_rid) + __field(__u32, crc) + __field(__u8, log_nr) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = le64_to_cpu(blk->blkno); + __entry->term = le64_to_cpu(blk->term); + __entry->write_nr = le64_to_cpu(blk->write_nr); + __entry->voter_rid = le64_to_cpu(blk->voter_rid); + __entry->vote_for_rid = le64_to_cpu(blk->vote_for_rid); + __entry->crc = le32_to_cpu(blk->crc); + __entry->log_nr = blk->log_nr; + ), + + TP_printk(SCSBF" blkno %llu term %llu write_nr %llu voter_rid %016llx vote_for_rid %016llx crc 0x%08x log_nr %u", + SCSB_TRACE_ARGS, __entry->blkno, __entry->term, + __entry->write_nr, __entry->voter_rid, __entry->vote_for_rid, + __entry->crc, __entry->log_nr) ); DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_read_block, - TP_PROTO(struct super_block *sb, u64 io_blkno, - struct scoutfs_quorum_block *blk), - TP_ARGS(sb, io_blkno, blk) + TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk), + TP_ARGS(sb, blk) ); DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_write_block, - TP_PROTO(struct super_block *sb, u64 io_blkno, - struct scoutfs_quorum_block *blk), - TP_ARGS(sb, io_blkno, blk) + TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk), + TP_ARGS(sb, blk) ); /* diff --git a/kmod/src/server.c b/kmod/src/server.c index d077d858..291c7df5 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -42,8 +42,8 @@ * connection and accepts connections from all the other mounts acting * as clients. * - * The server is started when raft elects the mount as the leader. If - * it sees errors it shuts down the server in the hopes that another + * The server is started by the mount that is elected leader by quorum. + * If it sees errors it shuts down the server in the hopes that another * mount will become the leader and have less trouble. */ @@ -61,8 +61,6 @@ struct server_info { u64 term; struct scoutfs_net_connection *conn; - struct scoutfs_quorum_elected_info qei; - /* request processing coordinates committing manifest and alloc */ struct rw_semaphore commit_rwsem; struct llist_head commit_waiters; @@ -1185,14 +1183,16 @@ int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, } static int insert_mounted_client(struct super_block *sb, u64 node_id, - char *name) + u64 gr_flags) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_mounted_client_btree_key mck; struct scoutfs_mounted_client_btree_val mcv; mck.node_id = cpu_to_be64(node_id); - strncpy(mcv.name, name, sizeof(mcv.name)); + mcv.flags = 0; + if (gr_flags & SCOUTFS_NET_GREETING_FLAG_VOTER) + mcv.flags |= SCOUTFS_MOUNTED_CLIENT_VOTER; return scoutfs_btree_insert(sb, &super->mounted_clients, &mck, sizeof(mck), &mcv, sizeof(mcv)); @@ -1260,6 +1260,7 @@ static int server_greeting(struct super_block *sb, DECLARE_SERVER_INFO(sb, server); struct commit_waiter cw; __le64 node_id = 0; + __le64 umb = 0; bool sent_node_id; bool first_contact; bool farewell; @@ -1293,10 +1294,12 @@ static int server_greeting(struct super_block *sb, spin_lock(&server->lock); node_id = super->next_node_id; le64_add_cpu(&super->next_node_id, 1); + umb = super->unmount_barrier; spin_unlock(&server->lock); mutex_lock(&server->farewell_mutex); - ret = insert_mounted_client(sb, le64_to_cpu(node_id), gr->name); + ret = insert_mounted_client(sb, le64_to_cpu(node_id), + le64_to_cpu(gr->flags)); mutex_unlock(&server->farewell_mutex); if (ret == 0) @@ -1308,6 +1311,7 @@ static int server_greeting(struct super_block *sb, } } else { node_id = gr->node_id; + umb = gr->unmount_barrier; } send_err: @@ -1315,11 +1319,10 @@ send_err: if (err) node_id = 0; - memset(greet.name, 0, sizeof(greet.name)); greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; greet.server_term = cpu_to_le64(server->term); - greet.unmount_barrier = cpu_to_le64(server->qei.unmount_barrier); + greet.unmount_barrier = umb; greet.node_id = node_id; greet.flags = 0; @@ -1379,31 +1382,20 @@ static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) /* * This work processes farewell requests asynchronously. Requests from - * voting quorum members can be held until they're no longer needed to - * vote for quorum and elect a server to process farewell requests. - * - * This will hold farewell requests from voting clients until either it - * isn't needed for quorum because a majority remains without it, or it - * won't be needed for quorum because all the remaining mounted clients - * are voting and waiting for farewell. + * voting clients can be held until only the final quorum remains and + * they've all sent farewell requests. * * When we remove the last mounted client record for the last voting - * client then we increase the unmount_barrier and write it to the - * server's quorum block. If voting clients don't get their farewell - * response they'll attempt to form quorum again to start the server for - * their farewell response but will find the increased umount_barrier. - * The'll know that their farewell has been processed and they can exit - * without forming quorum. + * client then we increase the unmount_barrier and write it to the super + * block. If voting clients don't get their farewell response they'll + * see the greater umount_barrier in the super and will know that their + * farewell has been processed and that they can exit. * * Responses that are waiting for clients who aren't voting are * immediately sent. Clients that don't have a mounted client record * have already had their farewell processed by another server and can * proceed. * - * This can trust the quorum config found in the super that was read - * when the server started. Only the current server can rewrite the - * working config. - * * Farewell responses are unique in that sending them causes the server * to shutdown the connection to the client next time the socket * disconnects. If the socket is destroyed before the client gets the @@ -1420,7 +1412,6 @@ static void farewell_worker(struct work_struct *work) farewell_work); struct super_block *sb = server->sb; struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_quorum_config *conf = &super->quorum_config; struct scoutfs_mounted_client_btree_key mck; struct scoutfs_mounted_client_btree_val *mcv; struct farewell_request *tmp; @@ -1429,7 +1420,6 @@ static void farewell_worker(struct work_struct *work) struct commit_waiter cw; unsigned int nr_unmounting = 0; unsigned int nr_mounted = 0; - unsigned int majority; LIST_HEAD(reqs); LIST_HEAD(send); bool deleted = false; @@ -1437,8 +1427,6 @@ static void farewell_worker(struct work_struct *work) bool more_reqs; int ret; - majority = scoutfs_quorum_majority(sb, conf); - /* grab all the requests that are waiting */ mutex_lock(&server->farewell_mutex); list_splice_init(&server->farewell_requests, &reqs); @@ -1463,7 +1451,7 @@ static void farewell_worker(struct work_struct *work) } mcv = iref.val; - voting = scoutfs_quorum_voting_member(sb, conf, mcv->name); + voting = (mcv->flags & SCOUTFS_MOUNTED_CLIENT_VOTER) != 0; scoutfs_btree_put_iref(&iref); if (!voting) { @@ -1492,7 +1480,7 @@ static void farewell_worker(struct work_struct *work) memcpy(&mck, iref.key, sizeof(mck)); mcv = iref.val; - if (scoutfs_quorum_voting_member(sb, conf, mcv->name)) + if (mcv->flags & SCOUTFS_MOUNTED_CLIENT_VOTER) nr_mounted++; scoutfs_btree_put_iref(&iref); @@ -1503,7 +1491,8 @@ static void farewell_worker(struct work_struct *work) /* send as many responses as we can to maintain quorum */ while ((fw = list_first_entry_or_null(&reqs, struct farewell_request, entry)) && - (nr_mounted > majority || nr_unmounting >= nr_mounted)) { + (nr_mounted > super->quorum_count || + nr_unmounting >= nr_mounted)) { list_move_tail(&fw->entry, &send); nr_mounted--; @@ -1529,10 +1518,13 @@ static void farewell_worker(struct work_struct *work) goto out; } - /* update the unmount barrier the first time we delete all mounted */ + /* update the unmount barrier if we deleted all voting clients */ if (deleted && nr_mounted == 0) { - ret = scoutfs_quorum_update_barrier(sb, &server->qei, - server->qei.unmount_barrier + 1); + down_read(&server->commit_rwsem); + le64_add_cpu(&super->unmount_barrier, 1); + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + ret = wait_for_commit(&cw); if (ret) goto out; } @@ -2290,9 +2282,14 @@ static void scoutfs_server_worker(struct work_struct *work) struct sockaddr_in sin; LIST_HEAD(conn_list); int ret; + int err; trace_scoutfs_server_work_enter(sb, 0, 0); + sin = server->listen_sin; + + scoutfs_info(sb, "server setting up at "SIN_FMT, SIN_ARG(&sin)); + conn = scoutfs_net_alloc_conn(sb, server_notify_up, server_notify_down, sizeof(struct server_client_info), server_req_funcs, "server"); @@ -2301,8 +2298,6 @@ static void scoutfs_server_worker(struct work_struct *work) goto out; } - sin = server->listen_sin; - ret = scoutfs_net_bind(sb, conn, &sin); if (ret) { scoutfs_err(sb, "server failed to bind to "SIN_FMT", err %d%s", @@ -2312,37 +2307,44 @@ static void scoutfs_server_worker(struct work_struct *work) goto out; } - ret = scoutfs_read_super(sb, super); if (ret) goto out; /* start up the server subsystems before accepting */ - ret = scoutfs_btree_setup(sb) ?: + ret = scoutfs_read_super(sb, super) ?: + scoutfs_btree_setup(sb) ?: scoutfs_manifest_setup(sb) ?: scoutfs_lock_server_setup(sb); if (ret) goto shutdown; - complete(&server->start_comp); + /* + * Write our address in the super before it's possible for net + * processing to start writing the super as part of + * transactions. In theory clients could be trying to connect + * to our address without having seen it in the super (maybe + * they saw it a long time ago). + */ + scoutfs_addr_from_sin(&super->server_addr, &sin); + super->quorum_server_term = cpu_to_le64(server->term); + ret = scoutfs_write_super(sb, super); + if (ret < 0) + goto shutdown; server->stable_manifest_root = super->manifest.root; - scoutfs_info(sb, "server started on "SIN_FMT, SIN_ARG(&sin)); - /* start accepting connections and processing work */ server->conn = conn; scoutfs_net_listen(sb, conn); - ret = scoutfs_quorum_set_listening(sb, &server->qei); + scoutfs_info(sb, "server ready at "SIN_FMT, SIN_ARG(&sin)); + complete(&server->start_comp); - if (ret == 0) { - /* wait_event/wake_up provide barriers */ - wait_event_interruptible(server->waitq, server->shutting_down); - } - - scoutfs_info(sb, "server shutting down on "SIN_FMT, SIN_ARG(&sin)); + /* wait_event/wake_up provide barriers */ + wait_event_interruptible(server->waitq, server->shutting_down); shutdown: + scoutfs_info(sb, "server shutting down at "SIN_FMT, SIN_ARG(&sin)); /* wait for request processing */ scoutfs_net_shutdown(sb, conn); /* drain compact work queued by responses */ @@ -2357,17 +2359,41 @@ shutdown: scoutfs_lock_server_destroy(sb); out: + scoutfs_quorum_clear_leader(sb); scoutfs_net_free_conn(sb, conn); + scoutfs_info(sb, "server stopped at "SIN_FMT, SIN_ARG(&sin)); trace_scoutfs_server_work_exit(sb, 0, ret); + /* + * Always try to clear our presence in the super so that we're + * not fenced. We do this last because other mounts will try to + * reach quorum the moment they see zero here. The later we do + * this the longer we have to finish shutdown while clients + * timeout. + */ + err = scoutfs_read_super(sb, super); + if (err == 0) { + super->quorum_fenced_term = cpu_to_le64(server->term); + memset(&super->server_addr, 0, sizeof(super->server_addr)); + err = scoutfs_write_super(sb, super); + } + if (err < 0) { + scoutfs_err(sb, "failed to clear election term %llu at "SIN_FMT", this mount could be fenced", + server->term, SIN_ARG(&sin)); + } + server->err = ret; complete(&server->start_comp); } -/* XXX can we call start multiple times? */ +/* + * Wait for the server to successfully start. If this returns error then + * the super block's fence_term has been set to the new server's term so + * that it won't be fenced. + */ int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, - u64 term, struct scoutfs_quorum_elected_info *qei) + u64 term) { DECLARE_SERVER_INFO(sb, server); @@ -2375,7 +2401,6 @@ int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, server->shutting_down = false; server->listen_sin = *sin; server->term = term; - server->qei = *qei; init_completion(&server->start_comp); queue_work(server->wq, &server->work); @@ -2398,8 +2423,7 @@ void scoutfs_server_abort(struct super_block *sb) * Once the server is stopped we give the caller our election info * which might have been modified while we were running. */ -void scoutfs_server_stop(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei) +void scoutfs_server_stop(struct super_block *sb) { DECLARE_SERVER_INFO(sb, server); @@ -2407,8 +2431,6 @@ void scoutfs_server_stop(struct super_block *sb, /* XXX not sure both are needed */ cancel_work_sync(&server->work); cancel_work_sync(&server->commit_work); - - *qei = server->qei; } int scoutfs_server_setup(struct super_block *sb) diff --git a/kmod/src/server.h b/kmod/src/server.h index fee6ac0e..32c6ccea 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -74,10 +74,9 @@ int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, struct sockaddr_in; struct scoutfs_quorum_elected_info; int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, - u64 term, struct scoutfs_quorum_elected_info *qei); + u64 term); void scoutfs_server_abort(struct super_block *sb); -void scoutfs_server_stop(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei); +void scoutfs_server_stop(struct super_block *sb); int scoutfs_server_setup(struct super_block *sb); void scoutfs_server_destroy(struct super_block *sb); diff --git a/kmod/src/super.c b/kmod/src/super.c index e65f0c2a..1ac127ff 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -200,7 +200,6 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_shutdown_trans(sb); scoutfs_client_destroy(sb); - scoutfs_quorum_destroy(sb); scoutfs_inode_destroy(sb); /* the server locks the listen address and compacts */ @@ -210,6 +209,9 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_seg_destroy(sb); scoutfs_lock_destroy(sb); + /* server clears quorum leader flag during shutdown */ + scoutfs_quorum_destroy(sb); + scoutfs_item_destroy(sb); scoutfs_destroy_triggers(sb); scoutfs_options_destroy(sb); @@ -319,6 +321,16 @@ int scoutfs_read_super(struct super_block *sb, goto out; } + /* XXX do we want more rigorous invalid super checking? */ + + if (super->quorum_count == 0 || + super->quorum_count > SCOUTFS_QUORUM_MAX_COUNT) { + scoutfs_err(sb, "super block has invalid quorum count %u, must be > 0 and <= %u", + super->quorum_count, SCOUTFS_QUORUM_MAX_COUNT); + ret = -EINVAL; + goto out; + } + *super_res = *super; ret = 0; out: