mirrors/scoutfs
1
0
Fork 0
mirror of https://github.com/versity/scoutfs.git synced 2026-01-04 11:24:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

More accurately describe unmounting quorum members

Browse Source 
As a client unmounts it sends a farewell request to the server.  We have
to carefully manage unmounting the final quorum members so that there is
always a remaining quorum to elect a leader to start a server to process
all their farewell requests.

The mechanism for doing this described these clients as "voters".
That's not really right, in our terminology voters and candidates are
temporary roles taken on by members during a specific election term in
the raft protocol.  It's more accurate to describe the final set of
clients as quorum members.  They can be voters or candidates depending
on how the raft protocol timeouts workout in any given election.

So we rename the greeting flag, mounted client flag, and the code and
comments on either side of the client and server to be a little clearer.

This only changes symbols and comments, there should be no functional
change.

Signed-off-by: Zach Brown <zab@versity.com>
This commit is contained in:
Zach Brown
2021-01-29 11:38:36 -08:00
parent 3ad18b0f3b
commit 1c7bbd6260
3 changed files with 39 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
kmod/src/client.c
View File

@@ -305,25 +305,25 @@ out:
*
* In the typical case a mount reads the super blocks and finds the
* address of the currently running server and connects to it.
* Non-voting clients who can't connect will keep trying alternating
* reading the address and getting connect timeouts.
* Non-quorum member clients who can't connect will keep trying
* alternating reading the address and getting connect timeouts.
*
* Voting mounts will try to elect a leader if they can't connect to the
* server. When a quorum can't connect and are able to elect a leader
* Quorum members will try to elect a leader if they can't connect to
* the server. When then can't connect and are able to elect a leader
* then a new server is started. The new server will write its address
* in the super and everyone will be able to connect.
*
* There's a tricky bit of coordination required to safely unmount.
* Clients need to tell the server that they won't be coming back with a
* farewell request. Once a client receives its farewell response it
* can exit. But a majority of clients need to stick around to elect a
* server to process all their farewell requests. This is coordinated
* by having the greeting tell the server that a client is a voter. The
* server then holds on to farewell requests from voters until only
* requests from the final quorum remain. These farewell responses are
* only sent after updating an unmount barrier in the super to indicate
* to the final quorum that they can safely exit without having received
* a farewell response over the network.
* can exit. But a majority of quorum members need to stick around to
* elect a server to process all their farewell requests. This is
* coordinated by having the greeting tell the server that a client is a
* quorum member. The server then holds on to farewell requests from
* members until only requests from the final quorum remain. These
* farewell responses are only sent after updating an unmount barrier in
* the super to indicate to the final quorum that they can safely exit
* without having received a farewell response over the network.
*/
static void scoutfs_client_connect_worker(struct work_struct *work)
{
@@ -333,7 +333,7 @@ static void scoutfs_client_connect_worker(struct work_struct *work)
struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
struct scoutfs_super_block *super = NULL;
struct mount_options *opts = &sbi->opts;
const bool am_voter = opts->server_addr.sin_addr.s_addr != 0;
const bool am_quorum = opts->server_addr.sin_addr.s_addr != 0;
struct scoutfs_net_greeting greet;
struct sockaddr_in sin;
ktime_t timeout_abs;
@@ -351,7 +351,7 @@ static void scoutfs_client_connect_worker(struct work_struct *work)
goto out;
/* can safely unmount if we see that server processed our farewell */
if (am_voter && client->sending_farewell &&
if (am_quorum && client->sending_farewell &&
(le64_to_cpu(super->unmount_barrier) > client->greeting_umb)) {
client->farewell_error = 0;
complete(&client->farewell_comp);
@@ -367,12 +367,12 @@ static void scoutfs_client_connect_worker(struct work_struct *work)
else
ret = -ENOTCONN;
/* voters try to elect a leader if they couldn't connect */
if (ret < 0) {
/* non-voters will keep retrying */
if (!am_voter)
/* non-quorum members will delay then retry connect */
if (!am_quorum)
goto out;
/* quorum members try to elect a leader */
/* make sure local server isn't writing super during votes */
scoutfs_server_stop(sb);
@@ -399,8 +399,8 @@ static void scoutfs_client_connect_worker(struct work_struct *work)
greet.flags = 0;
if (client->sending_farewell)
greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL);
if (am_voter)
greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_VOTER);
if (am_quorum)
greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_QUORUM);
ret = scoutfs_net_submit_request(sb, client->conn,
SCOUTFS_NET_CMD_GREETING,

4
kmod/src/format.h
View File

@@ -306,7 +306,7 @@ struct scoutfs_mounted_client_btree_val {
__u8 flags;
};
#define SCOUTFS_MOUNTED_CLIENT_VOTER (1 << 0)
#define SCOUTFS_MOUNTED_CLIENT_QUORUM (1 << 0)
/*
* srch files are a contiguous run of blocks with compressed entries
@@ -757,7 +757,7 @@ struct scoutfs_net_greeting {
};
#define SCOUTFS_NET_GREETING_FLAG_FAREWELL (1 << 0)
#define SCOUTFS_NET_GREETING_FLAG_VOTER (1 << 1)
#define SCOUTFS_NET_GREETING_FLAG_QUORUM (1 << 1)
#define SCOUTFS_NET_GREETING_FLAG_INVALID (~(__u64)0 << 2)
/*

40
kmod/src/server.c
View File

@@ -1034,8 +1034,8 @@ static int insert_mounted_client(struct super_block *sb, u64 rid,
init_mounted_client_key(&key, rid);
mcv.flags = 0;
if (gr_flags & SCOUTFS_NET_GREETING_FLAG_VOTER)
mcv.flags |= SCOUTFS_MOUNTED_CLIENT_VOTER;
if (gr_flags & SCOUTFS_NET_GREETING_FLAG_QUORUM)
mcv.flags |= SCOUTFS_MOUNTED_CLIENT_QUORUM;
return scoutfs_btree_insert(sb, &server->alloc, &server->wri,
&super->mounted_clients, &key, &mcv,
@@ -1047,9 +1047,6 @@ static int insert_mounted_client(struct super_block *sb, u64 rid,
* removed if we're processing a farewell on behalf of a client that
* already had a previous server process its farewell.
*
* When we remove the last mounted client that's voting we write a new
* quorum block with the updated unmount_barrier.
*
* The caller has to serialize with farewell processing.
*/
static int delete_mounted_client(struct super_block *sb, u64 rid)
@@ -1259,19 +1256,18 @@ static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref)
/*
* This work processes farewell requests asynchronously. Requests from
* voting clients can be held until only the final quorum remains and
* quorum members can be held until only the final quorum remains and
* they've all sent farewell requests.
*
* When we remove the last mounted client record for the last voting
* client then we increase the unmount_barrier and write it to the super
* block. If voting clients don't get their farewell response they'll
* see the greater umount_barrier in the super and will know that their
* farewell has been processed and that they can exit.
* When we remove the last mounted client record for the last quorum
* member then we increase the unmount_barrier and write it to the super
* block. If members don't get their farewell response they'll see the
* greater umount_barrier in the super and will know that their farewell
* has been processed and that they can exit.
*
* Responses that are waiting for clients who aren't voting are
* immediately sent. Clients that don't have a mounted client record
* have already had their farewell processed by another server and can
* proceed.
* Responses for clients who aren't members are immediately sent.
* Clients that don't have a mounted client record have already had
* their farewell processed by another server and can proceed.
*
* Farewell responses are unique in that sending them causes the server
* to shutdown the connection to the client next time the socket
@@ -1299,7 +1295,7 @@ static void farewell_worker(struct work_struct *work)
LIST_HEAD(reqs);
LIST_HEAD(send);
bool deleted = false;
bool voting;
bool is_quorum;
bool more_reqs;
int ret;
@@ -1308,7 +1304,7 @@ static void farewell_worker(struct work_struct *work)
list_splice_init(&server->farewell_requests, &reqs);
mutex_unlock(&server->farewell_mutex);
/* count how many reqs requests are from voting clients */
/* count how many reqs requests are from quorum members */
nr_unmounting = 0;
list_for_each_entry_safe(fw, tmp, &reqs, entry) {
init_mounted_client_key(&key, fw->rid);
@@ -1327,10 +1323,10 @@ static void farewell_worker(struct work_struct *work)
}
mcv = iref.val;
voting = (mcv->flags & SCOUTFS_MOUNTED_CLIENT_VOTER) != 0;
is_quorum = (mcv->flags & SCOUTFS_MOUNTED_CLIENT_QUORUM) != 0;
scoutfs_btree_put_iref(&iref);
if (!voting) {
if (!is_quorum) {
list_move_tail(&fw->entry, &send);
continue;
}
@@ -1338,7 +1334,7 @@ static void farewell_worker(struct work_struct *work)
nr_unmounting++;
}
/* see how many mounted clients could vote for quorum */
/* see how many mounted clients are quorum members */
init_mounted_client_key(&key, 0);
for (;;) {
ret = scoutfs_btree_next(sb, &super->mounted_clients, &key,
@@ -1356,7 +1352,7 @@ static void farewell_worker(struct work_struct *work)
key = *iref.key;
mcv = iref.val;
if (mcv->flags & SCOUTFS_MOUNTED_CLIENT_VOTER)
if (mcv->flags & SCOUTFS_MOUNTED_CLIENT_QUORUM)
nr_mounted++;
scoutfs_btree_put_iref(&iref);
@@ -1392,7 +1388,7 @@ static void farewell_worker(struct work_struct *work)
goto out;
}
/* update the unmount barrier if we deleted all voting clients */
/* update the unmount barrier if we deleted all quorum members */
if (deleted && nr_mounted == 0) {
ret = scoutfs_server_hold_commit(sb);
if (ret)