mirrors/scoutfs
1
0
Fork 0
mirror of https://github.com/versity/scoutfs.git synced 2026-06-08 04:32:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

scoutfs: replace node_id with rid in networking

Browse Source 
Use the client's rid in networking instead of the node_id.

The node_id no longer has to be allocated by the server and sent in the
greeting.  Instead the client sends it to the server in its greeting.

The server then uses the client's announced rid just like it used to use
the its node_id.  It's used to record clients in the btree and to
identify clients in sending and receive processing.

The use of the rid in networking calls makes its way to locking and
compaction which now use the rid to identify clients intead of the
node_id.

Signed-off-by: Zach Brown <zab@versity.com>
This commit is contained in:
Zach Brown
2019-07-10 14:06:18 -07:00
committed by Zach Brown
parent f0a86f05f8
commit ab7bde9e2c
12 changed files with 251 additions and 302 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
kmod/src/client.c
View File

@@ -49,7 +49,6 @@ struct client_info {
struct super_block *sb;
struct scoutfs_net_connection *conn;
struct completion node_id_comp;
atomic_t shutting_down;
struct workqueue_struct *workq;
@@ -299,8 +298,9 @@ static int client_lock_recover(struct super_block *sb,
/*
* Process a greeting response in the client from the server. This is
* called for every connected socket on the connection. The first
* response will have the node_id that the server assigned the client.
* called for every connected socket on the connection. Each response
* contains the remote server's elected term which can be used to
* identify server failover.
*/
static int client_greeting(struct super_block *sb,
struct scoutfs_net_connection *conn,
@@ -309,7 +309,6 @@ static int client_greeting(struct super_block *sb,
{
struct client_info *client = SCOUTFS_SB(sb)->client_info;
struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
struct scoutfs_net_greeting *gr = resp;
bool new_server;
int ret;
@@ -340,25 +339,6 @@ static int client_greeting(struct super_block *sb,
goto out;
}
if (sbi->node_id != 0 && le64_to_cpu(gr->node_id) != sbi->node_id) {
scoutfs_warn(sb, "server sent node_id %llu, client has %llu",
le64_to_cpu(gr->node_id),
sbi->node_id);
ret = -EINVAL;
goto out;
}
if (sbi->node_id == 0 && gr->node_id == 0) {
scoutfs_warn(sb, "server sent node_id 0, client also has 0");
ret = -EINVAL;
goto out;
}
if (sbi->node_id == 0) {
sbi->node_id = le64_to_cpu(gr->node_id);
complete(&client->node_id_comp);
}
new_server = le64_to_cpu(gr->server_term) != client->server_term;
scoutfs_net_client_greeting(sb, conn, new_server);
@@ -466,7 +446,7 @@ static void scoutfs_client_connect_worker(struct work_struct *work)
greet.format_hash = super->format_hash;
greet.server_term = cpu_to_le64(client->server_term);
greet.unmount_barrier = cpu_to_le64(client->greeting_umb);
greet.node_id = cpu_to_le64(sbi->node_id);
greet.rid = cpu_to_le64(sbi->rid);
greet.flags = 0;
if (client->sending_farewell)
greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL);
@@ -553,7 +533,7 @@ static scoutfs_net_request_t client_req_funcs[] = {
*/
static void client_notify_down(struct super_block *sb,
struct scoutfs_net_connection *conn, void *info,
u64 node_id)
u64 rid)
{
struct client_info *client = SCOUTFS_SB(sb)->client_info;
@@ -561,18 +541,6 @@ static void client_notify_down(struct super_block *sb,
queue_delayed_work(client->workq, &client->connect_dwork, 0);
}
/*
* Wait for the first connected socket on the connection that assigns
* the node_id that will be used for the rest of the life time of the
* mount.
*/
int scoutfs_client_wait_node_id(struct super_block *sb)
{
struct client_info *client = SCOUTFS_SB(sb)->client_info;
return wait_for_completion_interruptible(&client->node_id_comp);
}
int scoutfs_client_setup(struct super_block *sb)
{
struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
@@ -587,7 +555,6 @@ int scoutfs_client_setup(struct super_block *sb)
sbi->client_info = client;
client->sb = sb;
init_completion(&client->node_id_comp);
atomic_set(&client->shutting_down, 0);
INIT_DELAYED_WORK(&client->connect_dwork,
scoutfs_client_connect_worker);
@@ -640,12 +607,12 @@ static int client_farewell_response(struct super_block *sb,
* so that they don't wait for us to reconnect and trigger a timeout.
*
* This decision is a little racy. The server considers us connected
* when it assigns us a node_id as it processes the greeting. We can
* disconnect before receiving the response and leave without sending a
* farewell. So given that awkward initial race, we also have a bit of
* a race where we just test the server_term to see if we've ever gotten
* a greeting reply from any server. We don't try to synchronize with
* pending connection attempts.
* when it records a persistent record of our rid as it processes our
* greeting. We can disconnect before receiving the greeting response
* and leave without sending a farewell. So given that awkward initial
* race, we also have a bit of a race where we just test the server_term
* to see if we've ever gotten a greeting reply from any server. We
* don't try to synchronize with pending connection attempts.
*
* The consequences of aborting a mount at just the wrong time and
* disconnecting without the farewell handshake depend on what the

1
kmod/src/client.h
View File

@@ -24,7 +24,6 @@ int scoutfs_client_lock_response(struct super_block *sb, u64 net_id,
int scoutfs_client_lock_recover_response(struct super_block *sb, u64 net_id,
struct scoutfs_net_lock_recover *nlr);
int scoutfs_client_wait_node_id(struct super_block *sb);
int scoutfs_client_setup(struct super_block *sb);
void scoutfs_client_destroy(struct super_block *sb);

22
kmod/src/format.h
View File

@@ -287,7 +287,7 @@ struct scoutfs_extent_btree_key {
* server failover knows who to wait for before resuming operations.
*/
struct scoutfs_lock_client_btree_key {
__be64 node_id;
__be64 rid;
} __packed;
/*
@@ -296,14 +296,14 @@ struct scoutfs_lock_client_btree_key {
*/
struct scoutfs_trans_seq_btree_key {
__be64 trans_seq;
__be64 node_id;
__be64 rid;
} __packed;
/*
* The server keeps a persistent record of mounted clients.
*/
struct scoutfs_mounted_client_btree_key {
__be64 node_id;
__be64 rid;
} __packed;
struct scoutfs_mounted_client_btree_val {
@@ -479,7 +479,6 @@ struct scoutfs_super_block {
__le64 alloc_cursor;
struct scoutfs_btree_ring bring;
__le64 next_seg_seq;
__le64 next_node_id;
__le64 next_compact_id;
__le64 quorum_fenced_term;
__le64 quorum_server_term;
@@ -611,8 +610,9 @@ enum {
*
* @server_term: The raft term that elected the server. Initially 0
* from the client, sent by the server, then sent by the client as it
* tries to reconnect. Used to identify a client reconnecting to a
* server that has timed out its connection.
* tries to reconnect. Used to identify a client reconnecting to both
* the same serer after receiving a greeting response and to a new
* server after failover.
*
* @unmount_barrier: Incremented every time the remaining majority of
* quorum members all agree to leave. The server tells a quorum member
@@ -620,17 +620,17 @@ enum {
* value increase in the super block then it knows that the server has
* processed its farewell and can safely unmount.
*
* @node_id: The id of the client. Initially 0 from the client,
* assigned by the server, and sent by the client as it reconnects.
* Used by the server to identify reconnecting clients whose existing
* state must be dealt with.
* @rid: The client's random id that was generated once as the mount
* started up. This identifies a specific remote mount across
* connections and servers. It's set to the client's rid in both the
* request and response for consistency.
*/
struct scoutfs_net_greeting {
__le64 fsid;
__le64 format_hash;
__le64 server_term;
__le64 unmount_barrier;
__le64 node_id;
__le64 rid;
__le64 flags;
} __packed;

111
kmod/src/lock_server.c
View File

@@ -50,10 +50,10 @@
* server doesn't use the modes specified by the clients but they're
* provided to add context.
*
* The server relies on the node_id allocation and reliable messaging
* layers of the system. Each client has a node_id that is unique for
* its life time. Message requests and responses are reliably
* delivered in order across reconnection.
* The server relies on the client's static rid and on reliable
* messaging. Each client has a rid that is unique for its life time.
* Message requests and responses are reliably delivered in order across
* reconnection.
*
* The server maintains a persistent record of connected clients. A new
* server instance discovers these and waits for previously connected
@@ -126,7 +126,7 @@ enum {
* that the mode is actively granted, a pending request from the client,
* or a pending invalidation sent to the client.
*
* @node_id: The client's node_id used to send messages and tear down
* @rid: The client's rid used to send messages and tear down
* state as client's exit.
*
* @net_id: The id of a client's request used to send grant responses. The
@@ -139,7 +139,7 @@ enum {
*/
struct client_lock_entry {
struct list_head head;
u64 node_id;
u64 rid;
u64 net_id;
u8 mode;
@@ -221,7 +221,7 @@ static bool client_entries_compatible(struct client_lock_entry *granted,
return (granted->mode == requested->mode &&
(granted->mode == SCOUTFS_LOCK_READ ||
granted->mode == SCOUTFS_LOCK_WRITE_ONLY)) ||
(granted->node_id == requested->node_id &&
(granted->rid == requested->rid &&
granted->mode == SCOUTFS_LOCK_READ &&
requested->mode == SCOUTFS_LOCK_WRITE);
}
@@ -340,14 +340,14 @@ static void put_server_lock(struct lock_server_info *inf,
static struct client_lock_entry *find_entry(struct server_lock_node *snode,
struct list_head *list,
u64 node_id)
u64 rid)
{
struct client_lock_entry *clent;
WARN_ON_ONCE(!mutex_is_locked(&snode->mutex));
list_for_each_entry(clent, list, head) {
if (clent->node_id == node_id)
if (clent->rid == rid)
return clent;
}
@@ -363,7 +363,7 @@ static int process_waiting_requests(struct super_block *sb,
*
* XXX shut down if we get enomem?
*/
int scoutfs_lock_server_request(struct super_block *sb, u64 node_id,
int scoutfs_lock_server_request(struct super_block *sb, u64 rid,
u64 net_id, struct scoutfs_net_lock *nl)
{
DECLARE_LOCK_SERVER_INFO(sb, inf);
@@ -372,7 +372,7 @@ int scoutfs_lock_server_request(struct super_block *sb, u64 node_id,
int ret;
trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_GRANT, SLT_REQUEST,
node_id, net_id, nl);
rid, net_id, nl);
if (invalid_mode(nl->old_mode) || invalid_mode(nl->new_mode)) {
ret = -EINVAL;
@@ -386,7 +386,7 @@ int scoutfs_lock_server_request(struct super_block *sb, u64 node_id,
}
INIT_LIST_HEAD(&clent->head);
clent->node_id = node_id;
clent->rid = rid;
clent->net_id = net_id;
clent->mode = nl->new_mode;
@@ -414,7 +414,7 @@ out:
*
* XXX what to do with errors? kick the client?
*/
int scoutfs_lock_server_response(struct super_block *sb, u64 node_id,
int scoutfs_lock_server_response(struct super_block *sb, u64 rid,
struct scoutfs_net_lock *nl)
{
DECLARE_LOCK_SERVER_INFO(sb, inf);
@@ -423,7 +423,7 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 node_id,
int ret;
trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_INVALIDATE, SLT_RESPONSE,
node_id, 0, nl);
rid, 0, nl);
if (invalid_mode(nl->old_mode) || invalid_mode(nl->new_mode)) {
ret = -EINVAL;
@@ -437,7 +437,7 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 node_id,
goto out;
}
clent = find_entry(snode, &snode->invalidated, node_id);
clent = find_entry(snode, &snode->invalidated, rid);
if (!clent) {
put_server_lock(inf, snode);
ret = -EINVAL;
@@ -512,13 +512,13 @@ static int process_waiting_requests(struct super_block *sb,
nl.old_mode = gr->mode;
nl.new_mode = invalidation_mode(gr->mode, req->mode);
ret = scoutfs_server_lock_request(sb, gr->node_id, &nl);
ret = scoutfs_server_lock_request(sb, gr->rid, &nl);
if (ret)
goto out;
trace_scoutfs_lock_message(sb, SLT_SERVER,
SLT_INVALIDATE, SLT_REQUEST,
gr->node_id, 0, &nl);
gr->rid, 0, &nl);
add_client_entry(snode, &snode->invalidated, gr);
}
@@ -531,7 +531,7 @@ static int process_waiting_requests(struct super_block *sb,
nl.new_mode = req->mode;
/* see if there's an existing compatible grant to replace */
gr = find_entry(snode, &snode->granted, req->node_id);
gr = find_entry(snode, &snode->granted, req->rid);
if (gr) {
nl.old_mode = gr->mode;
free_client_entry(inf, snode, gr);
@@ -539,13 +539,13 @@ static int process_waiting_requests(struct super_block *sb,
nl.old_mode = SCOUTFS_LOCK_NULL;
}
ret = scoutfs_server_lock_response(sb, req->node_id,
ret = scoutfs_server_lock_response(sb, req->rid,
req->net_id, &nl);
if (ret)
goto out;
trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_GRANT,
SLT_RESPONSE, req->node_id,
SLT_RESPONSE, req->rid,
req->net_id, &nl);
/* don't track null client locks, track all else */
@@ -571,7 +571,7 @@ out:
*
* This is running in concurrent client greeting processing contexts.
*/
int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id,
int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid,
bool should_exist)
{
DECLARE_LOCK_SERVER_INFO(sb, inf);
@@ -581,7 +581,7 @@ int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id,
struct scoutfs_key key;
int ret;
cbk.node_id = cpu_to_be64(node_id);
cbk.rid = cpu_to_be64(rid);
mutex_lock(&inf->mutex);
if (should_exist) {
@@ -597,7 +597,7 @@ int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id,
if (should_exist && ret == 0) {
scoutfs_key_set_zeros(&key);
ret = scoutfs_server_lock_recover_request(sb, node_id, &key);
ret = scoutfs_server_lock_recover_request(sb, rid, &key);
if (ret)
goto out;
}
@@ -611,7 +611,7 @@ out:
* they were the last client in recovery then we can process all the
* server locks that had requests.
*/
static int finished_recovery(struct super_block *sb, u64 node_id, bool cancel)
static int finished_recovery(struct super_block *sb, u64 rid, bool cancel)
{
DECLARE_LOCK_SERVER_INFO(sb, inf);
struct server_lock_node *snode;
@@ -620,7 +620,7 @@ static int finished_recovery(struct super_block *sb, u64 node_id, bool cancel)
int ret = 0;
spin_lock(&inf->lock);
scoutfs_spbm_clear(&inf->recovery_pending, node_id);
scoutfs_spbm_clear(&inf->recovery_pending, rid);
still_pending = !scoutfs_spbm_empty(&inf->recovery_pending);
spin_unlock(&inf->lock);
if (still_pending)
@@ -654,7 +654,7 @@ static int finished_recovery(struct super_block *sb, u64 node_id, bool cancel)
* gave us in response and send another request from the next key.
* We're done once we receive an empty response.
*/
int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id,
int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid,
struct scoutfs_net_lock_recover *nlr)
{
DECLARE_LOCK_SERVER_INFO(sb, inf);
@@ -667,7 +667,7 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id,
/* client must be in recovery */
spin_lock(&inf->lock);
if (!scoutfs_spbm_test(&inf->recovery_pending, node_id))
if (!scoutfs_spbm_test(&inf->recovery_pending, rid))
ret = -EINVAL;
spin_unlock(&inf->lock);
if (ret)
@@ -675,7 +675,7 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id,
/* client has sent us all their locks */
if (nlr->nr == 0) {
ret = finished_recovery(sb, node_id, true);
ret = finished_recovery(sb, rid, true);
goto out;
}
@@ -687,7 +687,7 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id,
}
INIT_LIST_HEAD(&clent->head);
clent->node_id = node_id;
clent->rid = rid;
clent->net_id = 0;
clent->mode = nlr->locks[i].new_mode;
@@ -698,7 +698,7 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id,
goto out;
}
existing = find_entry(snode, &snode->granted, node_id);
existing = find_entry(snode, &snode->granted, rid);
if (existing) {
kfree(clent);
put_server_lock(inf, snode);
@@ -717,20 +717,20 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id,
key = nlr->locks[le16_to_cpu(nlr->nr) - 1].key;
scoutfs_key_inc(&key);
ret = scoutfs_server_lock_recover_request(sb, node_id, &key);
ret = scoutfs_server_lock_recover_request(sb, rid, &key);
out:
return ret;
}
static int node_id_and_put_iref(struct scoutfs_btree_item_ref *iref,
u64 *node_id)
static int get_rid_and_put_ref(struct scoutfs_btree_item_ref *iref,
u64 *rid)
{
struct scoutfs_lock_client_btree_key *cbk;
int ret;
if (iref->key_len == sizeof(*cbk) && iref->val_len == 0) {
cbk = iref->key;
*node_id = be64_to_cpu(cbk->node_id);
*rid = be64_to_cpu(cbk->rid);
ret = 0;
} else {
ret = -EIO;
@@ -754,12 +754,12 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work)
struct scoutfs_lock_client_btree_key cbk;
SCOUTFS_BTREE_ITEM_REF(iref);
bool timed_out;
u64 node_id;
u64 rid;
int ret;
/* we enter recovery if there are any client records */
for (node_id = 0; ; node_id++) {
cbk.node_id = cpu_to_be64(node_id);
for (rid = 0; ; rid++) {
cbk.rid = cpu_to_be64(rid);
ret = scoutfs_btree_next(sb, &super->lock_clients,
&cbk, sizeof(cbk), &iref);
if (ret == -ENOENT) {
@@ -767,13 +767,13 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work)
break;
}
if (ret == 0)
ret = node_id_and_put_iref(&iref, &node_id);
ret = get_rid_and_put_ref(&iref, &rid);
if (ret < 0)
break;
spin_lock(&inf->lock);
if (scoutfs_spbm_test(&inf->recovery_pending, node_id)) {
scoutfs_spbm_clear(&inf->recovery_pending, node_id);
if (scoutfs_spbm_test(&inf->recovery_pending, rid)) {
scoutfs_spbm_clear(&inf->recovery_pending, rid);
timed_out = true;
} else {
timed_out = false;
@@ -783,11 +783,11 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work)
if (!timed_out)
continue;
scoutfs_err(sb, "client node_id %llu lock recovery timed out",
node_id);
scoutfs_err(sb, "client rid %016llx lock recovery timed out",
rid);
/* XXX these aren't immediately committed */
cbk.node_id = cpu_to_be64(node_id);
cbk.rid = cpu_to_be64(rid);
ret = scoutfs_btree_delete(sb, &super->lock_clients,
&cbk, sizeof(cbk));
if (ret)
@@ -812,7 +812,7 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work)
* If we fail to delete a persistent entry then we have to shut down and
* hope that the next server has more luck.
*/
int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id)
int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid)
{
DECLARE_LOCK_SERVER_INFO(sb, inf);
struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
@@ -825,7 +825,7 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id)
bool freed;
int ret = 0;
cli.node_id = cpu_to_be64(node_id);
cli.rid = cpu_to_be64(rid);
mutex_lock(&inf->mutex);
ret = scoutfs_btree_delete(sb, &super->lock_clients, &cli, sizeof(cli));
mutex_unlock(&inf->mutex);
@@ -847,7 +847,7 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id)
NULL) {
list_for_each_entry_safe(clent, tmp, list, head) {
if (clent->node_id == node_id) {
if (clent->rid == rid) {
free_client_entry(inf, snode, clent);
freed = true;
}
@@ -869,7 +869,8 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id)
out:
if (ret < 0) {
scoutfs_err(sb, "lock server err %d during node %llu farewell, shutting down", ret, node_id);
scoutfs_err(sb, "lock server err %d during client rid %016llx farewell, shutting down",
ret, rid);
scoutfs_server_abort(sb);
}
@@ -913,9 +914,9 @@ static void lock_server_tseq_show(struct seq_file *m,
tseq_entry);
struct server_lock_node *snode = clent->snode;
seq_printf(m, SK_FMT" %s %s node_id %llu net_id %llu\n",
seq_printf(m, SK_FMT" %s %s rid %016llx net_id %llu\n",
SK_ARG(&snode->key), lock_mode_string(clent->mode),
lock_on_list_string(clent->on_list), clent->node_id,
lock_on_list_string(clent->on_list), clent->rid,
clent->net_id);
}
@@ -934,7 +935,7 @@ int scoutfs_lock_server_setup(struct super_block *sb)
SCOUTFS_BTREE_ITEM_REF(iref);
struct scoutfs_lock_client_btree_key cbk;
unsigned int nr;
u64 node_id;
u64 rid;
int ret;
inf = kzalloc(sizeof(struct lock_server_info), GFP_KERNEL);
@@ -961,23 +962,23 @@ int scoutfs_lock_server_setup(struct super_block *sb)
/* we enter recovery if there are any client records */
nr = 0;
for (node_id = 0; ; node_id++) {
cbk.node_id = cpu_to_be64(node_id);
for (rid = 0; ; rid++) {
cbk.rid = cpu_to_be64(rid);
ret = scoutfs_btree_next(sb, &super->lock_clients,
&cbk, sizeof(cbk), &iref);
if (ret == -ENOENT)
break;
if (ret == 0)
ret = node_id_and_put_iref(&iref, &node_id);
ret = get_rid_and_put_ref(&iref, &rid);
if (ret < 0)
goto out;
ret = scoutfs_spbm_set(&inf->recovery_pending, node_id);
ret = scoutfs_spbm_set(&inf->recovery_pending, rid);
if (ret)
goto out;
nr++;
if (node_id == U64_MAX)
if (rid == U64_MAX)
break;
}
ret = 0;

10
kmod/src/lock_server.h
View File

@@ -1,15 +1,15 @@
#ifndef _SCOUTFS_LOCK_SERVER_H_
#define _SCOUTFS_LOCK_SERVER_H_
int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id,
int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid,
struct scoutfs_net_lock_recover *nlr);
int scoutfs_lock_server_request(struct super_block *sb, u64 node_id,
int scoutfs_lock_server_request(struct super_block *sb, u64 rid,
u64 net_id, struct scoutfs_net_lock *nl);
int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id,
int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid,
bool should_exist);
int scoutfs_lock_server_response(struct super_block *sb, u64 node_id,
int scoutfs_lock_server_response(struct super_block *sb, u64 rid,
struct scoutfs_net_lock *nl);
int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id);
int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid);
int scoutfs_lock_server_setup(struct super_block *sb);
void scoutfs_lock_server_destroy(struct super_block *sb);

80
kmod/src/net.c
View File

@@ -109,7 +109,7 @@ struct scoutfs_net_connection {
unsigned long connect_timeout_ms;
struct socket *sock;
u64 node_id; /* assigned during greeting */
u64 rid;
u64 greeting_id;
struct sockaddr_in sockname;
struct sockaddr_in peername;
@@ -344,12 +344,12 @@ static void shutdown_conn(struct scoutfs_net_connection *conn)
* connection has passed the greeting and isn't being shut down. At all
* other times we add new sends to the resend queue.
*
* If a non-zero node_id is specified then the conn argument is a listening
* If a non-zero rid is specified then the conn argument is a listening
* connection and the connection to send the message down is found by
* searching for the node_id in its accepted connections.
* searching for the rid in its accepted connections.
*/
static int submit_send(struct super_block *sb,
struct scoutfs_net_connection *conn, u64 node_id,
struct scoutfs_net_connection *conn, u64 rid,
u8 cmd, u8 flags, u64 id, u8 net_err,
void *data, u16 data_len,
scoutfs_net_response_t resp_func, void *resp_data,
@@ -376,19 +376,19 @@ static int submit_send(struct super_block *sb,
spin_lock_nested(&conn->lock, CONN_LOCK_LISTENER);
if (node_id != 0) {
if (rid != 0) {
list_for_each_entry(acc_conn, &conn->accepted_list,
accepted_head) {
if (acc_conn->node_id == node_id) {
if (acc_conn->rid == rid) {
spin_lock_nested(&acc_conn->lock,
CONN_LOCK_ACCEPTED);
spin_unlock(&conn->lock);
conn = acc_conn;
node_id = 0;
rid = 0;
break;
}
}
if (node_id != 0) {
if (rid != 0) {
spin_unlock(&conn->lock);
return -ENOTCONN;
}
@@ -845,7 +845,7 @@ static void destroy_conn(struct scoutfs_net_connection *conn)
/* tell callers that accepted connection finally done */
if (conn->listening_conn && conn->notify_down)
conn->notify_down(sb, conn, conn->info, conn->node_id);
conn->notify_down(sb, conn, conn->info, conn->rid);
/* free all messages, refactor and complete for forced unmount? */
list_splice_init(&conn->resend_queue, &conn->send_queue);
@@ -1182,7 +1182,7 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work)
conn->shutting_down = 0;
if (conn->notify_down)
conn->notify_down(sb, conn, conn->info,
conn->node_id);
conn->rid);
}
spin_unlock(&conn->lock);
@@ -1251,12 +1251,12 @@ restart:
* Accepted connections inherit the callbacks from their listening
* connection.
*
* notify_up is called once a valid greeting is received. node_id is
* notify_up is called once a valid greeting is received. rid is
* non-zero on accepted sockets once they've seen a valid greeting.
* Connected and listening connections have a node_id of 0.
* Connected and listening connections have a rid of 0.
*
* notify_down is always called as connections are shut down. It can be
* called without notify_up ever being called. The node_id is only
* called without notify_up ever being called. The rid is only
* non-zero for accepted connections.
*/
struct scoutfs_net_connection *
@@ -1317,14 +1317,15 @@ scoutfs_net_alloc_conn(struct super_block *sb,
}
/*
* Give the caller the client node_id of the connection. This used by
* rare server processing callers who want to send async responses after
* request processing has returned. We didn't want to plumb the
* requesting node_id into all the request handlers but that'd work too.
* Give the caller the client rid of the connection. This used by rare
* server processing callers who want to send async responses after
* request processing has returned. We didn't want the churn of
* providing the requesting rid to all the request handlers, but we
* probably should.
*/
u64 scoutfs_net_client_node_id(struct scoutfs_net_connection *conn)
u64 scoutfs_net_client_rid(struct scoutfs_net_connection *conn)
{
return conn->node_id;
return conn->rid;
}
/*
@@ -1520,7 +1521,7 @@ void scoutfs_net_client_greeting(struct super_block *sb,
/*
* The calling server has received a valid greeting from a client. If
* the server is reconnecting to us then we need to find its old
* the client is reconnecting to us then we need to find its old
* connection that held its state and transfer it to this connection
* (connection and socket life cycles make this easier than migrating
* the socket between the connections).
@@ -1532,9 +1533,8 @@ void scoutfs_net_client_greeting(struct super_block *sb,
* referring to the same original connection. We use the increasing
* greeting id to have the most recent connection attempt win.
*
* A node can be reconnecting to us for the first time. In that case we
* just trust its node_id. It will notice the new server term and take
* steps to recover.
* A node can be reconnecting to us for the first time. It will notice
* the new server term and take steps to recover.
*
* A client can be reconnecting to us after we've destroyed their state.
* This is fatal for the client if they just took too long to reconnect.
@@ -1550,8 +1550,8 @@ void scoutfs_net_client_greeting(struct super_block *sb,
*/
void scoutfs_net_server_greeting(struct super_block *sb,
struct scoutfs_net_connection *conn,
u64 node_id, u64 greeting_id,
bool sent_node_id, bool first_contact,
u64 rid, u64 greeting_id,
bool reconnecting, bool first_contact,
bool farewell)
{
struct scoutfs_net_connection *listener;
@@ -1561,15 +1561,15 @@ void scoutfs_net_server_greeting(struct super_block *sb,
/* only called on accepted server connections :/ */
BUG_ON(!conn->listening_conn);
/* see if we have a previous conn for the client's sent node_id */
/* see if we have a previous conn for the client's sent rid */
reconn = NULL;
if (sent_node_id) {
if (reconnecting) {
listener = conn->listening_conn;
restart:
spin_lock_nested(&listener->lock, CONN_LOCK_LISTENER);
list_for_each_entry(acc, &listener->accepted_list,
accepted_head) {
if (acc->node_id != node_id ||
if (acc->rid != rid ||
acc->greeting_id >= greeting_id ||
acc->reconn_freeing)
continue;
@@ -1592,12 +1592,12 @@ restart:
}
/* drop a connection if we can't find its necessary old conn */
if (sent_node_id && !reconn && !first_contact && !farewell) {
if (reconnecting && !reconn && !first_contact && !farewell) {
shutdown_conn(conn);
return;
}
/* migrate state from previous conn for this reconnecting node_id */
/* migrate state from previous conn for this reconnecting rid */
if (reconn) {
spin_lock(&conn->lock);
@@ -1623,15 +1623,15 @@ restart:
spin_lock(&conn->lock);
conn->node_id = node_id;
conn->rid = rid;
conn->greeting_id = greeting_id;
set_valid_greeting(conn);
spin_unlock(&conn->lock);
/* only call notify_up the first time we see the node_id */
/* only call notify_up the first time we see the rid */
if (conn->notify_up && first_contact)
conn->notify_up(sb, conn, conn->info, node_id);
conn->notify_up(sb, conn, conn->info, rid);
}
/*
@@ -1651,17 +1651,17 @@ int scoutfs_net_submit_request(struct super_block *sb,
}
/*
* Send a request to a specific node_id that was accepted by this listening
* Send a request to a specific rid that was accepted by this listening
* connection.
*/
int scoutfs_net_submit_request_node(struct super_block *sb,
struct scoutfs_net_connection *conn,
u64 node_id, u8 cmd,
u64 rid, u8 cmd,
void *arg, u16 arg_len,
scoutfs_net_response_t resp_func,
void *resp_data, u64 *id_ret)
{
return submit_send(sb, conn, node_id, cmd, 0, 0, 0, arg, arg_len,
return submit_send(sb, conn, rid, cmd, 0, 0, 0, arg, arg_len,
resp_func, resp_data, id_ret);
}
@@ -1687,7 +1687,7 @@ int scoutfs_net_response(struct super_block *sb,
int scoutfs_net_response_node(struct super_block *sb,
struct scoutfs_net_connection *conn,
u64 node_id, u8 cmd, u64 id, int error,
u64 rid, u8 cmd, u64 id, int error,
void *resp, u16 resp_len)
{
if (error) {
@@ -1695,7 +1695,7 @@ int scoutfs_net_response_node(struct super_block *sb,
resp_len = 0;
}
return submit_send(sb, conn, node_id, cmd, SCOUTFS_NET_FLAG_RESPONSE,
return submit_send(sb, conn, rid, cmd, SCOUTFS_NET_FLAG_RESPONSE,
id, net_err_from_host(sb, error), resp, resp_len,
NULL, NULL, NULL);
}
@@ -1787,9 +1787,9 @@ static void net_tseq_show_conn(struct seq_file *m,
struct scoutfs_net_connection *conn =
container_of(ent, struct scoutfs_net_connection, tseq_entry);
seq_printf(m, "name "SIN_FMT" peer "SIN_FMT" node_id %llu greeting_id %llu vg %u est %u sd %u sg %u sf %u rw %u rf %u cto_ms rdl_j %lu %lu nss %llu rs %llu nsi %llu\n",
seq_printf(m, "name "SIN_FMT" peer "SIN_FMT" rid %016llx greeting_id %llu vg %u est %u sd %u sg %u sf %u rw %u rf %u cto_ms rdl_j %lu %lu nss %llu rs %llu nsi %llu\n",
SIN_ARG(&conn->sockname), SIN_ARG(&conn->peername),
conn->node_id, conn->greeting_id, conn->valid_greeting,
conn->rid, conn->greeting_id, conn->valid_greeting,
conn->established, conn->shutting_down, conn->saw_greeting,
conn->saw_farewell, conn->reconn_wait, conn->reconn_freeing,
conn->connect_timeout_ms, conn->reconn_deadline,

13
kmod/src/net.h
View File

@@ -37,14 +37,14 @@ typedef int (*scoutfs_net_response_t)(struct super_block *sb,
typedef void (*scoutfs_net_notify_t)(struct super_block *sb,
struct scoutfs_net_connection *conn,
void *info, u64 node_id);
void *info, u64 rid);
struct scoutfs_net_connection *
scoutfs_net_alloc_conn(struct super_block *sb,
scoutfs_net_notify_t notify_up,
scoutfs_net_notify_t notify_down, size_t info_size,
scoutfs_net_request_t *req_funcs, char *name_suffix);
u64 scoutfs_net_client_node_id(struct scoutfs_net_connection *conn);
u64 scoutfs_net_client_rid(struct scoutfs_net_connection *conn);
int scoutfs_net_connect(struct super_block *sb,
struct scoutfs_net_connection *conn,
struct sockaddr_in *sin, unsigned long timeout_ms);
@@ -60,8 +60,7 @@ int scoutfs_net_submit_request(struct super_block *sb,
void *resp_data, u64 *id_ret);
int scoutfs_net_submit_request_node(struct super_block *sb,
struct scoutfs_net_connection *conn,
u64 node_id, u8 cmd,
void *arg, u16 arg_len,
u64 rid, u8 cmd, void *arg, u16 arg_len,
scoutfs_net_response_t resp_func,
void *resp_data, u64 *id_ret);
void scoutfs_net_cancel_request(struct super_block *sb,
@@ -76,7 +75,7 @@ int scoutfs_net_response(struct super_block *sb,
u8 cmd, u64 id, int error, void *resp, u16 resp_len);
int scoutfs_net_response_node(struct super_block *sb,
struct scoutfs_net_connection *conn,
u64 node_id, u8 cmd, u64 id, int error,
u64 rid, u8 cmd, u64 id, int error,
void *resp, u16 resp_len);
void scoutfs_net_shutdown(struct super_block *sb,
struct scoutfs_net_connection *conn);
@@ -88,8 +87,8 @@ void scoutfs_net_client_greeting(struct super_block *sb,
bool new_server);
void scoutfs_net_server_greeting(struct super_block *sb,
struct scoutfs_net_connection *conn,
u64 node_id, u64 greeting_id,
bool sent_node_id, bool first_contact,
u64 rid, u64 greeting_id,
bool reconnecting, bool first_contact,
bool farewell);
void scoutfs_net_farewell(struct super_block *sb,
struct scoutfs_net_connection *conn);

92
kmod/src/scoutfs_trace.h
View File

@@ -1154,17 +1154,17 @@ TRACE_EVENT(scoutfs_client_compact_stop,
);
TRACE_EVENT(scoutfs_server_compact_start,
TP_PROTO(struct super_block *sb, u64 id, u8 level, u64 node_id,
TP_PROTO(struct super_block *sb, u64 id, u8 level, u64 rid,
unsigned long client_nr, unsigned long server_nr,
unsigned long per_client),
TP_ARGS(sb, id, level, node_id, client_nr, server_nr, per_client),
TP_ARGS(sb, id, level, rid, client_nr, server_nr, per_client),
TP_STRUCT__entry(
SCSB_TRACE_FIELDS
__field(__u64, id)
__field(__u8, level)
__field(__u64, node_id)
__field(__u64, c_rid)
__field(unsigned long, client_nr)
__field(unsigned long, server_nr)
__field(unsigned long, per_client)
@@ -1174,40 +1174,40 @@ TRACE_EVENT(scoutfs_server_compact_start,
SCSB_TRACE_ASSIGN(sb);
__entry->id = id;
__entry->level = level;
__entry->node_id = node_id;
__entry->c_rid = rid;
__entry->client_nr = client_nr;
__entry->server_nr = server_nr;
__entry->per_client = per_client;
),
TP_printk(SCSBF" id %llu level %u node_id %llu client_nr %lu server_nr %lu per_client %lu",
TP_printk(SCSBF" id %llu level %u rid %016llx client_nr %lu server_nr %lu per_client %lu",
SCSB_TRACE_ARGS, __entry->id, __entry->level,
__entry->node_id, __entry->client_nr, __entry->server_nr,
__entry->c_rid, __entry->client_nr, __entry->server_nr,
__entry->per_client)
);
TRACE_EVENT(scoutfs_server_compact_done,
TP_PROTO(struct super_block *sb, u64 id, u64 node_id,
TP_PROTO(struct super_block *sb, u64 id, u64 rid,
unsigned long server_nr),
TP_ARGS(sb, id, node_id, server_nr),
TP_ARGS(sb, id, rid, server_nr),
TP_STRUCT__entry(
SCSB_TRACE_FIELDS
__field(__u64, id)
__field(__u64, node_id)
__field(__u64, c_rid)
__field(unsigned long, server_nr)
),
TP_fast_assign(
SCSB_TRACE_ASSIGN(sb);
__entry->id = id;
__entry->node_id = node_id;
__entry->rid = rid;
__entry->server_nr = server_nr;
),
TP_printk(SCSBF" id %llu node_id %llu server_nr %lu",
SCSB_TRACE_ARGS, __entry->id, __entry->node_id,
TP_printk(SCSBF" id %llu rid %016llx server_nr %lu",
SCSB_TRACE_ARGS, __entry->id, __entry->c_rid,
__entry->server_nr)
);
@@ -2470,32 +2470,32 @@ DEFINE_EVENT(scoutfs_segno_class, scoutfs_remove_segno,
);
DECLARE_EVENT_CLASS(scoutfs_server_client_count_class,
TP_PROTO(struct super_block *sb, u64 node_id, unsigned long nr_clients),
TP_PROTO(struct super_block *sb, u64 rid, unsigned long nr_clients),
TP_ARGS(sb, node_id, nr_clients),
TP_ARGS(sb, rid, nr_clients),
TP_STRUCT__entry(
SCSB_TRACE_FIELDS
__field(__s64, node_id)
__field(__s64, c_rid)
__field(unsigned long, nr_clients)
),
TP_fast_assign(
SCSB_TRACE_ASSIGN(sb);
__entry->node_id = node_id;
__entry->c_rid = rid;
__entry->nr_clients = nr_clients;
),
TP_printk(SCSBF" node_id %llu nr_clients %lu",
SCSB_TRACE_ARGS, __entry->node_id, __entry->nr_clients)
TP_printk(SCSBF" rid %016llx nr_clients %lu",
SCSB_TRACE_ARGS, __entry->c_rid, __entry->nr_clients)
);
DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_up,
TP_PROTO(struct super_block *sb, u64 node_id, unsigned long nr_clients),
TP_ARGS(sb, node_id, nr_clients)
TP_PROTO(struct super_block *sb, u64 rid, unsigned long nr_clients),
TP_ARGS(sb, rid, nr_clients)
);
DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_down,
TP_PROTO(struct super_block *sb, u64 node_id, unsigned long nr_clients),
TP_ARGS(sb, node_id, nr_clients)
TP_PROTO(struct super_block *sb, u64 rid, unsigned long nr_clients),
TP_ARGS(sb, rid, nr_clients)
);
#define slt_symbolic(mode) \
@@ -2509,16 +2509,16 @@ DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_down,
TRACE_EVENT(scoutfs_lock_message,
TP_PROTO(struct super_block *sb, int who, int what, int dir,
u64 node_id, u64 net_id, struct scoutfs_net_lock *nl),
u64 rid, u64 net_id, struct scoutfs_net_lock *nl),
TP_ARGS(sb, who, what, dir, node_id, net_id, nl),
TP_ARGS(sb, who, what, dir, rid, net_id, nl),
TP_STRUCT__entry(
SCSB_TRACE_FIELDS
__field(int, who)
__field(int, what)
__field(int, dir)
__field(__u64, node_id)
__field(__u64, m_rid)
__field(__u64, net_id)
sk_trace_define(key)
__field(__u8, old_mode)
@@ -2530,17 +2530,17 @@ TRACE_EVENT(scoutfs_lock_message,
__entry->who = who;
__entry->what = what;
__entry->dir = dir;
__entry->node_id = node_id;
__entry->m_rid = rid;
__entry->net_id = net_id;
sk_trace_assign(key, &nl->key);
__entry->old_mode = nl->old_mode;
__entry->new_mode = nl->new_mode;
),
TP_printk(SCSBF" %s %s %s node_id %llu net_id %llu key "SK_FMT" old_mode %u new_mode %u",
TP_printk(SCSBF" %s %s %s rid %016llx net_id %llu key "SK_FMT" old_mode %u new_mode %u",
SCSB_TRACE_ARGS, slt_symbolic(__entry->who),
slt_symbolic(__entry->what), slt_symbolic(__entry->dir),
__entry->node_id, __entry->net_id, sk_trace_args(key),
__entry->m_rid, __entry->net_id, sk_trace_args(key),
__entry->old_mode, __entry->new_mode)
);
@@ -2685,70 +2685,70 @@ DEFINE_EVENT(scoutfs_clock_sync_class, scoutfs_recv_clock_sync,
);
TRACE_EVENT(scoutfs_trans_seq_advance,
TP_PROTO(struct super_block *sb, u64 node_id, u64 prev_seq,
TP_PROTO(struct super_block *sb, u64 rid, u64 prev_seq,
u64 next_seq),
TP_ARGS(sb, node_id, prev_seq, next_seq),
TP_ARGS(sb, rid, prev_seq, next_seq),
TP_STRUCT__entry(
SCSB_TRACE_FIELDS
__field(__u64, node_id)
__field(__u64, s_rid)
__field(__u64, prev_seq)
__field(__u64, next_seq)
),
TP_fast_assign(
SCSB_TRACE_ASSIGN(sb);
__entry->node_id = node_id;
__entry->s_rid = rid;
__entry->prev_seq = prev_seq;
__entry->next_seq = next_seq;
),
TP_printk(SCSBF" node_id %llu prev_seq %llu next_seq %llu",
SCSB_TRACE_ARGS, __entry->node_id, __entry->prev_seq,
TP_printk(SCSBF" rid %016llx prev_seq %llu next_seq %llu",
SCSB_TRACE_ARGS, __entry->s_rid, __entry->prev_seq,
__entry->next_seq)
);
TRACE_EVENT(scoutfs_trans_seq_farewell,
TP_PROTO(struct super_block *sb, u64 node_id, u64 trans_seq),
TP_PROTO(struct super_block *sb, u64 rid, u64 trans_seq),
TP_ARGS(sb, node_id, trans_seq),
TP_ARGS(sb, rid, trans_seq),
TP_STRUCT__entry(
SCSB_TRACE_FIELDS
__field(__u64, node_id)
__field(__u64, s_rid)
__field(__u64, trans_seq)
),
TP_fast_assign(
SCSB_TRACE_ASSIGN(sb);
__entry->node_id = node_id;
__entry->s_rid = rid;
__entry->trans_seq = trans_seq;
),
TP_printk(SCSBF" node_id %llu trans_seq %llu",
SCSB_TRACE_ARGS, __entry->node_id, __entry->trans_seq)
TP_printk(SCSBF" rid %016llx trans_seq %llu",
SCSB_TRACE_ARGS, __entry->s_rid, __entry->trans_seq)
);
TRACE_EVENT(scoutfs_trans_seq_last,
TP_PROTO(struct super_block *sb, u64 node_id, u64 trans_seq),
TP_PROTO(struct super_block *sb, u64 rid, u64 trans_seq),
TP_ARGS(sb, node_id, trans_seq),
TP_ARGS(sb, rid, trans_seq),
TP_STRUCT__entry(
SCSB_TRACE_FIELDS
__field(__u64, node_id)
__field(__u64, s_rid)
__field(__u64, trans_seq)
),
TP_fast_assign(
SCSB_TRACE_ASSIGN(sb);
__entry->node_id = node_id;
__entry->s_rid = rid;
__entry->trans_seq = trans_seq;
),
TP_printk(SCSBF" node_id %llu trans_seq %llu",
SCSB_TRACE_ARGS, __entry->node_id, __entry->trans_seq)
TP_printk(SCSBF" rid %016llx trans_seq %llu",
SCSB_TRACE_ARGS, __entry->s_rid, __entry->trans_seq)
);
#endif /* _TRACE_SCOUTFS_H */

161
kmod/src/server.c
View File

@@ -99,7 +99,7 @@ struct server_info {
* The server tracks each connected client.
*/
struct server_client_info {
u64 node_id;
u64 rid;
struct list_head head;
unsigned long nr_compacts;
};
@@ -894,7 +894,7 @@ static int server_advance_seq(struct super_block *sb,
__le64 their_seq;
__le64 next_seq;
struct scoutfs_trans_seq_btree_key tsk;
u64 node_id = scoutfs_net_client_node_id(conn);
u64 rid = scoutfs_net_client_rid(conn);
int ret;
if (arg_len != sizeof(__le64)) {
@@ -908,7 +908,7 @@ static int server_advance_seq(struct super_block *sb,
if (their_seq != 0) {
tsk.trans_seq = le64_to_be64(their_seq);
tsk.node_id = cpu_to_be64(node_id);
tsk.rid = cpu_to_be64(rid);
ret = scoutfs_btree_delete(sb, &super->trans_seqs,
&tsk, sizeof(tsk));
@@ -919,11 +919,11 @@ static int server_advance_seq(struct super_block *sb,
next_seq = super->next_trans_seq;
le64_add_cpu(&super->next_trans_seq, 1);
trace_scoutfs_trans_seq_advance(sb, node_id, le64_to_cpu(their_seq),
trace_scoutfs_trans_seq_advance(sb, rid, le64_to_cpu(their_seq),
le64_to_cpu(next_seq));
tsk.trans_seq = le64_to_be64(next_seq);
tsk.node_id = cpu_to_be64(node_id);
tsk.rid = cpu_to_be64(rid);
ret = scoutfs_btree_insert(sb, &super->trans_seqs,
&tsk, sizeof(tsk), NULL, 0);
@@ -946,7 +946,7 @@ out:
* client's farewell is retransmitted so it's OK to not find any
* entries. This is called with the server commit rwsem held.
*/
static int remove_trans_seq(struct super_block *sb, u64 node_id)
static int remove_trans_seq(struct super_block *sb, u64 rid)
{
DECLARE_SERVER_INFO(sb, server);
struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
@@ -958,7 +958,7 @@ static int remove_trans_seq(struct super_block *sb, u64 node_id)
down_write(&server->seq_rwsem);
tsk.trans_seq = 0;
tsk.node_id = 0;
tsk.rid = 0;
for (;;) {
ret = scoutfs_btree_next(sb, &super->trans_seqs,
@@ -972,8 +972,8 @@ static int remove_trans_seq(struct super_block *sb, u64 node_id)
memcpy(&tsk, iref.key, iref.key_len);
scoutfs_btree_put_iref(&iref);
if (be64_to_cpu(tsk.node_id) == node_id) {
trace_scoutfs_trans_seq_farewell(sb, node_id,
if (be64_to_cpu(tsk.rid) == rid) {
trace_scoutfs_trans_seq_farewell(sb, rid,
be64_to_cpu(tsk.trans_seq));
ret = scoutfs_btree_delete(sb, &super->trans_seqs,
&tsk, sizeof(tsk));
@@ -981,7 +981,7 @@ static int remove_trans_seq(struct super_block *sb, u64 node_id)
}
be64_add_cpu(&tsk.trans_seq, 1);
tsk.node_id = 0;
tsk.rid = 0;
}
up_write(&server->seq_rwsem);
@@ -1006,7 +1006,7 @@ static int server_get_last_seq(struct super_block *sb,
struct scoutfs_super_block *super = &sbi->super;
struct scoutfs_trans_seq_btree_key tsk;
SCOUTFS_BTREE_ITEM_REF(iref);
u64 node_id = scoutfs_net_client_node_id(conn);
u64 rid = scoutfs_net_client_rid(conn);
__le64 last_seq = 0;
int ret;
@@ -1018,7 +1018,7 @@ static int server_get_last_seq(struct super_block *sb,
down_read(&server->seq_rwsem);
tsk.trans_seq = 0;
tsk.node_id = 0;
tsk.rid = 0;
ret = scoutfs_btree_next(sb, &super->trans_seqs,
&tsk, sizeof(tsk), &iref);
@@ -1037,7 +1037,7 @@ static int server_get_last_seq(struct super_block *sb,
ret = 0;
}
trace_scoutfs_trans_seq_last(sb, node_id, le64_to_cpu(last_seq));
trace_scoutfs_trans_seq_last(sb, rid, le64_to_cpu(last_seq));
up_read(&server->seq_rwsem);
out:
@@ -1107,12 +1107,12 @@ static int server_lock(struct super_block *sb,
struct scoutfs_net_connection *conn,
u8 cmd, u64 id, void *arg, u16 arg_len)
{
u64 node_id = scoutfs_net_client_node_id(conn);
u64 rid = scoutfs_net_client_rid(conn);
if (arg_len != sizeof(struct scoutfs_net_lock))
return -EINVAL;
return scoutfs_lock_server_request(sb, node_id, id, arg);
return scoutfs_lock_server_request(sb, rid, id, arg);
}
static int lock_response(struct super_block *sb,
@@ -1120,31 +1120,31 @@ static int lock_response(struct super_block *sb,
void *resp, unsigned int resp_len,
int error, void *data)
{
u64 node_id = scoutfs_net_client_node_id(conn);
u64 rid = scoutfs_net_client_rid(conn);
if (resp_len != sizeof(struct scoutfs_net_lock))
return -EINVAL;
return scoutfs_lock_server_response(sb, node_id, resp);
return scoutfs_lock_server_response(sb, rid, resp);
}
int scoutfs_server_lock_request(struct super_block *sb, u64 node_id,
int scoutfs_server_lock_request(struct super_block *sb, u64 rid,
struct scoutfs_net_lock *nl)
{
struct server_info *server = SCOUTFS_SB(sb)->server_info;
return scoutfs_net_submit_request_node(sb, server->conn, node_id,
return scoutfs_net_submit_request_node(sb, server->conn, rid,
SCOUTFS_NET_CMD_LOCK,
nl, sizeof(*nl),
lock_response, NULL, NULL);
}
int scoutfs_server_lock_response(struct super_block *sb, u64 node_id,
int scoutfs_server_lock_response(struct super_block *sb, u64 rid,
u64 id, struct scoutfs_net_lock *nl)
{
struct server_info *server = SCOUTFS_SB(sb)->server_info;
return scoutfs_net_response_node(sb, server->conn, node_id,
return scoutfs_net_response_node(sb, server->conn, rid,
SCOUTFS_NET_CMD_LOCK, id, 0,
nl, sizeof(*nl));
}
@@ -1162,34 +1162,34 @@ static int lock_recover_response(struct super_block *sb,
void *resp, unsigned int resp_len,
int error, void *data)
{
u64 node_id = scoutfs_net_client_node_id(conn);
u64 rid = scoutfs_net_client_rid(conn);
if (invalid_recover(resp, resp_len))
return -EINVAL;
return scoutfs_lock_server_recover_response(sb, node_id, resp);
return scoutfs_lock_server_recover_response(sb, rid, resp);
}
int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id,
int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid,
struct scoutfs_key *key)
{
struct server_info *server = SCOUTFS_SB(sb)->server_info;
return scoutfs_net_submit_request_node(sb, server->conn, node_id,
return scoutfs_net_submit_request_node(sb, server->conn, rid,
SCOUTFS_NET_CMD_LOCK_RECOVER,
key, sizeof(*key),
lock_recover_response,
NULL, NULL);
}
static int insert_mounted_client(struct super_block *sb, u64 node_id,
static int insert_mounted_client(struct super_block *sb, u64 rid,
u64 gr_flags)
{
struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
struct scoutfs_mounted_client_btree_key mck;
struct scoutfs_mounted_client_btree_val mcv;
mck.node_id = cpu_to_be64(node_id);
mck.rid = cpu_to_be64(rid);
mcv.flags = 0;
if (gr_flags & SCOUTFS_NET_GREETING_FLAG_VOTER)
mcv.flags |= SCOUTFS_MOUNTED_CLIENT_VOTER;
@@ -1208,13 +1208,13 @@ static int insert_mounted_client(struct super_block *sb, u64 node_id,
*
* The caller has to serialize with farewell processing.
*/
static int delete_mounted_client(struct super_block *sb, u64 node_id)
static int delete_mounted_client(struct super_block *sb, u64 rid)
{
struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
struct scoutfs_mounted_client_btree_key mck;
int ret;
mck.node_id = cpu_to_be64(node_id);
mck.rid = cpu_to_be64(rid);
ret = scoutfs_btree_delete(sb, &super->mounted_clients,
&mck, sizeof(mck));
@@ -1230,17 +1230,8 @@ static int delete_mounted_client(struct super_block *sb, u64 node_id)
* log some detail before shutting down. A failure to send a greeting
* response shuts down the connection.
*
* We allocate a new node_id for the first connect attempt from a
* client.
*
* If a client reconnects they'll send their initially assigned node_id
* in their greeting request.
*
* XXX We can lose allocated node_ids here as we record the node_id as
* live as we send a valid greeting response. The client might
* disconnect before they receive the response and resent and initial
* blank greeting. We could use a client uuid to associate with
* allocated node_ids.
* If a client reconnects they'll send their previously received
* serer_term in their greeting request.
*
* XXX The logic of this has gotten convoluted. The lock server can
* send a recovery request so it needs to be called after the core net
@@ -1259,9 +1250,8 @@ static int server_greeting(struct super_block *sb,
struct scoutfs_net_greeting greet;
DECLARE_SERVER_INFO(sb, server);
struct commit_waiter cw;
__le64 node_id = 0;
__le64 umb = 0;
bool sent_node_id;
bool reconnecting;
bool first_contact;
bool farewell;
int ret = 0;
@@ -1288,17 +1278,15 @@ static int server_greeting(struct super_block *sb,
goto send_err;
}
if (gr->node_id == 0) {
if (gr->server_term == 0) {
down_read(&server->commit_rwsem);
spin_lock(&server->lock);
node_id = super->next_node_id;
le64_add_cpu(&super->next_node_id, 1);
umb = super->unmount_barrier;
spin_unlock(&server->lock);
mutex_lock(&server->farewell_mutex);
ret = insert_mounted_client(sb, le64_to_cpu(node_id),
ret = insert_mounted_client(sb, le64_to_cpu(gr->rid),
le64_to_cpu(gr->flags));
mutex_unlock(&server->farewell_mutex);
@@ -1310,20 +1298,17 @@ static int server_greeting(struct super_block *sb,
queue_work(server->wq, &server->farewell_work);
}
} else {
node_id = gr->node_id;
umb = gr->unmount_barrier;
}
send_err:
err = ret;
if (err)
node_id = 0;
greet.fsid = super->hdr.fsid;
greet.format_hash = super->format_hash;
greet.server_term = cpu_to_le64(server->term);
greet.unmount_barrier = umb;
greet.node_id = node_id;
greet.rid = gr->rid;
greet.flags = 0;
/* queue greeting response to be sent first once messaging enabled */
@@ -1335,15 +1320,15 @@ send_err:
goto out;
/* have the net core enable messaging and resend */
sent_node_id = gr->node_id != 0;
reconnecting = gr->server_term != 0;
first_contact = le64_to_cpu(gr->server_term) != server->term;
if (gr->flags & cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL))
farewell = true;
else
farewell = false;
scoutfs_net_server_greeting(sb, conn, le64_to_cpu(node_id), id,
sent_node_id, first_contact, farewell);
scoutfs_net_server_greeting(sb, conn, le64_to_cpu(gr->rid), id,
reconnecting, first_contact, farewell);
/* lock server might send recovery request */
if (le64_to_cpu(gr->server_term) != server->term) {
@@ -1351,7 +1336,7 @@ send_err:
/* we're now doing two commits per greeting, not great */
down_read(&server->commit_rwsem);
ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(node_id),
ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid),
gr->server_term != 0);
if (ret == 0)
queue_commit_work(server, &cw);
@@ -1369,7 +1354,7 @@ out:
struct farewell_request {
struct list_head entry;
u64 net_id;
u64 node_id;
u64 rid;
};
static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref)
@@ -1435,7 +1420,7 @@ static void farewell_worker(struct work_struct *work)
/* count how many reqs requests are from voting clients */
nr_unmounting = 0;
list_for_each_entry_safe(fw, tmp, &reqs, entry) {
mck.node_id = cpu_to_be64(fw->node_id);
mck.rid = cpu_to_be64(fw->rid);
ret = scoutfs_btree_lookup(sb, &super->mounted_clients,
&mck, sizeof(mck), &iref);
if (ret == 0 && invalid_mounted_client_item(&iref)) {
@@ -1484,7 +1469,7 @@ static void farewell_worker(struct work_struct *work)
nr_mounted++;
scoutfs_btree_put_iref(&iref);
be64_add_cpu(&mck.node_id, 1);
be64_add_cpu(&mck.rid, 1);
}
@@ -1505,9 +1490,9 @@ static void farewell_worker(struct work_struct *work)
down_read(&server->commit_rwsem);
ret = scoutfs_lock_server_farewell(sb, fw->node_id) ?:
remove_trans_seq(sb, fw->node_id) ?:
delete_mounted_client(sb, fw->node_id);
ret = scoutfs_lock_server_farewell(sb, fw->rid) ?:
remove_trans_seq(sb, fw->rid) ?:
delete_mounted_client(sb, fw->rid);
if (ret == 0)
queue_commit_work(server, &cw);
@@ -1532,7 +1517,7 @@ static void farewell_worker(struct work_struct *work)
/* and finally send all the responses */
list_for_each_entry_safe(fw, tmp, &send, entry) {
ret = scoutfs_net_response_node(sb, server->conn, fw->node_id,
ret = scoutfs_net_response_node(sb, server->conn, fw->rid,
SCOUTFS_NET_CMD_FAREWELL,
fw->net_id, 0, NULL, 0);
if (ret)
@@ -1556,7 +1541,7 @@ out:
queue_work(server->wq, &server->farewell_work);
}
static void free_farewell_requests(struct super_block *sb, u64 node_id)
static void free_farewell_requests(struct super_block *sb, u64 rid)
{
struct server_info *server = SCOUTFS_SB(sb)->server_info;
struct farewell_request *tmp;
@@ -1564,7 +1549,7 @@ static void free_farewell_requests(struct super_block *sb, u64 node_id)
mutex_lock(&server->farewell_mutex);
list_for_each_entry_safe(fw, tmp, &server->farewell_requests, entry) {
if (node_id == 0 || fw->node_id == node_id) {
if (rid == 0 || fw->rid == rid) {
list_del_init(&fw->entry);
kfree(fw);
}
@@ -1588,7 +1573,7 @@ static int server_farewell(struct super_block *sb,
u8 cmd, u64 id, void *arg, u16 arg_len)
{
struct server_info *server = SCOUTFS_SB(sb)->server_info;
u64 node_id = scoutfs_net_client_node_id(conn);
u64 rid = scoutfs_net_client_rid(conn);
struct farewell_request *fw;
if (arg_len != 0)
@@ -1600,7 +1585,7 @@ static int server_farewell(struct super_block *sb,
if (fw == NULL)
return -ENOMEM;
fw->node_id = node_id;
fw->rid = rid;
fw->net_id = id;
mutex_lock(&server->farewell_mutex);
@@ -1616,13 +1601,13 @@ static int server_farewell(struct super_block *sb,
/* requests sent to clients are tracked so we can free resources */
struct compact_request {
struct list_head head;
u64 node_id;
u64 rid;
struct scoutfs_net_compact_request req;
};
/*
* Find a node that can process our compaction request. Return a
* node_id if we found a client and added the compaction to the client
* rid if we found a client and added the compaction to the client
* and server counts. Returns 0 if no suitable clients were found.
*/
static u64 compact_request_start(struct super_block *sb,
@@ -1631,7 +1616,7 @@ static u64 compact_request_start(struct super_block *sb,
struct server_info *server = SCOUTFS_SB(sb)->server_info;
struct server_client_info *last;
struct server_client_info *sci;
u64 node_id = 0;
u64 rid = 0;
spin_lock(&server->lock);
@@ -1650,8 +1635,8 @@ static u64 compact_request_start(struct super_block *sb,
list_add(&cr->head, &server->compacts);
server->nr_compacts++;
sci->nr_compacts++;
node_id = sci->node_id;
cr->node_id = node_id;
rid = sci->rid;
cr->rid = rid;
break;
}
if (sci == last)
@@ -1659,14 +1644,14 @@ static u64 compact_request_start(struct super_block *sb,
}
trace_scoutfs_server_compact_start(sb, le64_to_cpu(cr->req.id),
cr->req.ents[0].level, node_id,
node_id ? sci->nr_compacts : 0,
cr->req.ents[0].level, rid,
rid ? sci->nr_compacts : 0,
server->nr_compacts,
server->compacts_per_client);
spin_unlock(&server->lock);
return node_id;
return rid;
}
/*
@@ -1688,7 +1673,7 @@ static struct compact_request *compact_request_done(struct super_block *sb,
continue;
list_for_each_entry(sci, &server->clients, head) {
if (sci->node_id == cr->node_id) {
if (sci->rid == cr->rid) {
sci->nr_compacts--;
break;
}
@@ -1700,7 +1685,7 @@ static struct compact_request *compact_request_done(struct super_block *sb,
break;
}
trace_scoutfs_server_compact_done(sb, id, ret ? ret->node_id : 0,
trace_scoutfs_server_compact_done(sb, id, ret ? ret->rid : 0,
server->nr_compacts);
spin_unlock(&server->lock);
@@ -1728,7 +1713,7 @@ static void forget_client_compacts(struct super_block *sb,
spin_lock(&server->lock);
list_for_each_entry_safe(cr, pos, &server->compacts, head) {
if (cr->node_id == sci->node_id) {
if (cr->rid == sci->rid) {
sci->nr_compacts--;
server->nr_compacts--;
list_move(&cr->head, &forget);
@@ -2129,7 +2114,7 @@ static void scoutfs_server_compact_worker(struct work_struct *work)
struct compact_request *cr;
struct commit_waiter cw;
int nr_segnos = 0;
u64 node_id;
u64 rid;
__le64 id;
int ret;
@@ -2169,14 +2154,14 @@ static void scoutfs_server_compact_worker(struct work_struct *work)
/* try to send to a node with capacity, they can disconnect */
retry:
req->id = id;
node_id = compact_request_start(sb, cr);
if (node_id == 0) {
rid = compact_request_start(sb, cr);
if (rid == 0) {
ret = 0;
goto out;
}
/* response processing can complete compaction before this returns */
ret = scoutfs_net_submit_request_node(sb, server->conn, node_id,
ret = scoutfs_net_submit_request_node(sb, server->conn, rid,
SCOUTFS_NET_CMD_COMPACT,
req, sizeof(*req),
compact_response, NULL, NULL);
@@ -2228,18 +2213,18 @@ static scoutfs_net_request_t server_req_funcs[] = {
static void server_notify_up(struct super_block *sb,
struct scoutfs_net_connection *conn,
void *info, u64 node_id)
void *info, u64 rid)
{
struct server_client_info *sci = info;
DECLARE_SERVER_INFO(sb, server);
if (node_id != 0) {
sci->node_id = node_id;
if (rid != 0) {
sci->rid = rid;
sci->nr_compacts = 0;
spin_lock(&server->lock);
list_add_tail(&sci->head, &server->clients);
server->nr_clients++;
trace_scoutfs_server_client_up(sb, node_id, server->nr_clients);
trace_scoutfs_server_client_up(sb, rid, server->nr_clients);
spin_unlock(&server->lock);
try_queue_compact(server);
@@ -2248,20 +2233,20 @@ static void server_notify_up(struct super_block *sb,
static void server_notify_down(struct super_block *sb,
struct scoutfs_net_connection *conn,
void *info, u64 node_id)
void *info, u64 rid)
{
struct server_client_info *sci = info;
DECLARE_SERVER_INFO(sb, server);
if (node_id != 0) {
if (rid != 0) {
spin_lock(&server->lock);
list_del_init(&sci->head);
server->nr_clients--;
trace_scoutfs_server_client_down(sb, node_id,
trace_scoutfs_server_client_down(sb, rid,
server->nr_clients);
spin_unlock(&server->lock);
free_farewell_requests(sb, node_id);
free_farewell_requests(sb, rid);
forget_client_compacts(sb, sci);
try_queue_compact(server);

6
kmod/src/server.h
View File

@@ -64,11 +64,11 @@ void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment,
void scoutfs_init_ment_from_net(struct scoutfs_manifest_entry *ment,
struct scoutfs_net_manifest_entry *net_ment);
int scoutfs_server_lock_request(struct super_block *sb, u64 node_id,
int scoutfs_server_lock_request(struct super_block *sb, u64 rid,
struct scoutfs_net_lock *nl);
int scoutfs_server_lock_response(struct super_block *sb, u64 node_id,
int scoutfs_server_lock_response(struct super_block *sb, u64 rid,
u64 id, struct scoutfs_net_lock *nl);
int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id,
int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid,
struct scoutfs_key *key);
struct sockaddr_in;

1
kmod/src/super.c
View File

@@ -429,7 +429,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
scoutfs_quorum_setup(sb) ?:
scoutfs_server_setup(sb) ?:
scoutfs_client_setup(sb) ?:
scoutfs_client_wait_node_id(sb) ?:
scoutfs_lock_rid(sb, SCOUTFS_LOCK_WRITE, 0, sbi->rid,
&sbi->rid_lock);
if (ret)

1
kmod/src/super.h
View File

@@ -32,7 +32,6 @@ struct scoutfs_sb_info {
/* assigned once at the start of each mount, read-only */
u64 rid;
u64 node_id;
struct scoutfs_lock *rid_lock;
struct scoutfs_super_block super;