diff --git a/kmod/src/client.c b/kmod/src/client.c index 8ef58589..e9d7d480 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -49,7 +49,6 @@ struct client_info { struct super_block *sb; struct scoutfs_net_connection *conn; - struct completion node_id_comp; atomic_t shutting_down; struct workqueue_struct *workq; @@ -299,8 +298,9 @@ static int client_lock_recover(struct super_block *sb, /* * Process a greeting response in the client from the server. This is - * called for every connected socket on the connection. The first - * response will have the node_id that the server assigned the client. + * called for every connected socket on the connection. Each response + * contains the remote server's elected term which can be used to + * identify server failover. */ static int client_greeting(struct super_block *sb, struct scoutfs_net_connection *conn, @@ -309,7 +309,6 @@ static int client_greeting(struct super_block *sb, { struct client_info *client = SCOUTFS_SB(sb)->client_info; struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_net_greeting *gr = resp; bool new_server; int ret; @@ -340,25 +339,6 @@ static int client_greeting(struct super_block *sb, goto out; } - if (sbi->node_id != 0 && le64_to_cpu(gr->node_id) != sbi->node_id) { - scoutfs_warn(sb, "server sent node_id %llu, client has %llu", - le64_to_cpu(gr->node_id), - sbi->node_id); - ret = -EINVAL; - goto out; - } - - if (sbi->node_id == 0 && gr->node_id == 0) { - scoutfs_warn(sb, "server sent node_id 0, client also has 0"); - ret = -EINVAL; - goto out; - } - - if (sbi->node_id == 0) { - sbi->node_id = le64_to_cpu(gr->node_id); - complete(&client->node_id_comp); - } - new_server = le64_to_cpu(gr->server_term) != client->server_term; scoutfs_net_client_greeting(sb, conn, new_server); @@ -466,7 +446,7 @@ static void scoutfs_client_connect_worker(struct work_struct *work) greet.format_hash = super->format_hash; greet.server_term = cpu_to_le64(client->server_term); greet.unmount_barrier = cpu_to_le64(client->greeting_umb); - greet.node_id = cpu_to_le64(sbi->node_id); + greet.rid = cpu_to_le64(sbi->rid); greet.flags = 0; if (client->sending_farewell) greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL); @@ -553,7 +533,7 @@ static scoutfs_net_request_t client_req_funcs[] = { */ static void client_notify_down(struct super_block *sb, struct scoutfs_net_connection *conn, void *info, - u64 node_id) + u64 rid) { struct client_info *client = SCOUTFS_SB(sb)->client_info; @@ -561,18 +541,6 @@ static void client_notify_down(struct super_block *sb, queue_delayed_work(client->workq, &client->connect_dwork, 0); } -/* - * Wait for the first connected socket on the connection that assigns - * the node_id that will be used for the rest of the life time of the - * mount. - */ -int scoutfs_client_wait_node_id(struct super_block *sb) -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - - return wait_for_completion_interruptible(&client->node_id_comp); -} - int scoutfs_client_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -587,7 +555,6 @@ int scoutfs_client_setup(struct super_block *sb) sbi->client_info = client; client->sb = sb; - init_completion(&client->node_id_comp); atomic_set(&client->shutting_down, 0); INIT_DELAYED_WORK(&client->connect_dwork, scoutfs_client_connect_worker); @@ -640,12 +607,12 @@ static int client_farewell_response(struct super_block *sb, * so that they don't wait for us to reconnect and trigger a timeout. * * This decision is a little racy. The server considers us connected - * when it assigns us a node_id as it processes the greeting. We can - * disconnect before receiving the response and leave without sending a - * farewell. So given that awkward initial race, we also have a bit of - * a race where we just test the server_term to see if we've ever gotten - * a greeting reply from any server. We don't try to synchronize with - * pending connection attempts. + * when it records a persistent record of our rid as it processes our + * greeting. We can disconnect before receiving the greeting response + * and leave without sending a farewell. So given that awkward initial + * race, we also have a bit of a race where we just test the server_term + * to see if we've ever gotten a greeting reply from any server. We + * don't try to synchronize with pending connection attempts. * * The consequences of aborting a mount at just the wrong time and * disconnecting without the farewell handshake depend on what the diff --git a/kmod/src/client.h b/kmod/src/client.h index dd0c2eae..fe938306 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -24,7 +24,6 @@ int scoutfs_client_lock_response(struct super_block *sb, u64 net_id, int scoutfs_client_lock_recover_response(struct super_block *sb, u64 net_id, struct scoutfs_net_lock_recover *nlr); -int scoutfs_client_wait_node_id(struct super_block *sb); int scoutfs_client_setup(struct super_block *sb); void scoutfs_client_destroy(struct super_block *sb); diff --git a/kmod/src/format.h b/kmod/src/format.h index c2f2fa15..2534fe16 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -287,7 +287,7 @@ struct scoutfs_extent_btree_key { * server failover knows who to wait for before resuming operations. */ struct scoutfs_lock_client_btree_key { - __be64 node_id; + __be64 rid; } __packed; /* @@ -296,14 +296,14 @@ struct scoutfs_lock_client_btree_key { */ struct scoutfs_trans_seq_btree_key { __be64 trans_seq; - __be64 node_id; + __be64 rid; } __packed; /* * The server keeps a persistent record of mounted clients. */ struct scoutfs_mounted_client_btree_key { - __be64 node_id; + __be64 rid; } __packed; struct scoutfs_mounted_client_btree_val { @@ -479,7 +479,6 @@ struct scoutfs_super_block { __le64 alloc_cursor; struct scoutfs_btree_ring bring; __le64 next_seg_seq; - __le64 next_node_id; __le64 next_compact_id; __le64 quorum_fenced_term; __le64 quorum_server_term; @@ -611,8 +610,9 @@ enum { * * @server_term: The raft term that elected the server. Initially 0 * from the client, sent by the server, then sent by the client as it - * tries to reconnect. Used to identify a client reconnecting to a - * server that has timed out its connection. + * tries to reconnect. Used to identify a client reconnecting to both + * the same serer after receiving a greeting response and to a new + * server after failover. * * @unmount_barrier: Incremented every time the remaining majority of * quorum members all agree to leave. The server tells a quorum member @@ -620,17 +620,17 @@ enum { * value increase in the super block then it knows that the server has * processed its farewell and can safely unmount. * - * @node_id: The id of the client. Initially 0 from the client, - * assigned by the server, and sent by the client as it reconnects. - * Used by the server to identify reconnecting clients whose existing - * state must be dealt with. + * @rid: The client's random id that was generated once as the mount + * started up. This identifies a specific remote mount across + * connections and servers. It's set to the client's rid in both the + * request and response for consistency. */ struct scoutfs_net_greeting { __le64 fsid; __le64 format_hash; __le64 server_term; __le64 unmount_barrier; - __le64 node_id; + __le64 rid; __le64 flags; } __packed; diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index da7e1062..3f93c150 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -50,10 +50,10 @@ * server doesn't use the modes specified by the clients but they're * provided to add context. * - * The server relies on the node_id allocation and reliable messaging - * layers of the system. Each client has a node_id that is unique for - * its life time. Message requests and responses are reliably - * delivered in order across reconnection. + * The server relies on the client's static rid and on reliable + * messaging. Each client has a rid that is unique for its life time. + * Message requests and responses are reliably delivered in order across + * reconnection. * * The server maintains a persistent record of connected clients. A new * server instance discovers these and waits for previously connected @@ -126,7 +126,7 @@ enum { * that the mode is actively granted, a pending request from the client, * or a pending invalidation sent to the client. * - * @node_id: The client's node_id used to send messages and tear down + * @rid: The client's rid used to send messages and tear down * state as client's exit. * * @net_id: The id of a client's request used to send grant responses. The @@ -139,7 +139,7 @@ enum { */ struct client_lock_entry { struct list_head head; - u64 node_id; + u64 rid; u64 net_id; u8 mode; @@ -221,7 +221,7 @@ static bool client_entries_compatible(struct client_lock_entry *granted, return (granted->mode == requested->mode && (granted->mode == SCOUTFS_LOCK_READ || granted->mode == SCOUTFS_LOCK_WRITE_ONLY)) || - (granted->node_id == requested->node_id && + (granted->rid == requested->rid && granted->mode == SCOUTFS_LOCK_READ && requested->mode == SCOUTFS_LOCK_WRITE); } @@ -340,14 +340,14 @@ static void put_server_lock(struct lock_server_info *inf, static struct client_lock_entry *find_entry(struct server_lock_node *snode, struct list_head *list, - u64 node_id) + u64 rid) { struct client_lock_entry *clent; WARN_ON_ONCE(!mutex_is_locked(&snode->mutex)); list_for_each_entry(clent, list, head) { - if (clent->node_id == node_id) + if (clent->rid == rid) return clent; } @@ -363,7 +363,7 @@ static int process_waiting_requests(struct super_block *sb, * * XXX shut down if we get enomem? */ -int scoutfs_lock_server_request(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_request(struct super_block *sb, u64 rid, u64 net_id, struct scoutfs_net_lock *nl) { DECLARE_LOCK_SERVER_INFO(sb, inf); @@ -372,7 +372,7 @@ int scoutfs_lock_server_request(struct super_block *sb, u64 node_id, int ret; trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_GRANT, SLT_REQUEST, - node_id, net_id, nl); + rid, net_id, nl); if (invalid_mode(nl->old_mode) || invalid_mode(nl->new_mode)) { ret = -EINVAL; @@ -386,7 +386,7 @@ int scoutfs_lock_server_request(struct super_block *sb, u64 node_id, } INIT_LIST_HEAD(&clent->head); - clent->node_id = node_id; + clent->rid = rid; clent->net_id = net_id; clent->mode = nl->new_mode; @@ -414,7 +414,7 @@ out: * * XXX what to do with errors? kick the client? */ -int scoutfs_lock_server_response(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_response(struct super_block *sb, u64 rid, struct scoutfs_net_lock *nl) { DECLARE_LOCK_SERVER_INFO(sb, inf); @@ -423,7 +423,7 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 node_id, int ret; trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_INVALIDATE, SLT_RESPONSE, - node_id, 0, nl); + rid, 0, nl); if (invalid_mode(nl->old_mode) || invalid_mode(nl->new_mode)) { ret = -EINVAL; @@ -437,7 +437,7 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 node_id, goto out; } - clent = find_entry(snode, &snode->invalidated, node_id); + clent = find_entry(snode, &snode->invalidated, rid); if (!clent) { put_server_lock(inf, snode); ret = -EINVAL; @@ -512,13 +512,13 @@ static int process_waiting_requests(struct super_block *sb, nl.old_mode = gr->mode; nl.new_mode = invalidation_mode(gr->mode, req->mode); - ret = scoutfs_server_lock_request(sb, gr->node_id, &nl); + ret = scoutfs_server_lock_request(sb, gr->rid, &nl); if (ret) goto out; trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_INVALIDATE, SLT_REQUEST, - gr->node_id, 0, &nl); + gr->rid, 0, &nl); add_client_entry(snode, &snode->invalidated, gr); } @@ -531,7 +531,7 @@ static int process_waiting_requests(struct super_block *sb, nl.new_mode = req->mode; /* see if there's an existing compatible grant to replace */ - gr = find_entry(snode, &snode->granted, req->node_id); + gr = find_entry(snode, &snode->granted, req->rid); if (gr) { nl.old_mode = gr->mode; free_client_entry(inf, snode, gr); @@ -539,13 +539,13 @@ static int process_waiting_requests(struct super_block *sb, nl.old_mode = SCOUTFS_LOCK_NULL; } - ret = scoutfs_server_lock_response(sb, req->node_id, + ret = scoutfs_server_lock_response(sb, req->rid, req->net_id, &nl); if (ret) goto out; trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_GRANT, - SLT_RESPONSE, req->node_id, + SLT_RESPONSE, req->rid, req->net_id, &nl); /* don't track null client locks, track all else */ @@ -571,7 +571,7 @@ out: * * This is running in concurrent client greeting processing contexts. */ -int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid, bool should_exist) { DECLARE_LOCK_SERVER_INFO(sb, inf); @@ -581,7 +581,7 @@ int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id, struct scoutfs_key key; int ret; - cbk.node_id = cpu_to_be64(node_id); + cbk.rid = cpu_to_be64(rid); mutex_lock(&inf->mutex); if (should_exist) { @@ -597,7 +597,7 @@ int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id, if (should_exist && ret == 0) { scoutfs_key_set_zeros(&key); - ret = scoutfs_server_lock_recover_request(sb, node_id, &key); + ret = scoutfs_server_lock_recover_request(sb, rid, &key); if (ret) goto out; } @@ -611,7 +611,7 @@ out: * they were the last client in recovery then we can process all the * server locks that had requests. */ -static int finished_recovery(struct super_block *sb, u64 node_id, bool cancel) +static int finished_recovery(struct super_block *sb, u64 rid, bool cancel) { DECLARE_LOCK_SERVER_INFO(sb, inf); struct server_lock_node *snode; @@ -620,7 +620,7 @@ static int finished_recovery(struct super_block *sb, u64 node_id, bool cancel) int ret = 0; spin_lock(&inf->lock); - scoutfs_spbm_clear(&inf->recovery_pending, node_id); + scoutfs_spbm_clear(&inf->recovery_pending, rid); still_pending = !scoutfs_spbm_empty(&inf->recovery_pending); spin_unlock(&inf->lock); if (still_pending) @@ -654,7 +654,7 @@ static int finished_recovery(struct super_block *sb, u64 node_id, bool cancel) * gave us in response and send another request from the next key. * We're done once we receive an empty response. */ -int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid, struct scoutfs_net_lock_recover *nlr) { DECLARE_LOCK_SERVER_INFO(sb, inf); @@ -667,7 +667,7 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, /* client must be in recovery */ spin_lock(&inf->lock); - if (!scoutfs_spbm_test(&inf->recovery_pending, node_id)) + if (!scoutfs_spbm_test(&inf->recovery_pending, rid)) ret = -EINVAL; spin_unlock(&inf->lock); if (ret) @@ -675,7 +675,7 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, /* client has sent us all their locks */ if (nlr->nr == 0) { - ret = finished_recovery(sb, node_id, true); + ret = finished_recovery(sb, rid, true); goto out; } @@ -687,7 +687,7 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, } INIT_LIST_HEAD(&clent->head); - clent->node_id = node_id; + clent->rid = rid; clent->net_id = 0; clent->mode = nlr->locks[i].new_mode; @@ -698,7 +698,7 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, goto out; } - existing = find_entry(snode, &snode->granted, node_id); + existing = find_entry(snode, &snode->granted, rid); if (existing) { kfree(clent); put_server_lock(inf, snode); @@ -717,20 +717,20 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, key = nlr->locks[le16_to_cpu(nlr->nr) - 1].key; scoutfs_key_inc(&key); - ret = scoutfs_server_lock_recover_request(sb, node_id, &key); + ret = scoutfs_server_lock_recover_request(sb, rid, &key); out: return ret; } -static int node_id_and_put_iref(struct scoutfs_btree_item_ref *iref, - u64 *node_id) +static int get_rid_and_put_ref(struct scoutfs_btree_item_ref *iref, + u64 *rid) { struct scoutfs_lock_client_btree_key *cbk; int ret; if (iref->key_len == sizeof(*cbk) && iref->val_len == 0) { cbk = iref->key; - *node_id = be64_to_cpu(cbk->node_id); + *rid = be64_to_cpu(cbk->rid); ret = 0; } else { ret = -EIO; @@ -754,12 +754,12 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) struct scoutfs_lock_client_btree_key cbk; SCOUTFS_BTREE_ITEM_REF(iref); bool timed_out; - u64 node_id; + u64 rid; int ret; /* we enter recovery if there are any client records */ - for (node_id = 0; ; node_id++) { - cbk.node_id = cpu_to_be64(node_id); + for (rid = 0; ; rid++) { + cbk.rid = cpu_to_be64(rid); ret = scoutfs_btree_next(sb, &super->lock_clients, &cbk, sizeof(cbk), &iref); if (ret == -ENOENT) { @@ -767,13 +767,13 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) break; } if (ret == 0) - ret = node_id_and_put_iref(&iref, &node_id); + ret = get_rid_and_put_ref(&iref, &rid); if (ret < 0) break; spin_lock(&inf->lock); - if (scoutfs_spbm_test(&inf->recovery_pending, node_id)) { - scoutfs_spbm_clear(&inf->recovery_pending, node_id); + if (scoutfs_spbm_test(&inf->recovery_pending, rid)) { + scoutfs_spbm_clear(&inf->recovery_pending, rid); timed_out = true; } else { timed_out = false; @@ -783,11 +783,11 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) if (!timed_out) continue; - scoutfs_err(sb, "client node_id %llu lock recovery timed out", - node_id); + scoutfs_err(sb, "client rid %016llx lock recovery timed out", + rid); /* XXX these aren't immediately committed */ - cbk.node_id = cpu_to_be64(node_id); + cbk.rid = cpu_to_be64(rid); ret = scoutfs_btree_delete(sb, &super->lock_clients, &cbk, sizeof(cbk)); if (ret) @@ -812,7 +812,7 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) * If we fail to delete a persistent entry then we have to shut down and * hope that the next server has more luck. */ -int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id) +int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid) { DECLARE_LOCK_SERVER_INFO(sb, inf); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; @@ -825,7 +825,7 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id) bool freed; int ret = 0; - cli.node_id = cpu_to_be64(node_id); + cli.rid = cpu_to_be64(rid); mutex_lock(&inf->mutex); ret = scoutfs_btree_delete(sb, &super->lock_clients, &cli, sizeof(cli)); mutex_unlock(&inf->mutex); @@ -847,7 +847,7 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id) NULL) { list_for_each_entry_safe(clent, tmp, list, head) { - if (clent->node_id == node_id) { + if (clent->rid == rid) { free_client_entry(inf, snode, clent); freed = true; } @@ -869,7 +869,8 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id) out: if (ret < 0) { - scoutfs_err(sb, "lock server err %d during node %llu farewell, shutting down", ret, node_id); + scoutfs_err(sb, "lock server err %d during client rid %016llx farewell, shutting down", + ret, rid); scoutfs_server_abort(sb); } @@ -913,9 +914,9 @@ static void lock_server_tseq_show(struct seq_file *m, tseq_entry); struct server_lock_node *snode = clent->snode; - seq_printf(m, SK_FMT" %s %s node_id %llu net_id %llu\n", + seq_printf(m, SK_FMT" %s %s rid %016llx net_id %llu\n", SK_ARG(&snode->key), lock_mode_string(clent->mode), - lock_on_list_string(clent->on_list), clent->node_id, + lock_on_list_string(clent->on_list), clent->rid, clent->net_id); } @@ -934,7 +935,7 @@ int scoutfs_lock_server_setup(struct super_block *sb) SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_lock_client_btree_key cbk; unsigned int nr; - u64 node_id; + u64 rid; int ret; inf = kzalloc(sizeof(struct lock_server_info), GFP_KERNEL); @@ -961,23 +962,23 @@ int scoutfs_lock_server_setup(struct super_block *sb) /* we enter recovery if there are any client records */ nr = 0; - for (node_id = 0; ; node_id++) { - cbk.node_id = cpu_to_be64(node_id); + for (rid = 0; ; rid++) { + cbk.rid = cpu_to_be64(rid); ret = scoutfs_btree_next(sb, &super->lock_clients, &cbk, sizeof(cbk), &iref); if (ret == -ENOENT) break; if (ret == 0) - ret = node_id_and_put_iref(&iref, &node_id); + ret = get_rid_and_put_ref(&iref, &rid); if (ret < 0) goto out; - ret = scoutfs_spbm_set(&inf->recovery_pending, node_id); + ret = scoutfs_spbm_set(&inf->recovery_pending, rid); if (ret) goto out; nr++; - if (node_id == U64_MAX) + if (rid == U64_MAX) break; } ret = 0; diff --git a/kmod/src/lock_server.h b/kmod/src/lock_server.h index cc0606a8..0ac2f772 100644 --- a/kmod/src/lock_server.h +++ b/kmod/src/lock_server.h @@ -1,15 +1,15 @@ #ifndef _SCOUTFS_LOCK_SERVER_H_ #define _SCOUTFS_LOCK_SERVER_H_ -int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid, struct scoutfs_net_lock_recover *nlr); -int scoutfs_lock_server_request(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_request(struct super_block *sb, u64 rid, u64 net_id, struct scoutfs_net_lock *nl); -int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid, bool should_exist); -int scoutfs_lock_server_response(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_response(struct super_block *sb, u64 rid, struct scoutfs_net_lock *nl); -int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id); +int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid); int scoutfs_lock_server_setup(struct super_block *sb); void scoutfs_lock_server_destroy(struct super_block *sb); diff --git a/kmod/src/net.c b/kmod/src/net.c index 8449ce56..1177336d 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -109,7 +109,7 @@ struct scoutfs_net_connection { unsigned long connect_timeout_ms; struct socket *sock; - u64 node_id; /* assigned during greeting */ + u64 rid; u64 greeting_id; struct sockaddr_in sockname; struct sockaddr_in peername; @@ -344,12 +344,12 @@ static void shutdown_conn(struct scoutfs_net_connection *conn) * connection has passed the greeting and isn't being shut down. At all * other times we add new sends to the resend queue. * - * If a non-zero node_id is specified then the conn argument is a listening + * If a non-zero rid is specified then the conn argument is a listening * connection and the connection to send the message down is found by - * searching for the node_id in its accepted connections. + * searching for the rid in its accepted connections. */ static int submit_send(struct super_block *sb, - struct scoutfs_net_connection *conn, u64 node_id, + struct scoutfs_net_connection *conn, u64 rid, u8 cmd, u8 flags, u64 id, u8 net_err, void *data, u16 data_len, scoutfs_net_response_t resp_func, void *resp_data, @@ -376,19 +376,19 @@ static int submit_send(struct super_block *sb, spin_lock_nested(&conn->lock, CONN_LOCK_LISTENER); - if (node_id != 0) { + if (rid != 0) { list_for_each_entry(acc_conn, &conn->accepted_list, accepted_head) { - if (acc_conn->node_id == node_id) { + if (acc_conn->rid == rid) { spin_lock_nested(&acc_conn->lock, CONN_LOCK_ACCEPTED); spin_unlock(&conn->lock); conn = acc_conn; - node_id = 0; + rid = 0; break; } } - if (node_id != 0) { + if (rid != 0) { spin_unlock(&conn->lock); return -ENOTCONN; } @@ -845,7 +845,7 @@ static void destroy_conn(struct scoutfs_net_connection *conn) /* tell callers that accepted connection finally done */ if (conn->listening_conn && conn->notify_down) - conn->notify_down(sb, conn, conn->info, conn->node_id); + conn->notify_down(sb, conn, conn->info, conn->rid); /* free all messages, refactor and complete for forced unmount? */ list_splice_init(&conn->resend_queue, &conn->send_queue); @@ -1182,7 +1182,7 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) conn->shutting_down = 0; if (conn->notify_down) conn->notify_down(sb, conn, conn->info, - conn->node_id); + conn->rid); } spin_unlock(&conn->lock); @@ -1251,12 +1251,12 @@ restart: * Accepted connections inherit the callbacks from their listening * connection. * - * notify_up is called once a valid greeting is received. node_id is + * notify_up is called once a valid greeting is received. rid is * non-zero on accepted sockets once they've seen a valid greeting. - * Connected and listening connections have a node_id of 0. + * Connected and listening connections have a rid of 0. * * notify_down is always called as connections are shut down. It can be - * called without notify_up ever being called. The node_id is only + * called without notify_up ever being called. The rid is only * non-zero for accepted connections. */ struct scoutfs_net_connection * @@ -1317,14 +1317,15 @@ scoutfs_net_alloc_conn(struct super_block *sb, } /* - * Give the caller the client node_id of the connection. This used by - * rare server processing callers who want to send async responses after - * request processing has returned. We didn't want to plumb the - * requesting node_id into all the request handlers but that'd work too. + * Give the caller the client rid of the connection. This used by rare + * server processing callers who want to send async responses after + * request processing has returned. We didn't want the churn of + * providing the requesting rid to all the request handlers, but we + * probably should. */ -u64 scoutfs_net_client_node_id(struct scoutfs_net_connection *conn) +u64 scoutfs_net_client_rid(struct scoutfs_net_connection *conn) { - return conn->node_id; + return conn->rid; } /* @@ -1520,7 +1521,7 @@ void scoutfs_net_client_greeting(struct super_block *sb, /* * The calling server has received a valid greeting from a client. If - * the server is reconnecting to us then we need to find its old + * the client is reconnecting to us then we need to find its old * connection that held its state and transfer it to this connection * (connection and socket life cycles make this easier than migrating * the socket between the connections). @@ -1532,9 +1533,8 @@ void scoutfs_net_client_greeting(struct super_block *sb, * referring to the same original connection. We use the increasing * greeting id to have the most recent connection attempt win. * - * A node can be reconnecting to us for the first time. In that case we - * just trust its node_id. It will notice the new server term and take - * steps to recover. + * A node can be reconnecting to us for the first time. It will notice + * the new server term and take steps to recover. * * A client can be reconnecting to us after we've destroyed their state. * This is fatal for the client if they just took too long to reconnect. @@ -1550,8 +1550,8 @@ void scoutfs_net_client_greeting(struct super_block *sb, */ void scoutfs_net_server_greeting(struct super_block *sb, struct scoutfs_net_connection *conn, - u64 node_id, u64 greeting_id, - bool sent_node_id, bool first_contact, + u64 rid, u64 greeting_id, + bool reconnecting, bool first_contact, bool farewell) { struct scoutfs_net_connection *listener; @@ -1561,15 +1561,15 @@ void scoutfs_net_server_greeting(struct super_block *sb, /* only called on accepted server connections :/ */ BUG_ON(!conn->listening_conn); - /* see if we have a previous conn for the client's sent node_id */ + /* see if we have a previous conn for the client's sent rid */ reconn = NULL; - if (sent_node_id) { + if (reconnecting) { listener = conn->listening_conn; restart: spin_lock_nested(&listener->lock, CONN_LOCK_LISTENER); list_for_each_entry(acc, &listener->accepted_list, accepted_head) { - if (acc->node_id != node_id || + if (acc->rid != rid || acc->greeting_id >= greeting_id || acc->reconn_freeing) continue; @@ -1592,12 +1592,12 @@ restart: } /* drop a connection if we can't find its necessary old conn */ - if (sent_node_id && !reconn && !first_contact && !farewell) { + if (reconnecting && !reconn && !first_contact && !farewell) { shutdown_conn(conn); return; } - /* migrate state from previous conn for this reconnecting node_id */ + /* migrate state from previous conn for this reconnecting rid */ if (reconn) { spin_lock(&conn->lock); @@ -1623,15 +1623,15 @@ restart: spin_lock(&conn->lock); - conn->node_id = node_id; + conn->rid = rid; conn->greeting_id = greeting_id; set_valid_greeting(conn); spin_unlock(&conn->lock); - /* only call notify_up the first time we see the node_id */ + /* only call notify_up the first time we see the rid */ if (conn->notify_up && first_contact) - conn->notify_up(sb, conn, conn->info, node_id); + conn->notify_up(sb, conn, conn->info, rid); } /* @@ -1651,17 +1651,17 @@ int scoutfs_net_submit_request(struct super_block *sb, } /* - * Send a request to a specific node_id that was accepted by this listening + * Send a request to a specific rid that was accepted by this listening * connection. */ int scoutfs_net_submit_request_node(struct super_block *sb, struct scoutfs_net_connection *conn, - u64 node_id, u8 cmd, + u64 rid, u8 cmd, void *arg, u16 arg_len, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret) { - return submit_send(sb, conn, node_id, cmd, 0, 0, 0, arg, arg_len, + return submit_send(sb, conn, rid, cmd, 0, 0, 0, arg, arg_len, resp_func, resp_data, id_ret); } @@ -1687,7 +1687,7 @@ int scoutfs_net_response(struct super_block *sb, int scoutfs_net_response_node(struct super_block *sb, struct scoutfs_net_connection *conn, - u64 node_id, u8 cmd, u64 id, int error, + u64 rid, u8 cmd, u64 id, int error, void *resp, u16 resp_len) { if (error) { @@ -1695,7 +1695,7 @@ int scoutfs_net_response_node(struct super_block *sb, resp_len = 0; } - return submit_send(sb, conn, node_id, cmd, SCOUTFS_NET_FLAG_RESPONSE, + return submit_send(sb, conn, rid, cmd, SCOUTFS_NET_FLAG_RESPONSE, id, net_err_from_host(sb, error), resp, resp_len, NULL, NULL, NULL); } @@ -1787,9 +1787,9 @@ static void net_tseq_show_conn(struct seq_file *m, struct scoutfs_net_connection *conn = container_of(ent, struct scoutfs_net_connection, tseq_entry); - seq_printf(m, "name "SIN_FMT" peer "SIN_FMT" node_id %llu greeting_id %llu vg %u est %u sd %u sg %u sf %u rw %u rf %u cto_ms rdl_j %lu %lu nss %llu rs %llu nsi %llu\n", + seq_printf(m, "name "SIN_FMT" peer "SIN_FMT" rid %016llx greeting_id %llu vg %u est %u sd %u sg %u sf %u rw %u rf %u cto_ms rdl_j %lu %lu nss %llu rs %llu nsi %llu\n", SIN_ARG(&conn->sockname), SIN_ARG(&conn->peername), - conn->node_id, conn->greeting_id, conn->valid_greeting, + conn->rid, conn->greeting_id, conn->valid_greeting, conn->established, conn->shutting_down, conn->saw_greeting, conn->saw_farewell, conn->reconn_wait, conn->reconn_freeing, conn->connect_timeout_ms, conn->reconn_deadline, diff --git a/kmod/src/net.h b/kmod/src/net.h index a15e005a..0113c16c 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -37,14 +37,14 @@ typedef int (*scoutfs_net_response_t)(struct super_block *sb, typedef void (*scoutfs_net_notify_t)(struct super_block *sb, struct scoutfs_net_connection *conn, - void *info, u64 node_id); + void *info, u64 rid); struct scoutfs_net_connection * scoutfs_net_alloc_conn(struct super_block *sb, scoutfs_net_notify_t notify_up, scoutfs_net_notify_t notify_down, size_t info_size, scoutfs_net_request_t *req_funcs, char *name_suffix); -u64 scoutfs_net_client_node_id(struct scoutfs_net_connection *conn); +u64 scoutfs_net_client_rid(struct scoutfs_net_connection *conn); int scoutfs_net_connect(struct super_block *sb, struct scoutfs_net_connection *conn, struct sockaddr_in *sin, unsigned long timeout_ms); @@ -60,8 +60,7 @@ int scoutfs_net_submit_request(struct super_block *sb, void *resp_data, u64 *id_ret); int scoutfs_net_submit_request_node(struct super_block *sb, struct scoutfs_net_connection *conn, - u64 node_id, u8 cmd, - void *arg, u16 arg_len, + u64 rid, u8 cmd, void *arg, u16 arg_len, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret); void scoutfs_net_cancel_request(struct super_block *sb, @@ -76,7 +75,7 @@ int scoutfs_net_response(struct super_block *sb, u8 cmd, u64 id, int error, void *resp, u16 resp_len); int scoutfs_net_response_node(struct super_block *sb, struct scoutfs_net_connection *conn, - u64 node_id, u8 cmd, u64 id, int error, + u64 rid, u8 cmd, u64 id, int error, void *resp, u16 resp_len); void scoutfs_net_shutdown(struct super_block *sb, struct scoutfs_net_connection *conn); @@ -88,8 +87,8 @@ void scoutfs_net_client_greeting(struct super_block *sb, bool new_server); void scoutfs_net_server_greeting(struct super_block *sb, struct scoutfs_net_connection *conn, - u64 node_id, u64 greeting_id, - bool sent_node_id, bool first_contact, + u64 rid, u64 greeting_id, + bool reconnecting, bool first_contact, bool farewell); void scoutfs_net_farewell(struct super_block *sb, struct scoutfs_net_connection *conn); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index d87dcb2c..b452620d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1154,17 +1154,17 @@ TRACE_EVENT(scoutfs_client_compact_stop, ); TRACE_EVENT(scoutfs_server_compact_start, - TP_PROTO(struct super_block *sb, u64 id, u8 level, u64 node_id, + TP_PROTO(struct super_block *sb, u64 id, u8 level, u64 rid, unsigned long client_nr, unsigned long server_nr, unsigned long per_client), - TP_ARGS(sb, id, level, node_id, client_nr, server_nr, per_client), + TP_ARGS(sb, id, level, rid, client_nr, server_nr, per_client), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, id) __field(__u8, level) - __field(__u64, node_id) + __field(__u64, c_rid) __field(unsigned long, client_nr) __field(unsigned long, server_nr) __field(unsigned long, per_client) @@ -1174,40 +1174,40 @@ TRACE_EVENT(scoutfs_server_compact_start, SCSB_TRACE_ASSIGN(sb); __entry->id = id; __entry->level = level; - __entry->node_id = node_id; + __entry->c_rid = rid; __entry->client_nr = client_nr; __entry->server_nr = server_nr; __entry->per_client = per_client; ), - TP_printk(SCSBF" id %llu level %u node_id %llu client_nr %lu server_nr %lu per_client %lu", + TP_printk(SCSBF" id %llu level %u rid %016llx client_nr %lu server_nr %lu per_client %lu", SCSB_TRACE_ARGS, __entry->id, __entry->level, - __entry->node_id, __entry->client_nr, __entry->server_nr, + __entry->c_rid, __entry->client_nr, __entry->server_nr, __entry->per_client) ); TRACE_EVENT(scoutfs_server_compact_done, - TP_PROTO(struct super_block *sb, u64 id, u64 node_id, + TP_PROTO(struct super_block *sb, u64 id, u64 rid, unsigned long server_nr), - TP_ARGS(sb, id, node_id, server_nr), + TP_ARGS(sb, id, rid, server_nr), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, id) - __field(__u64, node_id) + __field(__u64, c_rid) __field(unsigned long, server_nr) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); __entry->id = id; - __entry->node_id = node_id; + __entry->rid = rid; __entry->server_nr = server_nr; ), - TP_printk(SCSBF" id %llu node_id %llu server_nr %lu", - SCSB_TRACE_ARGS, __entry->id, __entry->node_id, + TP_printk(SCSBF" id %llu rid %016llx server_nr %lu", + SCSB_TRACE_ARGS, __entry->id, __entry->c_rid, __entry->server_nr) ); @@ -2470,32 +2470,32 @@ DEFINE_EVENT(scoutfs_segno_class, scoutfs_remove_segno, ); DECLARE_EVENT_CLASS(scoutfs_server_client_count_class, - TP_PROTO(struct super_block *sb, u64 node_id, unsigned long nr_clients), + TP_PROTO(struct super_block *sb, u64 rid, unsigned long nr_clients), - TP_ARGS(sb, node_id, nr_clients), + TP_ARGS(sb, rid, nr_clients), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__s64, node_id) + __field(__s64, c_rid) __field(unsigned long, nr_clients) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->node_id = node_id; + __entry->c_rid = rid; __entry->nr_clients = nr_clients; ), - TP_printk(SCSBF" node_id %llu nr_clients %lu", - SCSB_TRACE_ARGS, __entry->node_id, __entry->nr_clients) + TP_printk(SCSBF" rid %016llx nr_clients %lu", + SCSB_TRACE_ARGS, __entry->c_rid, __entry->nr_clients) ); DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_up, - TP_PROTO(struct super_block *sb, u64 node_id, unsigned long nr_clients), - TP_ARGS(sb, node_id, nr_clients) + TP_PROTO(struct super_block *sb, u64 rid, unsigned long nr_clients), + TP_ARGS(sb, rid, nr_clients) ); DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_down, - TP_PROTO(struct super_block *sb, u64 node_id, unsigned long nr_clients), - TP_ARGS(sb, node_id, nr_clients) + TP_PROTO(struct super_block *sb, u64 rid, unsigned long nr_clients), + TP_ARGS(sb, rid, nr_clients) ); #define slt_symbolic(mode) \ @@ -2509,16 +2509,16 @@ DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_down, TRACE_EVENT(scoutfs_lock_message, TP_PROTO(struct super_block *sb, int who, int what, int dir, - u64 node_id, u64 net_id, struct scoutfs_net_lock *nl), + u64 rid, u64 net_id, struct scoutfs_net_lock *nl), - TP_ARGS(sb, who, what, dir, node_id, net_id, nl), + TP_ARGS(sb, who, what, dir, rid, net_id, nl), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(int, who) __field(int, what) __field(int, dir) - __field(__u64, node_id) + __field(__u64, m_rid) __field(__u64, net_id) sk_trace_define(key) __field(__u8, old_mode) @@ -2530,17 +2530,17 @@ TRACE_EVENT(scoutfs_lock_message, __entry->who = who; __entry->what = what; __entry->dir = dir; - __entry->node_id = node_id; + __entry->m_rid = rid; __entry->net_id = net_id; sk_trace_assign(key, &nl->key); __entry->old_mode = nl->old_mode; __entry->new_mode = nl->new_mode; ), - TP_printk(SCSBF" %s %s %s node_id %llu net_id %llu key "SK_FMT" old_mode %u new_mode %u", + TP_printk(SCSBF" %s %s %s rid %016llx net_id %llu key "SK_FMT" old_mode %u new_mode %u", SCSB_TRACE_ARGS, slt_symbolic(__entry->who), slt_symbolic(__entry->what), slt_symbolic(__entry->dir), - __entry->node_id, __entry->net_id, sk_trace_args(key), + __entry->m_rid, __entry->net_id, sk_trace_args(key), __entry->old_mode, __entry->new_mode) ); @@ -2685,70 +2685,70 @@ DEFINE_EVENT(scoutfs_clock_sync_class, scoutfs_recv_clock_sync, ); TRACE_EVENT(scoutfs_trans_seq_advance, - TP_PROTO(struct super_block *sb, u64 node_id, u64 prev_seq, + TP_PROTO(struct super_block *sb, u64 rid, u64 prev_seq, u64 next_seq), - TP_ARGS(sb, node_id, prev_seq, next_seq), + TP_ARGS(sb, rid, prev_seq, next_seq), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__u64, node_id) + __field(__u64, s_rid) __field(__u64, prev_seq) __field(__u64, next_seq) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->node_id = node_id; + __entry->s_rid = rid; __entry->prev_seq = prev_seq; __entry->next_seq = next_seq; ), - TP_printk(SCSBF" node_id %llu prev_seq %llu next_seq %llu", - SCSB_TRACE_ARGS, __entry->node_id, __entry->prev_seq, + TP_printk(SCSBF" rid %016llx prev_seq %llu next_seq %llu", + SCSB_TRACE_ARGS, __entry->s_rid, __entry->prev_seq, __entry->next_seq) ); TRACE_EVENT(scoutfs_trans_seq_farewell, - TP_PROTO(struct super_block *sb, u64 node_id, u64 trans_seq), + TP_PROTO(struct super_block *sb, u64 rid, u64 trans_seq), - TP_ARGS(sb, node_id, trans_seq), + TP_ARGS(sb, rid, trans_seq), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__u64, node_id) + __field(__u64, s_rid) __field(__u64, trans_seq) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->node_id = node_id; + __entry->s_rid = rid; __entry->trans_seq = trans_seq; ), - TP_printk(SCSBF" node_id %llu trans_seq %llu", - SCSB_TRACE_ARGS, __entry->node_id, __entry->trans_seq) + TP_printk(SCSBF" rid %016llx trans_seq %llu", + SCSB_TRACE_ARGS, __entry->s_rid, __entry->trans_seq) ); TRACE_EVENT(scoutfs_trans_seq_last, - TP_PROTO(struct super_block *sb, u64 node_id, u64 trans_seq), + TP_PROTO(struct super_block *sb, u64 rid, u64 trans_seq), - TP_ARGS(sb, node_id, trans_seq), + TP_ARGS(sb, rid, trans_seq), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__u64, node_id) + __field(__u64, s_rid) __field(__u64, trans_seq) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->node_id = node_id; + __entry->s_rid = rid; __entry->trans_seq = trans_seq; ), - TP_printk(SCSBF" node_id %llu trans_seq %llu", - SCSB_TRACE_ARGS, __entry->node_id, __entry->trans_seq) + TP_printk(SCSBF" rid %016llx trans_seq %llu", + SCSB_TRACE_ARGS, __entry->s_rid, __entry->trans_seq) ); #endif /* _TRACE_SCOUTFS_H */ diff --git a/kmod/src/server.c b/kmod/src/server.c index 291c7df5..4fabfff6 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -99,7 +99,7 @@ struct server_info { * The server tracks each connected client. */ struct server_client_info { - u64 node_id; + u64 rid; struct list_head head; unsigned long nr_compacts; }; @@ -894,7 +894,7 @@ static int server_advance_seq(struct super_block *sb, __le64 their_seq; __le64 next_seq; struct scoutfs_trans_seq_btree_key tsk; - u64 node_id = scoutfs_net_client_node_id(conn); + u64 rid = scoutfs_net_client_rid(conn); int ret; if (arg_len != sizeof(__le64)) { @@ -908,7 +908,7 @@ static int server_advance_seq(struct super_block *sb, if (their_seq != 0) { tsk.trans_seq = le64_to_be64(their_seq); - tsk.node_id = cpu_to_be64(node_id); + tsk.rid = cpu_to_be64(rid); ret = scoutfs_btree_delete(sb, &super->trans_seqs, &tsk, sizeof(tsk)); @@ -919,11 +919,11 @@ static int server_advance_seq(struct super_block *sb, next_seq = super->next_trans_seq; le64_add_cpu(&super->next_trans_seq, 1); - trace_scoutfs_trans_seq_advance(sb, node_id, le64_to_cpu(their_seq), + trace_scoutfs_trans_seq_advance(sb, rid, le64_to_cpu(their_seq), le64_to_cpu(next_seq)); tsk.trans_seq = le64_to_be64(next_seq); - tsk.node_id = cpu_to_be64(node_id); + tsk.rid = cpu_to_be64(rid); ret = scoutfs_btree_insert(sb, &super->trans_seqs, &tsk, sizeof(tsk), NULL, 0); @@ -946,7 +946,7 @@ out: * client's farewell is retransmitted so it's OK to not find any * entries. This is called with the server commit rwsem held. */ -static int remove_trans_seq(struct super_block *sb, u64 node_id) +static int remove_trans_seq(struct super_block *sb, u64 rid) { DECLARE_SERVER_INFO(sb, server); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -958,7 +958,7 @@ static int remove_trans_seq(struct super_block *sb, u64 node_id) down_write(&server->seq_rwsem); tsk.trans_seq = 0; - tsk.node_id = 0; + tsk.rid = 0; for (;;) { ret = scoutfs_btree_next(sb, &super->trans_seqs, @@ -972,8 +972,8 @@ static int remove_trans_seq(struct super_block *sb, u64 node_id) memcpy(&tsk, iref.key, iref.key_len); scoutfs_btree_put_iref(&iref); - if (be64_to_cpu(tsk.node_id) == node_id) { - trace_scoutfs_trans_seq_farewell(sb, node_id, + if (be64_to_cpu(tsk.rid) == rid) { + trace_scoutfs_trans_seq_farewell(sb, rid, be64_to_cpu(tsk.trans_seq)); ret = scoutfs_btree_delete(sb, &super->trans_seqs, &tsk, sizeof(tsk)); @@ -981,7 +981,7 @@ static int remove_trans_seq(struct super_block *sb, u64 node_id) } be64_add_cpu(&tsk.trans_seq, 1); - tsk.node_id = 0; + tsk.rid = 0; } up_write(&server->seq_rwsem); @@ -1006,7 +1006,7 @@ static int server_get_last_seq(struct super_block *sb, struct scoutfs_super_block *super = &sbi->super; struct scoutfs_trans_seq_btree_key tsk; SCOUTFS_BTREE_ITEM_REF(iref); - u64 node_id = scoutfs_net_client_node_id(conn); + u64 rid = scoutfs_net_client_rid(conn); __le64 last_seq = 0; int ret; @@ -1018,7 +1018,7 @@ static int server_get_last_seq(struct super_block *sb, down_read(&server->seq_rwsem); tsk.trans_seq = 0; - tsk.node_id = 0; + tsk.rid = 0; ret = scoutfs_btree_next(sb, &super->trans_seqs, &tsk, sizeof(tsk), &iref); @@ -1037,7 +1037,7 @@ static int server_get_last_seq(struct super_block *sb, ret = 0; } - trace_scoutfs_trans_seq_last(sb, node_id, le64_to_cpu(last_seq)); + trace_scoutfs_trans_seq_last(sb, rid, le64_to_cpu(last_seq)); up_read(&server->seq_rwsem); out: @@ -1107,12 +1107,12 @@ static int server_lock(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, void *arg, u16 arg_len) { - u64 node_id = scoutfs_net_client_node_id(conn); + u64 rid = scoutfs_net_client_rid(conn); if (arg_len != sizeof(struct scoutfs_net_lock)) return -EINVAL; - return scoutfs_lock_server_request(sb, node_id, id, arg); + return scoutfs_lock_server_request(sb, rid, id, arg); } static int lock_response(struct super_block *sb, @@ -1120,31 +1120,31 @@ static int lock_response(struct super_block *sb, void *resp, unsigned int resp_len, int error, void *data) { - u64 node_id = scoutfs_net_client_node_id(conn); + u64 rid = scoutfs_net_client_rid(conn); if (resp_len != sizeof(struct scoutfs_net_lock)) return -EINVAL; - return scoutfs_lock_server_response(sb, node_id, resp); + return scoutfs_lock_server_response(sb, rid, resp); } -int scoutfs_server_lock_request(struct super_block *sb, u64 node_id, +int scoutfs_server_lock_request(struct super_block *sb, u64 rid, struct scoutfs_net_lock *nl) { struct server_info *server = SCOUTFS_SB(sb)->server_info; - return scoutfs_net_submit_request_node(sb, server->conn, node_id, + return scoutfs_net_submit_request_node(sb, server->conn, rid, SCOUTFS_NET_CMD_LOCK, nl, sizeof(*nl), lock_response, NULL, NULL); } -int scoutfs_server_lock_response(struct super_block *sb, u64 node_id, +int scoutfs_server_lock_response(struct super_block *sb, u64 rid, u64 id, struct scoutfs_net_lock *nl) { struct server_info *server = SCOUTFS_SB(sb)->server_info; - return scoutfs_net_response_node(sb, server->conn, node_id, + return scoutfs_net_response_node(sb, server->conn, rid, SCOUTFS_NET_CMD_LOCK, id, 0, nl, sizeof(*nl)); } @@ -1162,34 +1162,34 @@ static int lock_recover_response(struct super_block *sb, void *resp, unsigned int resp_len, int error, void *data) { - u64 node_id = scoutfs_net_client_node_id(conn); + u64 rid = scoutfs_net_client_rid(conn); if (invalid_recover(resp, resp_len)) return -EINVAL; - return scoutfs_lock_server_recover_response(sb, node_id, resp); + return scoutfs_lock_server_recover_response(sb, rid, resp); } -int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, +int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, struct scoutfs_key *key) { struct server_info *server = SCOUTFS_SB(sb)->server_info; - return scoutfs_net_submit_request_node(sb, server->conn, node_id, + return scoutfs_net_submit_request_node(sb, server->conn, rid, SCOUTFS_NET_CMD_LOCK_RECOVER, key, sizeof(*key), lock_recover_response, NULL, NULL); } -static int insert_mounted_client(struct super_block *sb, u64 node_id, +static int insert_mounted_client(struct super_block *sb, u64 rid, u64 gr_flags) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_mounted_client_btree_key mck; struct scoutfs_mounted_client_btree_val mcv; - mck.node_id = cpu_to_be64(node_id); + mck.rid = cpu_to_be64(rid); mcv.flags = 0; if (gr_flags & SCOUTFS_NET_GREETING_FLAG_VOTER) mcv.flags |= SCOUTFS_MOUNTED_CLIENT_VOTER; @@ -1208,13 +1208,13 @@ static int insert_mounted_client(struct super_block *sb, u64 node_id, * * The caller has to serialize with farewell processing. */ -static int delete_mounted_client(struct super_block *sb, u64 node_id) +static int delete_mounted_client(struct super_block *sb, u64 rid) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_mounted_client_btree_key mck; int ret; - mck.node_id = cpu_to_be64(node_id); + mck.rid = cpu_to_be64(rid); ret = scoutfs_btree_delete(sb, &super->mounted_clients, &mck, sizeof(mck)); @@ -1230,17 +1230,8 @@ static int delete_mounted_client(struct super_block *sb, u64 node_id) * log some detail before shutting down. A failure to send a greeting * response shuts down the connection. * - * We allocate a new node_id for the first connect attempt from a - * client. - * - * If a client reconnects they'll send their initially assigned node_id - * in their greeting request. - * - * XXX We can lose allocated node_ids here as we record the node_id as - * live as we send a valid greeting response. The client might - * disconnect before they receive the response and resent and initial - * blank greeting. We could use a client uuid to associate with - * allocated node_ids. + * If a client reconnects they'll send their previously received + * serer_term in their greeting request. * * XXX The logic of this has gotten convoluted. The lock server can * send a recovery request so it needs to be called after the core net @@ -1259,9 +1250,8 @@ static int server_greeting(struct super_block *sb, struct scoutfs_net_greeting greet; DECLARE_SERVER_INFO(sb, server); struct commit_waiter cw; - __le64 node_id = 0; __le64 umb = 0; - bool sent_node_id; + bool reconnecting; bool first_contact; bool farewell; int ret = 0; @@ -1288,17 +1278,15 @@ static int server_greeting(struct super_block *sb, goto send_err; } - if (gr->node_id == 0) { + if (gr->server_term == 0) { down_read(&server->commit_rwsem); spin_lock(&server->lock); - node_id = super->next_node_id; - le64_add_cpu(&super->next_node_id, 1); umb = super->unmount_barrier; spin_unlock(&server->lock); mutex_lock(&server->farewell_mutex); - ret = insert_mounted_client(sb, le64_to_cpu(node_id), + ret = insert_mounted_client(sb, le64_to_cpu(gr->rid), le64_to_cpu(gr->flags)); mutex_unlock(&server->farewell_mutex); @@ -1310,20 +1298,17 @@ static int server_greeting(struct super_block *sb, queue_work(server->wq, &server->farewell_work); } } else { - node_id = gr->node_id; umb = gr->unmount_barrier; } send_err: err = ret; - if (err) - node_id = 0; greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; greet.server_term = cpu_to_le64(server->term); greet.unmount_barrier = umb; - greet.node_id = node_id; + greet.rid = gr->rid; greet.flags = 0; /* queue greeting response to be sent first once messaging enabled */ @@ -1335,15 +1320,15 @@ send_err: goto out; /* have the net core enable messaging and resend */ - sent_node_id = gr->node_id != 0; + reconnecting = gr->server_term != 0; first_contact = le64_to_cpu(gr->server_term) != server->term; if (gr->flags & cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL)) farewell = true; else farewell = false; - scoutfs_net_server_greeting(sb, conn, le64_to_cpu(node_id), id, - sent_node_id, first_contact, farewell); + scoutfs_net_server_greeting(sb, conn, le64_to_cpu(gr->rid), id, + reconnecting, first_contact, farewell); /* lock server might send recovery request */ if (le64_to_cpu(gr->server_term) != server->term) { @@ -1351,7 +1336,7 @@ send_err: /* we're now doing two commits per greeting, not great */ down_read(&server->commit_rwsem); - ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(node_id), + ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid), gr->server_term != 0); if (ret == 0) queue_commit_work(server, &cw); @@ -1369,7 +1354,7 @@ out: struct farewell_request { struct list_head entry; u64 net_id; - u64 node_id; + u64 rid; }; static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) @@ -1435,7 +1420,7 @@ static void farewell_worker(struct work_struct *work) /* count how many reqs requests are from voting clients */ nr_unmounting = 0; list_for_each_entry_safe(fw, tmp, &reqs, entry) { - mck.node_id = cpu_to_be64(fw->node_id); + mck.rid = cpu_to_be64(fw->rid); ret = scoutfs_btree_lookup(sb, &super->mounted_clients, &mck, sizeof(mck), &iref); if (ret == 0 && invalid_mounted_client_item(&iref)) { @@ -1484,7 +1469,7 @@ static void farewell_worker(struct work_struct *work) nr_mounted++; scoutfs_btree_put_iref(&iref); - be64_add_cpu(&mck.node_id, 1); + be64_add_cpu(&mck.rid, 1); } @@ -1505,9 +1490,9 @@ static void farewell_worker(struct work_struct *work) down_read(&server->commit_rwsem); - ret = scoutfs_lock_server_farewell(sb, fw->node_id) ?: - remove_trans_seq(sb, fw->node_id) ?: - delete_mounted_client(sb, fw->node_id); + ret = scoutfs_lock_server_farewell(sb, fw->rid) ?: + remove_trans_seq(sb, fw->rid) ?: + delete_mounted_client(sb, fw->rid); if (ret == 0) queue_commit_work(server, &cw); @@ -1532,7 +1517,7 @@ static void farewell_worker(struct work_struct *work) /* and finally send all the responses */ list_for_each_entry_safe(fw, tmp, &send, entry) { - ret = scoutfs_net_response_node(sb, server->conn, fw->node_id, + ret = scoutfs_net_response_node(sb, server->conn, fw->rid, SCOUTFS_NET_CMD_FAREWELL, fw->net_id, 0, NULL, 0); if (ret) @@ -1556,7 +1541,7 @@ out: queue_work(server->wq, &server->farewell_work); } -static void free_farewell_requests(struct super_block *sb, u64 node_id) +static void free_farewell_requests(struct super_block *sb, u64 rid) { struct server_info *server = SCOUTFS_SB(sb)->server_info; struct farewell_request *tmp; @@ -1564,7 +1549,7 @@ static void free_farewell_requests(struct super_block *sb, u64 node_id) mutex_lock(&server->farewell_mutex); list_for_each_entry_safe(fw, tmp, &server->farewell_requests, entry) { - if (node_id == 0 || fw->node_id == node_id) { + if (rid == 0 || fw->rid == rid) { list_del_init(&fw->entry); kfree(fw); } @@ -1588,7 +1573,7 @@ static int server_farewell(struct super_block *sb, u8 cmd, u64 id, void *arg, u16 arg_len) { struct server_info *server = SCOUTFS_SB(sb)->server_info; - u64 node_id = scoutfs_net_client_node_id(conn); + u64 rid = scoutfs_net_client_rid(conn); struct farewell_request *fw; if (arg_len != 0) @@ -1600,7 +1585,7 @@ static int server_farewell(struct super_block *sb, if (fw == NULL) return -ENOMEM; - fw->node_id = node_id; + fw->rid = rid; fw->net_id = id; mutex_lock(&server->farewell_mutex); @@ -1616,13 +1601,13 @@ static int server_farewell(struct super_block *sb, /* requests sent to clients are tracked so we can free resources */ struct compact_request { struct list_head head; - u64 node_id; + u64 rid; struct scoutfs_net_compact_request req; }; /* * Find a node that can process our compaction request. Return a - * node_id if we found a client and added the compaction to the client + * rid if we found a client and added the compaction to the client * and server counts. Returns 0 if no suitable clients were found. */ static u64 compact_request_start(struct super_block *sb, @@ -1631,7 +1616,7 @@ static u64 compact_request_start(struct super_block *sb, struct server_info *server = SCOUTFS_SB(sb)->server_info; struct server_client_info *last; struct server_client_info *sci; - u64 node_id = 0; + u64 rid = 0; spin_lock(&server->lock); @@ -1650,8 +1635,8 @@ static u64 compact_request_start(struct super_block *sb, list_add(&cr->head, &server->compacts); server->nr_compacts++; sci->nr_compacts++; - node_id = sci->node_id; - cr->node_id = node_id; + rid = sci->rid; + cr->rid = rid; break; } if (sci == last) @@ -1659,14 +1644,14 @@ static u64 compact_request_start(struct super_block *sb, } trace_scoutfs_server_compact_start(sb, le64_to_cpu(cr->req.id), - cr->req.ents[0].level, node_id, - node_id ? sci->nr_compacts : 0, + cr->req.ents[0].level, rid, + rid ? sci->nr_compacts : 0, server->nr_compacts, server->compacts_per_client); spin_unlock(&server->lock); - return node_id; + return rid; } /* @@ -1688,7 +1673,7 @@ static struct compact_request *compact_request_done(struct super_block *sb, continue; list_for_each_entry(sci, &server->clients, head) { - if (sci->node_id == cr->node_id) { + if (sci->rid == cr->rid) { sci->nr_compacts--; break; } @@ -1700,7 +1685,7 @@ static struct compact_request *compact_request_done(struct super_block *sb, break; } - trace_scoutfs_server_compact_done(sb, id, ret ? ret->node_id : 0, + trace_scoutfs_server_compact_done(sb, id, ret ? ret->rid : 0, server->nr_compacts); spin_unlock(&server->lock); @@ -1728,7 +1713,7 @@ static void forget_client_compacts(struct super_block *sb, spin_lock(&server->lock); list_for_each_entry_safe(cr, pos, &server->compacts, head) { - if (cr->node_id == sci->node_id) { + if (cr->rid == sci->rid) { sci->nr_compacts--; server->nr_compacts--; list_move(&cr->head, &forget); @@ -2129,7 +2114,7 @@ static void scoutfs_server_compact_worker(struct work_struct *work) struct compact_request *cr; struct commit_waiter cw; int nr_segnos = 0; - u64 node_id; + u64 rid; __le64 id; int ret; @@ -2169,14 +2154,14 @@ static void scoutfs_server_compact_worker(struct work_struct *work) /* try to send to a node with capacity, they can disconnect */ retry: req->id = id; - node_id = compact_request_start(sb, cr); - if (node_id == 0) { + rid = compact_request_start(sb, cr); + if (rid == 0) { ret = 0; goto out; } /* response processing can complete compaction before this returns */ - ret = scoutfs_net_submit_request_node(sb, server->conn, node_id, + ret = scoutfs_net_submit_request_node(sb, server->conn, rid, SCOUTFS_NET_CMD_COMPACT, req, sizeof(*req), compact_response, NULL, NULL); @@ -2228,18 +2213,18 @@ static scoutfs_net_request_t server_req_funcs[] = { static void server_notify_up(struct super_block *sb, struct scoutfs_net_connection *conn, - void *info, u64 node_id) + void *info, u64 rid) { struct server_client_info *sci = info; DECLARE_SERVER_INFO(sb, server); - if (node_id != 0) { - sci->node_id = node_id; + if (rid != 0) { + sci->rid = rid; sci->nr_compacts = 0; spin_lock(&server->lock); list_add_tail(&sci->head, &server->clients); server->nr_clients++; - trace_scoutfs_server_client_up(sb, node_id, server->nr_clients); + trace_scoutfs_server_client_up(sb, rid, server->nr_clients); spin_unlock(&server->lock); try_queue_compact(server); @@ -2248,20 +2233,20 @@ static void server_notify_up(struct super_block *sb, static void server_notify_down(struct super_block *sb, struct scoutfs_net_connection *conn, - void *info, u64 node_id) + void *info, u64 rid) { struct server_client_info *sci = info; DECLARE_SERVER_INFO(sb, server); - if (node_id != 0) { + if (rid != 0) { spin_lock(&server->lock); list_del_init(&sci->head); server->nr_clients--; - trace_scoutfs_server_client_down(sb, node_id, + trace_scoutfs_server_client_down(sb, rid, server->nr_clients); spin_unlock(&server->lock); - free_farewell_requests(sb, node_id); + free_farewell_requests(sb, rid); forget_client_compacts(sb, sci); try_queue_compact(server); diff --git a/kmod/src/server.h b/kmod/src/server.h index 32c6ccea..83103b9e 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -64,11 +64,11 @@ void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, void scoutfs_init_ment_from_net(struct scoutfs_manifest_entry *ment, struct scoutfs_net_manifest_entry *net_ment); -int scoutfs_server_lock_request(struct super_block *sb, u64 node_id, +int scoutfs_server_lock_request(struct super_block *sb, u64 rid, struct scoutfs_net_lock *nl); -int scoutfs_server_lock_response(struct super_block *sb, u64 node_id, +int scoutfs_server_lock_response(struct super_block *sb, u64 rid, u64 id, struct scoutfs_net_lock *nl); -int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, +int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, struct scoutfs_key *key); struct sockaddr_in; diff --git a/kmod/src/super.c b/kmod/src/super.c index c5a8aeef..f160e1a3 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -429,7 +429,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_quorum_setup(sb) ?: scoutfs_server_setup(sb) ?: scoutfs_client_setup(sb) ?: - scoutfs_client_wait_node_id(sb) ?: scoutfs_lock_rid(sb, SCOUTFS_LOCK_WRITE, 0, sbi->rid, &sbi->rid_lock); if (ret) diff --git a/kmod/src/super.h b/kmod/src/super.h index 741f2ddc..0545b34c 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -32,7 +32,6 @@ struct scoutfs_sb_info { /* assigned once at the start of each mount, read-only */ u64 rid; - u64 node_id; struct scoutfs_lock *rid_lock; struct scoutfs_super_block super;