From 8b3193ea724883f5ffd4cc970ab24cf1b97ec690 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 30 Jul 2018 15:24:45 -0700 Subject: [PATCH] scoutfs: server allocates node_id Today node_ids are randomly assigned. This adds the risk of failure from random number generation and still allows for the risk of collisions. Switch to assigning strictly advancing node_ids on the server during the initial connection greeting message exchange. This simplifies the system and allows us to derive information from the relative values of node_ids in the system. To do this we refactor the greeting code from internal to the net layer to proper client and server request and response processing. This lets the server manage persistent node_id storage and allows the client to wait for a node_id during mount. Now that net_connect is sync in the client we don't need the notify_up callback anymore. The client can perform those duties when the connect returns. The net code still has to snoop on request and response processing to see when the greetings have been exchange and allow messages to flow. Signed-off-by: Zach Brown --- kmod/src/client.c | 116 ++++++++++++++++--- kmod/src/client.h | 1 + kmod/src/format.h | 2 + kmod/src/net.c | 275 +++++++++++++++++++--------------------------- kmod/src/net.h | 11 +- kmod/src/server.c | 71 ++++++++++++ kmod/src/super.c | 7 +- 7 files changed, 295 insertions(+), 188 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index a21cbefd..81fd7fb7 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -51,6 +51,7 @@ struct client_info { struct super_block *sb; struct scoutfs_net_connection *conn; + struct completion node_id_comp; atomic_t shutting_down; struct workqueue_struct *workq; @@ -230,13 +231,78 @@ int scoutfs_client_statfs(struct super_block *sb, sizeof(struct scoutfs_net_statfs)); } +/* + * Process a greeting response in the client from the server. This is + * called for every connected socket on the connection. The first + * response will have the node_id that the server assigned the client. + */ +static int client_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, int error, + void *data) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_net_greeting *gr = resp; + int ret = 0; + + if (error) { + ret = error; + goto out; + } + + if (resp_len != sizeof(struct scoutfs_net_greeting)) { + ret = -EINVAL; + goto out; + } + + if (gr->fsid != super->id) { + scoutfs_warn(sb, "server sent fsid 0x%llx, client has 0x%llx", + le64_to_cpu(gr->fsid), + le64_to_cpu(super->id)); + ret = -EINVAL; + goto out; + } + + if (gr->format_hash != super->format_hash) { + scoutfs_warn(sb, "server sent format 0x%llx, client has 0x%llx", + le64_to_cpu(gr->format_hash), + le64_to_cpu(super->format_hash)); + ret = -EINVAL; + goto out; + } + + if (sbi->node_id != 0 && le64_to_cpu(gr->node_id) != sbi->node_id) { + scoutfs_warn(sb, "server sent node_id %llu, client has %llu", + le64_to_cpu(gr->node_id), + sbi->node_id); + ret = -EINVAL; + goto out; + } + + if (sbi->node_id == 0 && gr->node_id == 0) { + scoutfs_warn(sb, "server sent node_id 0, client also has 0\n"); + ret = -EINVAL; + goto out; + } + + if (sbi->node_id == 0) { + sbi->node_id = le64_to_cpu(gr->node_id); + complete(&client->node_id_comp); + } + +out: + return ret; +} + /* * Attempt to connect to the listening address that the server wrote in * the super block. We keep trying indefinitely with an increasing * delay if we fail to either read the address or connect to it. * * We're careful to only ever have one connection attempt in flight. We - * only queue this work on mount, on error, or from the connection + * only queue this work on mount, on error, or from the notify_down * callback. */ static void scoutfs_client_connect_worker(struct work_struct *work) @@ -244,6 +310,8 @@ static void scoutfs_client_connect_worker(struct work_struct *work) struct client_info *client = container_of(work, struct client_info, connect_dwork.work); struct super_block *sb = client->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_net_greeting greet; struct scoutfs_super_block super; struct sockaddr_in sin; int ret; @@ -262,8 +330,24 @@ static void scoutfs_client_connect_worker(struct work_struct *work) sin.sin_addr.s_addr = le32_to_be32(super.server_addr.addr); sin.sin_port = le16_to_be16(super.server_addr.port); - scoutfs_net_connect(sb, client->conn, &sin, client->conn_retry_ms); - ret = 0; + ret = scoutfs_net_connect(sb, client->conn, &sin, + client->conn_retry_ms); + if (ret) + goto out; + + reset_connect_timeout(client); + + /* send a greeting to verify endpoints of each connection */ + greet.fsid = super.id; + greet.format_hash = super.format_hash; + greet.node_id = cpu_to_le64(sbi->node_id); + + ret = scoutfs_net_submit_greeting_request(sb, client->conn, + &greet, sizeof(greet), + client_greeting, NULL); + if (ret) + scoutfs_net_shutdown(sb, client->conn); + out: if (ret && !atomic_read(&client->shutting_down)) { queue_delayed_work(client->workq, &client->connect_dwork, @@ -272,14 +356,6 @@ out: } } -static void client_notify_up(struct super_block *sb, - struct scoutfs_net_connection *conn) -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - - reset_connect_timeout(client); -} - /* * Called when either a connect attempt or established connection times * out and fails. @@ -296,6 +372,18 @@ static void client_notify_down(struct super_block *sb, } } +/* + * Wait for the first connected socket on the connection that assigns + * the node_id that will be used for the rest of the life time of the + * mount. + */ +int scoutfs_client_wait_node_id(struct super_block *sb) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return wait_for_completion_interruptible(&client->node_id_comp); +} + int scoutfs_client_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -310,14 +398,14 @@ int scoutfs_client_setup(struct super_block *sb) sbi->client_info = client; client->sb = sb; + init_completion(&client->node_id_comp); atomic_set(&client->shutting_down, 0); INIT_DELAYED_WORK(&client->connect_dwork, scoutfs_client_connect_worker); /* client doesn't process any incoming requests yet */ - client->conn = scoutfs_net_alloc_conn(sb, client_notify_up, - client_notify_down, NULL, - "client"); + client->conn = scoutfs_net_alloc_conn(sb, NULL, client_notify_down, + NULL, "client"); if (!client->conn) { ret = -ENOMEM; goto out; diff --git a/kmod/src/client.h b/kmod/src/client.h index f0a8d609..410592eb 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -18,6 +18,7 @@ int scoutfs_client_get_manifest_root(struct super_block *sb, int scoutfs_client_statfs(struct super_block *sb, struct scoutfs_net_statfs *nstatfs); +int scoutfs_client_wait_node_id(struct super_block *sb); int scoutfs_client_setup(struct super_block *sb); void scoutfs_client_destroy(struct super_block *sb); diff --git a/kmod/src/format.h b/kmod/src/format.h index 5a01f57b..95f732c9 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -368,6 +368,7 @@ struct scoutfs_super_block { __le64 alloc_cursor; struct scoutfs_btree_ring bring; __le64 next_seg_seq; + __le64 next_node_id; struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; struct scoutfs_inet_addr server_addr; @@ -514,6 +515,7 @@ struct scoutfs_lock_name { struct scoutfs_net_greeting { __le64 fsid; __le64 format_hash; + __le64 node_id; } __packed; /* diff --git a/kmod/src/net.c b/kmod/src/net.c index 3ebc062a..775a25d7 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -50,7 +50,6 @@ * deliver messages across renumbering. * * XXX: - * - assign node_ids and validate with the greeting * - defer accepted conn destruction until reconnect timeout * - trace command and response data payloads * - checksum message contents? @@ -77,6 +76,7 @@ struct scoutfs_net_connection { scoutfs_net_request_t *req_funcs; spinlock_t lock; + wait_queue_head_t waitq; unsigned long valid_greeting:1, /* other commands can proceed */ established:1, /* added sends queue send work */ @@ -92,7 +92,6 @@ struct scoutfs_net_connection { struct list_head accepted_head; struct scoutfs_net_connection *listening_conn; struct list_head accepted_list; - wait_queue_head_t accepted_waitq; u64 next_send_id; u64 last_proc_id; @@ -371,124 +370,28 @@ static int submit_send(struct super_block *sb, } /* - * Messages can flow once we receive a valid greeting from our peer. - * Response callers are already called under the lock, request callers - * need to acquire it. + * Messages can flow once we receive and process a valid greeting from + * our peer. * - * At this point greeting request processing has queued the greeting - * response message on the send queue. All the sends waiting to be - * resent need to be added to the end of the send queue after the - * greeting response. Greeting acks are sent differently and can be - * received after resend messages. + * At this point recv processing has queued the greeting response or ack + * message on the send queue. All the sends waiting to be resent need + * to be added to the end of the send queue after the greeting message. */ static void saw_valid_greeting(struct scoutfs_net_connection *conn) { struct super_block *sb = conn->sb; - assert_spin_locked(&conn->lock); + spin_lock(&conn->lock); conn->valid_greeting = 1; if (conn->notify_up) conn->notify_up(sb, conn); list_splice_tail_init(&conn->resend_queue, &conn->send_queue); queue_work(conn->workq, &conn->send_work); + + spin_unlock(&conn->lock); } -static int greeting_response(struct super_block *sb, - struct scoutfs_net_connection *conn, - void *resp, unsigned int resp_len, int error, - void *data) -{ - struct scoutfs_net_greeting *gr = resp; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - int ret = 0; - - if (error) { - ret = error; - goto out; - } - - if (resp_len != sizeof(struct scoutfs_net_greeting)) { - ret = -EINVAL; - goto out; - } - - if (gr->fsid != super->id) { - scoutfs_warn(sb, "server "SIN_FMT" has fsid 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(gr->fsid), - le64_to_cpu(super->id)); - ret = -EINVAL; - goto out; - } - - if (gr->format_hash != super->format_hash) { - scoutfs_warn(sb, "server "SIN_FMT" has format hash 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(gr->format_hash), - le64_to_cpu(super->format_hash)); - ret = -EINVAL; - goto out; - } - - saw_valid_greeting(conn); - -out: - return ret; -} - -/* - * Process an incoming greeting request. We try to send responses to - * failed greetings so that the sender can log some detail before - * shutting down. A failure to send a greeting response shuts down the - * connection. - */ -static int greeting_request(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len) -{ - struct scoutfs_net_greeting *gr = arg; - struct scoutfs_net_greeting greet; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - int ret = 0; - - if (arg_len != sizeof(struct scoutfs_net_greeting)) { - ret = -EINVAL; - goto out; - } - - if (gr->fsid != super->id) { - scoutfs_warn(sb, "client "SIN_FMT" has fsid 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(gr->fsid), - le64_to_cpu(super->id)); - ret = -EINVAL; - goto out; - } - - if (gr->format_hash != super->format_hash) { - scoutfs_warn(sb, "client "SIN_FMT" has format hash 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(gr->format_hash), - le64_to_cpu(super->format_hash)); - ret = -EINVAL; - goto out; - } - - greet.fsid = super->id; - greet.format_hash = super->format_hash; -out: - ret = scoutfs_net_response(sb, conn, cmd, id, ret, - &greet, sizeof(greet)); - if (ret == 0) { - spin_lock(&conn->lock); - saw_valid_greeting(conn); - spin_unlock(&conn->lock); - } - return ret; -} - - /* * Process an incoming response. The greeting should ensure that the * sender won't send us unknown commands. We return an error if we see @@ -500,20 +403,27 @@ static int process_request(struct scoutfs_net_connection *conn, struct message_recv *mrecv) { struct super_block *sb = conn->sb; - scoutfs_net_request_t req_func = NULL; + scoutfs_net_request_t req_func; + int ret; - if (conn->listening_conn != NULL && - mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING) { - req_func = greeting_request; - } else if (mrecv->nh.cmd < SCOUTFS_NET_CMD_UNKNOWN) { + if (mrecv->nh.cmd < SCOUTFS_NET_CMD_UNKNOWN) req_func = conn->req_funcs[mrecv->nh.cmd]; - } if (req_func == NULL) { + else + req_func = NULL; + + if (req_func == NULL) { scoutfs_inc_counter(sb, net_unknown_request); return -EINVAL; } - return req_func(sb, conn, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), - mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); + ret = req_func(sb, conn, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), + mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); + + if (!conn->valid_greeting && + mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING && ret == 0) + saw_valid_greeting(conn); + + return ret; } /* @@ -547,6 +457,11 @@ static int process_response(struct scoutfs_net_connection *conn, ret = submit_send(sb, conn, SCOUTFS_NET_MSG_ACK, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), 0, NULL, 0, NULL, NULL, NULL); + + if (!conn->valid_greeting && + mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING && msend && ret == 0) + saw_valid_greeting(conn); + return ret; } @@ -883,7 +798,7 @@ static void destroy_conn(struct scoutfs_net_connection *conn) spin_lock(&listener->lock); list_del_init(&conn->accepted_head); if (list_empty(&listener->accepted_list)) - wake_up(&listener->accepted_waitq); + wake_up(&listener->waitq); spin_unlock(&listener->lock); } @@ -1038,8 +953,7 @@ static void scoutfs_net_connect_worker(struct work_struct *work) { DEFINE_CONN_FROM_WORK(conn, work, connect_work); struct super_block *sb = conn->sb; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_net_greeting greet; + struct message_send *msend; struct socket *sock; struct timeval tv; int ret; @@ -1074,26 +988,29 @@ static void scoutfs_net_connect_worker(struct work_struct *work) if (ret) goto out; - /* greeting is about to queue send work */ - spin_lock(&conn->lock); - conn->established = 1; - spin_unlock(&conn->lock); - - queue_work(conn->workq, &conn->recv_work); - - /* queue a new updated greeting send */ - greet.fsid = super->id; - greet.format_hash = super->format_hash; - - ret = submit_send(sb, conn, SCOUTFS_NET_MSG_REQUEST, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING, 0, - &greet, sizeof(greet), greeting_response, NULL, NULL); - if (ret) - goto out; - scoutfs_info(sb, "client connected "SIN_FMT" -> "SIN_FMT, SIN_ARG(&conn->sockname), SIN_ARG(&conn->peername)); + + spin_lock(&conn->lock); + + /* clear greeting state for next negotiation */ + conn->valid_greeting = 0; + msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: + find_send(conn, SCOUTFS_NET_MSG_RESPONSE, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: + find_send(conn, SCOUTFS_NET_MSG_ACK, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING); + if (msend) + complete_send(conn, msend, NULL, 0, 0); + + conn->established = 1; + wake_up(&conn->waitq); + + spin_unlock(&conn->lock); + + queue_work(conn->workq, &conn->recv_work); out: if (ret) shutdown_conn(conn); @@ -1135,7 +1052,6 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) DEFINE_CONN_FROM_WORK(conn, work, shutdown_work); struct super_block *sb = conn->sb; struct scoutfs_net_connection *acc_conn; - struct message_send *msend; trace_scoutfs_net_shutdown_work_enter(sb, 0, 0); @@ -1160,6 +1076,8 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) conn->sock = NULL; } + memset(&conn->peername, 0, sizeof(conn->peername)); + /* listening connections shut down all the connections they accepted */ spin_lock_nested(&conn->lock, CONN_LOCK_LISTENER); list_for_each_entry(acc_conn, &conn->accepted_list, accepted_head) { @@ -1168,28 +1086,16 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) spin_unlock(&acc_conn->lock); } spin_unlock(&conn->lock); - wait_event(conn->accepted_waitq, empty_accepted_list(conn)); + wait_event(conn->waitq, empty_accepted_list(conn)); spin_lock(&conn->lock); - /* all queued sends will be resent, protocol handles dupes */ list_splice_tail_init(&conn->send_queue, &conn->resend_queue); - - /* clear greeting state for next negotiation */ - conn->valid_greeting = 0; - msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: - find_send(conn, SCOUTFS_NET_MSG_RESPONSE, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: - find_send(conn, SCOUTFS_NET_MSG_ACK, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING); - if (msend) - complete_send(conn, msend, NULL, 0, 0); - + /* signal connect failure */ + memset(&conn->connect_sin, 0, sizeof(conn->connect_sin)); + wake_up(&conn->waitq); spin_unlock(&conn->lock); - memset(&conn->peername, 0, sizeof(conn->peername)); - /* tell the caller that the connection is down */ if (conn->notify_down) conn->notify_down(sb, conn); @@ -1215,11 +1121,6 @@ scoutfs_net_alloc_conn(struct super_block *sb, struct net_info *ninf = SCOUTFS_SB(sb)->net_info; struct scoutfs_net_connection *conn; - /* we handle greetings, the caller shouldn't attempt to */ - if (WARN_ON_ONCE(req_funcs != NULL && - req_funcs[SCOUTFS_NET_CMD_GREETING] != NULL)) - return NULL; - conn = kzalloc(sizeof(struct scoutfs_net_connection), GFP_NOFS); if (!conn) return NULL; @@ -1237,11 +1138,11 @@ scoutfs_net_alloc_conn(struct super_block *sb, conn->notify_down = notify_down; conn->req_funcs = req_funcs; spin_lock_init(&conn->lock); + init_waitqueue_head(&conn->waitq); conn->sockname.sin_family = AF_INET; conn->peername.sin_family = AF_INET; INIT_LIST_HEAD(&conn->accepted_head); INIT_LIST_HEAD(&conn->accepted_list); - init_waitqueue_head(&conn->accepted_waitq); conn->next_send_id = SCOUTFS_NET_ID_GREETING + 1; INIT_LIST_HEAD(&conn->send_queue); INIT_LIST_HEAD(&conn->resend_queue); @@ -1345,22 +1246,52 @@ void scoutfs_net_listen(struct super_block *sb, } /* - * Start connecting to the given address. notify_up may be called if - * the connection completes. notify_down will be called when either the - * connection disconnects or times out. Both could be called before - * this function returns. The caller must be careful not to call - * connect again until notify_down has been called. + * Return once a connection attempt has completed either successfully + * or in error. */ -void scoutfs_net_connect(struct super_block *sb, - struct scoutfs_net_connection *conn, - struct sockaddr_in *sin, unsigned long timeout_ms) +static bool connect_result(struct scoutfs_net_connection *conn, int *error) { + bool done = false; + + spin_lock(&conn->lock); + if (conn->established) { + done = true; + *error = 0; + } else if (conn->shutting_down || conn->connect_sin.sin_family == 0) { + done = true; + *error = -ESHUTDOWN; + } + spin_unlock(&conn->lock); + + return done; +} + +/* + * Connect to the given address. An error is returned if the socket was + * not connected before the given timeout. The connection isn't fully + * active until the connecting caller starts greeting negotiation by + * sending the initial greeting request. + * + * The conn notify_down callback can be called as the connection is + * shutdown before this returns. + */ +int scoutfs_net_connect(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin, unsigned long timeout_ms) +{ + int error = 0; + int ret; + spin_lock(&conn->lock); conn->connect_sin = *sin; conn->connect_timeout_ms = timeout_ms; spin_unlock(&conn->lock); queue_work(conn->workq, &conn->connect_work); + + ret = wait_event_interruptible(conn->waitq, + connect_result(conn, &error)); + return ret ?: error; } /* @@ -1379,6 +1310,20 @@ int scoutfs_net_submit_request(struct super_block *sb, arg, arg_len, resp_func, resp_data, id_ret); } +/* + * Greeting requests are special because they have a known id. + */ +int scoutfs_net_submit_greeting_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data) +{ + return submit_send(sb, conn, SCOUTFS_NET_MSG_REQUEST, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING, + 0, arg, arg_len, resp_func, resp_data, NULL); +} + /* * Send a response. Responses don't get callbacks and use the request's * id so caller's don't need to get an id in return. diff --git a/kmod/src/net.h b/kmod/src/net.h index 4e270760..f81d4bf7 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -27,9 +27,9 @@ scoutfs_net_alloc_conn(struct super_block *sb, scoutfs_net_notify_t notify_up, scoutfs_net_notify_t notify_down, scoutfs_net_request_t *req_funcs, char *name_suffix); -void scoutfs_net_connect(struct super_block *sb, - struct scoutfs_net_connection *conn, - struct sockaddr_in *sin, unsigned long timeout_ms); +int scoutfs_net_connect(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin, unsigned long timeout_ms); int scoutfs_net_bind(struct super_block *sb, struct scoutfs_net_connection *conn, struct sockaddr_in *sin); @@ -40,6 +40,11 @@ int scoutfs_net_submit_request(struct super_block *sb, u8 cmd, void *arg, u16 arg_len, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret); +int scoutfs_net_submit_greeting_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data); void scoutfs_net_cancel_request(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id); diff --git a/kmod/src/server.c b/kmod/src/server.c index 2ee1be3e..2b9cd0a5 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -47,6 +47,7 @@ struct server_info { struct super_block *sb; + spinlock_t lock; struct workqueue_struct *wq; struct delayed_work dwork; @@ -958,6 +959,74 @@ static int server_statfs(struct super_block *sb, &nstatfs, sizeof(nstatfs)); } +/* + * Process an incoming greeting request in the server from the client. + * We try to send responses to failed greetings so that the sender can + * log some detail before shutting down. A failure to send a greeting + * response shuts down the connection. + * + * We allocate a new node_id for the first connect attempt from a + * client. If they reconnect they'll send their initially assigned node_id + * in their greeting request. + */ +static int server_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_net_greeting *gr = arg; + struct scoutfs_net_greeting greet; + DECLARE_SERVER_INFO(sb, server); + struct commit_waiter cw; + __le64 node_id; + int ret = 0; + + if (arg_len != sizeof(struct scoutfs_net_greeting)) { + ret = -EINVAL; + goto out; + } + + if (gr->fsid != super->id) { + scoutfs_warn(sb, "client sent fsid 0x%llx, server has 0x%llx", + le64_to_cpu(gr->fsid), + le64_to_cpu(super->id)); + ret = -EINVAL; + goto out; + } + + if (gr->format_hash != super->format_hash) { + scoutfs_warn(sb, "client sent format 0x%llx, server has 0x%llx", + le64_to_cpu(gr->format_hash), + le64_to_cpu(super->format_hash)); + ret = -EINVAL; + goto out; + } + + if (gr->node_id == 0) { + down_read(&server->commit_rwsem); + + spin_lock(&server->lock); + node_id = super->next_node_id; + le64_add_cpu(&super->next_node_id, 1); + spin_unlock(&server->lock); + + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + ret = wait_for_commit(&cw); + if (ret) + goto out; + } else { + node_id = gr->node_id; + } + + greet.fsid = super->id; + greet.format_hash = super->format_hash; + greet.node_id = node_id; +out: + return scoutfs_net_response(sb, conn, cmd, id, ret, + &greet, sizeof(greet)); +} + /* * Eventually we're going to have messages that control compaction. * Each client mount would have long-lived work that sends requests @@ -1065,6 +1134,7 @@ static int write_server_addr(struct super_block *sb, struct sockaddr_in *sin) } static scoutfs_net_request_t server_req_funcs[] = { + [SCOUTFS_NET_CMD_GREETING] = server_greeting, [SCOUTFS_NET_CMD_ALLOC_INODES] = server_alloc_inodes, [SCOUTFS_NET_CMD_ALLOC_EXTENT] = server_alloc_extent, [SCOUTFS_NET_CMD_FREE_EXTENTS] = server_free_extents, @@ -1212,6 +1282,7 @@ int scoutfs_server_setup(struct super_block *sb) return -ENOMEM; server->sb = sb; + spin_lock_init(&server->lock); init_completion(&server->shutdown_comp); server->bind_warned = false; INIT_DELAYED_WORK(&server->dwork, scoutfs_server_worker); diff --git a/kmod/src/super.c b/kmod/src/super.c index 28773a3e..15387737 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -304,12 +304,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sbi) return -ENOMEM; - /* - * XXX this is random today for initial testing, but we'll want - * it to be assigned by the server. - */ - get_random_bytes_arch(&sbi->node_id, sizeof(sbi->node_id)); - spin_lock_init(&sbi->next_ino_lock); init_waitqueue_head(&sbi->trans_hold_wq); spin_lock_init(&sbi->trans_write_lock); @@ -338,6 +332,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_net_setup(sb) ?: scoutfs_server_setup(sb) ?: scoutfs_client_setup(sb) ?: + scoutfs_client_wait_node_id(sb) ?: scoutfs_lock_node_id(sb, DLM_LOCK_EX, 0, sbi->node_id, &sbi->node_id_lock); if (ret)