diff --git a/kmod/src/client.c b/kmod/src/client.c index a21cbefd..81fd7fb7 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -51,6 +51,7 @@ struct client_info { struct super_block *sb; struct scoutfs_net_connection *conn; + struct completion node_id_comp; atomic_t shutting_down; struct workqueue_struct *workq; @@ -230,13 +231,78 @@ int scoutfs_client_statfs(struct super_block *sb, sizeof(struct scoutfs_net_statfs)); } +/* + * Process a greeting response in the client from the server. This is + * called for every connected socket on the connection. The first + * response will have the node_id that the server assigned the client. + */ +static int client_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, int error, + void *data) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_net_greeting *gr = resp; + int ret = 0; + + if (error) { + ret = error; + goto out; + } + + if (resp_len != sizeof(struct scoutfs_net_greeting)) { + ret = -EINVAL; + goto out; + } + + if (gr->fsid != super->id) { + scoutfs_warn(sb, "server sent fsid 0x%llx, client has 0x%llx", + le64_to_cpu(gr->fsid), + le64_to_cpu(super->id)); + ret = -EINVAL; + goto out; + } + + if (gr->format_hash != super->format_hash) { + scoutfs_warn(sb, "server sent format 0x%llx, client has 0x%llx", + le64_to_cpu(gr->format_hash), + le64_to_cpu(super->format_hash)); + ret = -EINVAL; + goto out; + } + + if (sbi->node_id != 0 && le64_to_cpu(gr->node_id) != sbi->node_id) { + scoutfs_warn(sb, "server sent node_id %llu, client has %llu", + le64_to_cpu(gr->node_id), + sbi->node_id); + ret = -EINVAL; + goto out; + } + + if (sbi->node_id == 0 && gr->node_id == 0) { + scoutfs_warn(sb, "server sent node_id 0, client also has 0\n"); + ret = -EINVAL; + goto out; + } + + if (sbi->node_id == 0) { + sbi->node_id = le64_to_cpu(gr->node_id); + complete(&client->node_id_comp); + } + +out: + return ret; +} + /* * Attempt to connect to the listening address that the server wrote in * the super block. We keep trying indefinitely with an increasing * delay if we fail to either read the address or connect to it. * * We're careful to only ever have one connection attempt in flight. We - * only queue this work on mount, on error, or from the connection + * only queue this work on mount, on error, or from the notify_down * callback. */ static void scoutfs_client_connect_worker(struct work_struct *work) @@ -244,6 +310,8 @@ static void scoutfs_client_connect_worker(struct work_struct *work) struct client_info *client = container_of(work, struct client_info, connect_dwork.work); struct super_block *sb = client->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_net_greeting greet; struct scoutfs_super_block super; struct sockaddr_in sin; int ret; @@ -262,8 +330,24 @@ static void scoutfs_client_connect_worker(struct work_struct *work) sin.sin_addr.s_addr = le32_to_be32(super.server_addr.addr); sin.sin_port = le16_to_be16(super.server_addr.port); - scoutfs_net_connect(sb, client->conn, &sin, client->conn_retry_ms); - ret = 0; + ret = scoutfs_net_connect(sb, client->conn, &sin, + client->conn_retry_ms); + if (ret) + goto out; + + reset_connect_timeout(client); + + /* send a greeting to verify endpoints of each connection */ + greet.fsid = super.id; + greet.format_hash = super.format_hash; + greet.node_id = cpu_to_le64(sbi->node_id); + + ret = scoutfs_net_submit_greeting_request(sb, client->conn, + &greet, sizeof(greet), + client_greeting, NULL); + if (ret) + scoutfs_net_shutdown(sb, client->conn); + out: if (ret && !atomic_read(&client->shutting_down)) { queue_delayed_work(client->workq, &client->connect_dwork, @@ -272,14 +356,6 @@ out: } } -static void client_notify_up(struct super_block *sb, - struct scoutfs_net_connection *conn) -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - - reset_connect_timeout(client); -} - /* * Called when either a connect attempt or established connection times * out and fails. @@ -296,6 +372,18 @@ static void client_notify_down(struct super_block *sb, } } +/* + * Wait for the first connected socket on the connection that assigns + * the node_id that will be used for the rest of the life time of the + * mount. + */ +int scoutfs_client_wait_node_id(struct super_block *sb) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return wait_for_completion_interruptible(&client->node_id_comp); +} + int scoutfs_client_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -310,14 +398,14 @@ int scoutfs_client_setup(struct super_block *sb) sbi->client_info = client; client->sb = sb; + init_completion(&client->node_id_comp); atomic_set(&client->shutting_down, 0); INIT_DELAYED_WORK(&client->connect_dwork, scoutfs_client_connect_worker); /* client doesn't process any incoming requests yet */ - client->conn = scoutfs_net_alloc_conn(sb, client_notify_up, - client_notify_down, NULL, - "client"); + client->conn = scoutfs_net_alloc_conn(sb, NULL, client_notify_down, + NULL, "client"); if (!client->conn) { ret = -ENOMEM; goto out; diff --git a/kmod/src/client.h b/kmod/src/client.h index f0a8d609..410592eb 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -18,6 +18,7 @@ int scoutfs_client_get_manifest_root(struct super_block *sb, int scoutfs_client_statfs(struct super_block *sb, struct scoutfs_net_statfs *nstatfs); +int scoutfs_client_wait_node_id(struct super_block *sb); int scoutfs_client_setup(struct super_block *sb); void scoutfs_client_destroy(struct super_block *sb); diff --git a/kmod/src/format.h b/kmod/src/format.h index 5a01f57b..95f732c9 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -368,6 +368,7 @@ struct scoutfs_super_block { __le64 alloc_cursor; struct scoutfs_btree_ring bring; __le64 next_seg_seq; + __le64 next_node_id; struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; struct scoutfs_inet_addr server_addr; @@ -514,6 +515,7 @@ struct scoutfs_lock_name { struct scoutfs_net_greeting { __le64 fsid; __le64 format_hash; + __le64 node_id; } __packed; /* diff --git a/kmod/src/net.c b/kmod/src/net.c index 3ebc062a..775a25d7 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -50,7 +50,6 @@ * deliver messages across renumbering. * * XXX: - * - assign node_ids and validate with the greeting * - defer accepted conn destruction until reconnect timeout * - trace command and response data payloads * - checksum message contents? @@ -77,6 +76,7 @@ struct scoutfs_net_connection { scoutfs_net_request_t *req_funcs; spinlock_t lock; + wait_queue_head_t waitq; unsigned long valid_greeting:1, /* other commands can proceed */ established:1, /* added sends queue send work */ @@ -92,7 +92,6 @@ struct scoutfs_net_connection { struct list_head accepted_head; struct scoutfs_net_connection *listening_conn; struct list_head accepted_list; - wait_queue_head_t accepted_waitq; u64 next_send_id; u64 last_proc_id; @@ -371,124 +370,28 @@ static int submit_send(struct super_block *sb, } /* - * Messages can flow once we receive a valid greeting from our peer. - * Response callers are already called under the lock, request callers - * need to acquire it. + * Messages can flow once we receive and process a valid greeting from + * our peer. * - * At this point greeting request processing has queued the greeting - * response message on the send queue. All the sends waiting to be - * resent need to be added to the end of the send queue after the - * greeting response. Greeting acks are sent differently and can be - * received after resend messages. + * At this point recv processing has queued the greeting response or ack + * message on the send queue. All the sends waiting to be resent need + * to be added to the end of the send queue after the greeting message. */ static void saw_valid_greeting(struct scoutfs_net_connection *conn) { struct super_block *sb = conn->sb; - assert_spin_locked(&conn->lock); + spin_lock(&conn->lock); conn->valid_greeting = 1; if (conn->notify_up) conn->notify_up(sb, conn); list_splice_tail_init(&conn->resend_queue, &conn->send_queue); queue_work(conn->workq, &conn->send_work); + + spin_unlock(&conn->lock); } -static int greeting_response(struct super_block *sb, - struct scoutfs_net_connection *conn, - void *resp, unsigned int resp_len, int error, - void *data) -{ - struct scoutfs_net_greeting *gr = resp; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - int ret = 0; - - if (error) { - ret = error; - goto out; - } - - if (resp_len != sizeof(struct scoutfs_net_greeting)) { - ret = -EINVAL; - goto out; - } - - if (gr->fsid != super->id) { - scoutfs_warn(sb, "server "SIN_FMT" has fsid 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(gr->fsid), - le64_to_cpu(super->id)); - ret = -EINVAL; - goto out; - } - - if (gr->format_hash != super->format_hash) { - scoutfs_warn(sb, "server "SIN_FMT" has format hash 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(gr->format_hash), - le64_to_cpu(super->format_hash)); - ret = -EINVAL; - goto out; - } - - saw_valid_greeting(conn); - -out: - return ret; -} - -/* - * Process an incoming greeting request. We try to send responses to - * failed greetings so that the sender can log some detail before - * shutting down. A failure to send a greeting response shuts down the - * connection. - */ -static int greeting_request(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len) -{ - struct scoutfs_net_greeting *gr = arg; - struct scoutfs_net_greeting greet; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - int ret = 0; - - if (arg_len != sizeof(struct scoutfs_net_greeting)) { - ret = -EINVAL; - goto out; - } - - if (gr->fsid != super->id) { - scoutfs_warn(sb, "client "SIN_FMT" has fsid 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(gr->fsid), - le64_to_cpu(super->id)); - ret = -EINVAL; - goto out; - } - - if (gr->format_hash != super->format_hash) { - scoutfs_warn(sb, "client "SIN_FMT" has format hash 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(gr->format_hash), - le64_to_cpu(super->format_hash)); - ret = -EINVAL; - goto out; - } - - greet.fsid = super->id; - greet.format_hash = super->format_hash; -out: - ret = scoutfs_net_response(sb, conn, cmd, id, ret, - &greet, sizeof(greet)); - if (ret == 0) { - spin_lock(&conn->lock); - saw_valid_greeting(conn); - spin_unlock(&conn->lock); - } - return ret; -} - - /* * Process an incoming response. The greeting should ensure that the * sender won't send us unknown commands. We return an error if we see @@ -500,20 +403,27 @@ static int process_request(struct scoutfs_net_connection *conn, struct message_recv *mrecv) { struct super_block *sb = conn->sb; - scoutfs_net_request_t req_func = NULL; + scoutfs_net_request_t req_func; + int ret; - if (conn->listening_conn != NULL && - mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING) { - req_func = greeting_request; - } else if (mrecv->nh.cmd < SCOUTFS_NET_CMD_UNKNOWN) { + if (mrecv->nh.cmd < SCOUTFS_NET_CMD_UNKNOWN) req_func = conn->req_funcs[mrecv->nh.cmd]; - } if (req_func == NULL) { + else + req_func = NULL; + + if (req_func == NULL) { scoutfs_inc_counter(sb, net_unknown_request); return -EINVAL; } - return req_func(sb, conn, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), - mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); + ret = req_func(sb, conn, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), + mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); + + if (!conn->valid_greeting && + mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING && ret == 0) + saw_valid_greeting(conn); + + return ret; } /* @@ -547,6 +457,11 @@ static int process_response(struct scoutfs_net_connection *conn, ret = submit_send(sb, conn, SCOUTFS_NET_MSG_ACK, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), 0, NULL, 0, NULL, NULL, NULL); + + if (!conn->valid_greeting && + mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING && msend && ret == 0) + saw_valid_greeting(conn); + return ret; } @@ -883,7 +798,7 @@ static void destroy_conn(struct scoutfs_net_connection *conn) spin_lock(&listener->lock); list_del_init(&conn->accepted_head); if (list_empty(&listener->accepted_list)) - wake_up(&listener->accepted_waitq); + wake_up(&listener->waitq); spin_unlock(&listener->lock); } @@ -1038,8 +953,7 @@ static void scoutfs_net_connect_worker(struct work_struct *work) { DEFINE_CONN_FROM_WORK(conn, work, connect_work); struct super_block *sb = conn->sb; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_net_greeting greet; + struct message_send *msend; struct socket *sock; struct timeval tv; int ret; @@ -1074,26 +988,29 @@ static void scoutfs_net_connect_worker(struct work_struct *work) if (ret) goto out; - /* greeting is about to queue send work */ - spin_lock(&conn->lock); - conn->established = 1; - spin_unlock(&conn->lock); - - queue_work(conn->workq, &conn->recv_work); - - /* queue a new updated greeting send */ - greet.fsid = super->id; - greet.format_hash = super->format_hash; - - ret = submit_send(sb, conn, SCOUTFS_NET_MSG_REQUEST, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING, 0, - &greet, sizeof(greet), greeting_response, NULL, NULL); - if (ret) - goto out; - scoutfs_info(sb, "client connected "SIN_FMT" -> "SIN_FMT, SIN_ARG(&conn->sockname), SIN_ARG(&conn->peername)); + + spin_lock(&conn->lock); + + /* clear greeting state for next negotiation */ + conn->valid_greeting = 0; + msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: + find_send(conn, SCOUTFS_NET_MSG_RESPONSE, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: + find_send(conn, SCOUTFS_NET_MSG_ACK, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING); + if (msend) + complete_send(conn, msend, NULL, 0, 0); + + conn->established = 1; + wake_up(&conn->waitq); + + spin_unlock(&conn->lock); + + queue_work(conn->workq, &conn->recv_work); out: if (ret) shutdown_conn(conn); @@ -1135,7 +1052,6 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) DEFINE_CONN_FROM_WORK(conn, work, shutdown_work); struct super_block *sb = conn->sb; struct scoutfs_net_connection *acc_conn; - struct message_send *msend; trace_scoutfs_net_shutdown_work_enter(sb, 0, 0); @@ -1160,6 +1076,8 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) conn->sock = NULL; } + memset(&conn->peername, 0, sizeof(conn->peername)); + /* listening connections shut down all the connections they accepted */ spin_lock_nested(&conn->lock, CONN_LOCK_LISTENER); list_for_each_entry(acc_conn, &conn->accepted_list, accepted_head) { @@ -1168,28 +1086,16 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) spin_unlock(&acc_conn->lock); } spin_unlock(&conn->lock); - wait_event(conn->accepted_waitq, empty_accepted_list(conn)); + wait_event(conn->waitq, empty_accepted_list(conn)); spin_lock(&conn->lock); - /* all queued sends will be resent, protocol handles dupes */ list_splice_tail_init(&conn->send_queue, &conn->resend_queue); - - /* clear greeting state for next negotiation */ - conn->valid_greeting = 0; - msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: - find_send(conn, SCOUTFS_NET_MSG_RESPONSE, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: - find_send(conn, SCOUTFS_NET_MSG_ACK, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING); - if (msend) - complete_send(conn, msend, NULL, 0, 0); - + /* signal connect failure */ + memset(&conn->connect_sin, 0, sizeof(conn->connect_sin)); + wake_up(&conn->waitq); spin_unlock(&conn->lock); - memset(&conn->peername, 0, sizeof(conn->peername)); - /* tell the caller that the connection is down */ if (conn->notify_down) conn->notify_down(sb, conn); @@ -1215,11 +1121,6 @@ scoutfs_net_alloc_conn(struct super_block *sb, struct net_info *ninf = SCOUTFS_SB(sb)->net_info; struct scoutfs_net_connection *conn; - /* we handle greetings, the caller shouldn't attempt to */ - if (WARN_ON_ONCE(req_funcs != NULL && - req_funcs[SCOUTFS_NET_CMD_GREETING] != NULL)) - return NULL; - conn = kzalloc(sizeof(struct scoutfs_net_connection), GFP_NOFS); if (!conn) return NULL; @@ -1237,11 +1138,11 @@ scoutfs_net_alloc_conn(struct super_block *sb, conn->notify_down = notify_down; conn->req_funcs = req_funcs; spin_lock_init(&conn->lock); + init_waitqueue_head(&conn->waitq); conn->sockname.sin_family = AF_INET; conn->peername.sin_family = AF_INET; INIT_LIST_HEAD(&conn->accepted_head); INIT_LIST_HEAD(&conn->accepted_list); - init_waitqueue_head(&conn->accepted_waitq); conn->next_send_id = SCOUTFS_NET_ID_GREETING + 1; INIT_LIST_HEAD(&conn->send_queue); INIT_LIST_HEAD(&conn->resend_queue); @@ -1345,22 +1246,52 @@ void scoutfs_net_listen(struct super_block *sb, } /* - * Start connecting to the given address. notify_up may be called if - * the connection completes. notify_down will be called when either the - * connection disconnects or times out. Both could be called before - * this function returns. The caller must be careful not to call - * connect again until notify_down has been called. + * Return once a connection attempt has completed either successfully + * or in error. */ -void scoutfs_net_connect(struct super_block *sb, - struct scoutfs_net_connection *conn, - struct sockaddr_in *sin, unsigned long timeout_ms) +static bool connect_result(struct scoutfs_net_connection *conn, int *error) { + bool done = false; + + spin_lock(&conn->lock); + if (conn->established) { + done = true; + *error = 0; + } else if (conn->shutting_down || conn->connect_sin.sin_family == 0) { + done = true; + *error = -ESHUTDOWN; + } + spin_unlock(&conn->lock); + + return done; +} + +/* + * Connect to the given address. An error is returned if the socket was + * not connected before the given timeout. The connection isn't fully + * active until the connecting caller starts greeting negotiation by + * sending the initial greeting request. + * + * The conn notify_down callback can be called as the connection is + * shutdown before this returns. + */ +int scoutfs_net_connect(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin, unsigned long timeout_ms) +{ + int error = 0; + int ret; + spin_lock(&conn->lock); conn->connect_sin = *sin; conn->connect_timeout_ms = timeout_ms; spin_unlock(&conn->lock); queue_work(conn->workq, &conn->connect_work); + + ret = wait_event_interruptible(conn->waitq, + connect_result(conn, &error)); + return ret ?: error; } /* @@ -1379,6 +1310,20 @@ int scoutfs_net_submit_request(struct super_block *sb, arg, arg_len, resp_func, resp_data, id_ret); } +/* + * Greeting requests are special because they have a known id. + */ +int scoutfs_net_submit_greeting_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data) +{ + return submit_send(sb, conn, SCOUTFS_NET_MSG_REQUEST, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING, + 0, arg, arg_len, resp_func, resp_data, NULL); +} + /* * Send a response. Responses don't get callbacks and use the request's * id so caller's don't need to get an id in return. diff --git a/kmod/src/net.h b/kmod/src/net.h index 4e270760..f81d4bf7 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -27,9 +27,9 @@ scoutfs_net_alloc_conn(struct super_block *sb, scoutfs_net_notify_t notify_up, scoutfs_net_notify_t notify_down, scoutfs_net_request_t *req_funcs, char *name_suffix); -void scoutfs_net_connect(struct super_block *sb, - struct scoutfs_net_connection *conn, - struct sockaddr_in *sin, unsigned long timeout_ms); +int scoutfs_net_connect(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin, unsigned long timeout_ms); int scoutfs_net_bind(struct super_block *sb, struct scoutfs_net_connection *conn, struct sockaddr_in *sin); @@ -40,6 +40,11 @@ int scoutfs_net_submit_request(struct super_block *sb, u8 cmd, void *arg, u16 arg_len, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret); +int scoutfs_net_submit_greeting_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data); void scoutfs_net_cancel_request(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id); diff --git a/kmod/src/server.c b/kmod/src/server.c index 2ee1be3e..2b9cd0a5 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -47,6 +47,7 @@ struct server_info { struct super_block *sb; + spinlock_t lock; struct workqueue_struct *wq; struct delayed_work dwork; @@ -958,6 +959,74 @@ static int server_statfs(struct super_block *sb, &nstatfs, sizeof(nstatfs)); } +/* + * Process an incoming greeting request in the server from the client. + * We try to send responses to failed greetings so that the sender can + * log some detail before shutting down. A failure to send a greeting + * response shuts down the connection. + * + * We allocate a new node_id for the first connect attempt from a + * client. If they reconnect they'll send their initially assigned node_id + * in their greeting request. + */ +static int server_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_net_greeting *gr = arg; + struct scoutfs_net_greeting greet; + DECLARE_SERVER_INFO(sb, server); + struct commit_waiter cw; + __le64 node_id; + int ret = 0; + + if (arg_len != sizeof(struct scoutfs_net_greeting)) { + ret = -EINVAL; + goto out; + } + + if (gr->fsid != super->id) { + scoutfs_warn(sb, "client sent fsid 0x%llx, server has 0x%llx", + le64_to_cpu(gr->fsid), + le64_to_cpu(super->id)); + ret = -EINVAL; + goto out; + } + + if (gr->format_hash != super->format_hash) { + scoutfs_warn(sb, "client sent format 0x%llx, server has 0x%llx", + le64_to_cpu(gr->format_hash), + le64_to_cpu(super->format_hash)); + ret = -EINVAL; + goto out; + } + + if (gr->node_id == 0) { + down_read(&server->commit_rwsem); + + spin_lock(&server->lock); + node_id = super->next_node_id; + le64_add_cpu(&super->next_node_id, 1); + spin_unlock(&server->lock); + + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + ret = wait_for_commit(&cw); + if (ret) + goto out; + } else { + node_id = gr->node_id; + } + + greet.fsid = super->id; + greet.format_hash = super->format_hash; + greet.node_id = node_id; +out: + return scoutfs_net_response(sb, conn, cmd, id, ret, + &greet, sizeof(greet)); +} + /* * Eventually we're going to have messages that control compaction. * Each client mount would have long-lived work that sends requests @@ -1065,6 +1134,7 @@ static int write_server_addr(struct super_block *sb, struct sockaddr_in *sin) } static scoutfs_net_request_t server_req_funcs[] = { + [SCOUTFS_NET_CMD_GREETING] = server_greeting, [SCOUTFS_NET_CMD_ALLOC_INODES] = server_alloc_inodes, [SCOUTFS_NET_CMD_ALLOC_EXTENT] = server_alloc_extent, [SCOUTFS_NET_CMD_FREE_EXTENTS] = server_free_extents, @@ -1212,6 +1282,7 @@ int scoutfs_server_setup(struct super_block *sb) return -ENOMEM; server->sb = sb; + spin_lock_init(&server->lock); init_completion(&server->shutdown_comp); server->bind_warned = false; INIT_DELAYED_WORK(&server->dwork, scoutfs_server_worker); diff --git a/kmod/src/super.c b/kmod/src/super.c index 28773a3e..15387737 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -304,12 +304,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sbi) return -ENOMEM; - /* - * XXX this is random today for initial testing, but we'll want - * it to be assigned by the server. - */ - get_random_bytes_arch(&sbi->node_id, sizeof(sbi->node_id)); - spin_lock_init(&sbi->next_ino_lock); init_waitqueue_head(&sbi->trans_hold_wq); spin_lock_init(&sbi->trans_write_lock); @@ -338,6 +332,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_net_setup(sb) ?: scoutfs_server_setup(sb) ?: scoutfs_client_setup(sb) ?: + scoutfs_client_wait_node_id(sb) ?: scoutfs_lock_node_id(sb, DLM_LOCK_EX, 0, sbi->node_id, &sbi->node_id_lock); if (ret)