From 0c95388f3bb4be9a03e03f3767658d08f223270f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Sep 2021 13:51:27 -0700 Subject: [PATCH] Set TCP_USER_TIMEOUT in addition to keepalives TCP keepalive probes only work when the connection is idle. They're not sent when there's unacked send data being retramnsmitted. If the server fails while we're retransmitting we don't break the connection and try to elect and connect to a new server until the very long default conneciton timeouts or the server comes back and the stale connection is aborted. We can set TCP_USER_TIMEOUT to break an unresponsive connection when there's written data. It changes the behavior of the keepalive probes so we rework them a bit to clearly apply our timeout consistently between the two mechanisms. Signed-off-by: Zach Brown --- kmod/src/net.c | 46 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index 8368f49b..49ea837e 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -880,13 +880,31 @@ static void destroy_conn(struct scoutfs_net_connection *conn) } /* - * Have a pretty aggressive keepalive timeout of around 10 seconds. The - * TCP keepalives are being processed out of task context so they should - * be responsive even when mounts are under load. + * By default, TCP would maintain a connection to an unresponsive peer + * for a very long time indeed. We can't do that because quorum + * members will only participate in an election when they don't have a + * healthy connection to a server. We use the KEEPALIVE* and + * TCP_USER_TIMEOUT options to ensure that we'll break an unresponsive + * connection and return to the quorum and client connection paths to + * try and establish a new connection to an active server. + * + * The TCP_KEEP* and TCP_USER_TIMEOUT option interaction is subtle. + * TCP_USER_TIMEOUT only applies if there is unacked written data in the + * send queue. It doesn't work if the connection is idle. Adding + * keepalice probes with user_timeout set changes how the keepalive + * timeout is calculated. CNT no longer matters. Each time + * additional probes (not the first) are sent the user timeout is + * checked against the last time data was received. If none of the + * keepalives are responded to then eventually the user timeout applies. + * + * Given all this, we start with the overall unresponsive timeout. Then + * we set the probes to start sending towards the end of the timeout. + * We give it a few tries for a successful response before the timeout + * elapses during the probe timer processing after the unsuccessful + * probes. */ -#define KEEPCNT 3 -#define KEEPIDLE 7 -#define KEEPINTVL 1 +#define UNRESPONSIVE_TIMEOUT_SECS 10 +#define UNRESPONSIVE_PROBES 3 static int sock_opts_and_names(struct scoutfs_net_connection *conn, struct socket *sock) { @@ -895,7 +913,7 @@ static int sock_opts_and_names(struct scoutfs_net_connection *conn, int optval; int ret; - /* but use a keepalive timeout instead of send timeout */ + /* we use a keepalive timeout instead of send timeout */ tv.tv_sec = 0; tv.tv_usec = 0; ret = kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, @@ -903,24 +921,32 @@ static int sock_opts_and_names(struct scoutfs_net_connection *conn, if (ret) goto out; - optval = KEEPCNT; + /* not checked when user_timeout != 0, but for clarity */ + optval = UNRESPONSIVE_PROBES; ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT, (char *)&optval, sizeof(optval)); if (ret) goto out; - optval = KEEPIDLE; + BUILD_BUG_ON(UNRESPONSIVE_PROBES >= UNRESPONSIVE_TIMEOUT_SECS); + optval = UNRESPONSIVE_TIMEOUT_SECS - (UNRESPONSIVE_PROBES); ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE, (char *)&optval, sizeof(optval)); if (ret) goto out; - optval = KEEPINTVL; + optval = 1; ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL, (char *)&optval, sizeof(optval)); if (ret) goto out; + optval = UNRESPONSIVE_TIMEOUT_SECS * MSEC_PER_SEC; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_USER_TIMEOUT, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + optval = 1; ret = kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, (char *)&optval, sizeof(optval));