From e18ea24561242968c540891aaf586ed98a6c64ad Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 12 May 2023 10:55:51 -0700 Subject: [PATCH] Move quorum recv that sets timeout before check In the quorum work loop some message receive actions extend the timeout after the timeout expiration is checked. This is usually fine when the work runs soon after the messages are received and before the timeout expires. But under load the work might not schedule until long after both the message has been received and the timeout has expired. If the message was a heartbeat message then the wakeup delay would be mistaken for lack of activity on the server and it would try to take over for an otherwise active server. This moves the extension of the heartbeat on message receive to before the timeout is checked. In our case of a delayed heartbeat message it would still find it in the recv queue and extend the timeout, avoiding fencing an active server. Signed-off-by: Zach Brown --- kmod/src/quorum.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index 98f1b264..d45c4f81 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -699,6 +699,20 @@ static void scoutfs_quorum_worker(struct work_struct *work) goto out; } + /* receiving heartbeats extends timeout, delaying elections */ + if (msg.type == SCOUTFS_QUORUM_MSG_HEARTBEAT) { + qst.timeout = heartbeat_timeout(); + scoutfs_inc_counter(sb, quorum_recv_heartbeat); + } + + /* receiving a resignation from server starts election */ + if (msg.type == SCOUTFS_QUORUM_MSG_RESIGNATION && + qst.role == FOLLOWER && + msg.term == qst.term) { + qst.timeout = election_timeout(); + scoutfs_inc_counter(sb, quorum_recv_resignation); + } + /* followers and candidates start new election on timeout */ if (qst.role != LEADER && ktime_after(ktime_get(), qst.timeout)) { @@ -824,20 +838,6 @@ static void scoutfs_quorum_worker(struct work_struct *work) scoutfs_inc_counter(sb, quorum_send_heartbeat); } - /* receiving heartbeats extends timeout, delaying elections */ - if (msg.type == SCOUTFS_QUORUM_MSG_HEARTBEAT) { - qst.timeout = heartbeat_timeout(); - scoutfs_inc_counter(sb, quorum_recv_heartbeat); - } - - /* receiving a resignation from server starts election */ - if (msg.type == SCOUTFS_QUORUM_MSG_RESIGNATION && - qst.role == FOLLOWER && - msg.term == qst.term) { - qst.timeout = election_timeout(); - scoutfs_inc_counter(sb, quorum_recv_resignation); - } - /* followers vote once per term */ if (qst.role == FOLLOWER && msg.type == SCOUTFS_QUORUM_MSG_REQUEST_VOTE &&