From 38e5aa77c43180144d6a9f2ae88fc10e27089088 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 15 Mar 2022 15:07:57 -0700 Subject: [PATCH] Update quorum status files more frequently We were seeing rare test failures where it looked like is_leader wasn't set for any of the mounts. The test that couldn't find a set is_leader file had just perfomed some mounts so we know that a server was up and processing requests. The quorum task wasn't updating the status that's shown in sysfs and debugfs until after the server started up. This opened the race where the server was able to serve mount requests and have the test run to find no is_leader file set before the quorum task was able to update the stats and make its election visible. This updates the quorum task to make its status visible more often, typically before it does something that will take a while. The is_leader will now be visible before the server is started so the test will always see the file after server starts up and lets mounts finish. Signed-off-by: Zach Brown --- kmod/src/quorum.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index 91f5b492..bf87a98e 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -609,6 +609,21 @@ out: return ret; } +/* + * The main quorum task maintains its private status. It seemed cleaner + * to occasionally copy the status for showing in sysfs/debugfs files + * than to have the two lock access to shared status. The show copy is + * updated after being modified before the quorum task sleeps for a + * significant amount of time, either waiting on timeouts or interacting + * with the server. + */ +static void update_show_status(struct quorum_info *qinf, struct quorum_status *qst) +{ + spin_lock(&qinf->show_lock); + qinf->show_status = *qst; + spin_unlock(&qinf->show_lock); +} + /* * The quorum work always runs in the background of quorum member * mounts. It's responsible for starting and stopping the server if @@ -651,6 +666,8 @@ static void scoutfs_quorum_worker(struct work_struct *work) while (!(qinf->shutdown || scoutfs_forcing_unmount(sb))) { + update_show_status(qinf, &qst); + ret = recv_msg(sb, &msg, qst.timeout); if (ret < 0) { if (ret != -ETIMEDOUT && ret != -EAGAIN) { @@ -681,10 +698,6 @@ static void scoutfs_quorum_worker(struct work_struct *work) scoutfs_inc_counter(sb, quorum_send_resignation); } - spin_lock(&qinf->show_lock); - qinf->show_status = qst; - spin_unlock(&qinf->show_lock); - trace_scoutfs_quorum_loop(sb, qst.role, qst.term, qst.vote_for, qst.vote_bits, ktime_to_timespec64(qst.timeout)); @@ -695,6 +708,7 @@ static void scoutfs_quorum_worker(struct work_struct *work) if (qst.role == LEADER) { scoutfs_warn(sb, "saw msg type %u from %u for term %llu while leader in term %llu, shutting down server.", msg.type, msg.from, msg.term, qst.term); + update_show_status(qinf, &qst); scoutfs_server_stop(sb); } qst.role = FOLLOWER; @@ -758,6 +772,8 @@ static void scoutfs_quorum_worker(struct work_struct *work) qst.term); qst.timeout = heartbeat_interval(); + update_show_status(qinf, &qst); + /* record that we've been elected before starting up server */ ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_ELECT, qst.term, true); if (ret < 0) @@ -817,6 +833,8 @@ static void scoutfs_quorum_worker(struct work_struct *work) } } + update_show_status(qinf, &qst); + /* always try to stop a running server as we stop */ if (test_bit(QINF_FLAG_SERVER, &qinf->flags)) { scoutfs_server_stop(sb);