From e9d04dcf8dccad87ff9a1f4326895f7f5b129030 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 25 Feb 2021 14:01:23 -0800 Subject: [PATCH 01/22] Add forced unmount support Add super_ops->umount_begin so that we can implement a forced unmount which tries to avoid issuing any more network or storage ops. It can return errors and lose unsynchronized data. Signed-off-by: Zach Brown --- kmod/src/block.c | 16 ++++++++++++---- kmod/src/btree.c | 4 ++++ kmod/src/client.c | 6 ++++-- kmod/src/net.c | 29 ++++++++++++++++++++++------- kmod/src/server.c | 5 +++++ kmod/src/super.c | 16 ++++++++++++++++ kmod/src/super.h | 9 +++++++++ kmod/src/trans.c | 10 ++++++++-- 8 files changed, 80 insertions(+), 15 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 53578725..58c12d5b 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -200,7 +200,9 @@ static void block_free(struct super_block *sb, struct block_private *bp) else BUG(); - WARN_ON_ONCE(!list_empty(&bp->dirty_entry)); + /* ok to tear down dirty blocks when forcing unmount */ + WARN_ON_ONCE(!scoutfs_forcing_unmount(sb) && !list_empty(&bp->dirty_entry)); + WARN_ON_ONCE(atomic_read(&bp->refcount)); WARN_ON_ONCE(atomic_read(&bp->io_count)); kfree(bp); @@ -485,6 +487,9 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp, sector_t sector; int ret = 0; + if (scoutfs_forcing_unmount(sb)) + return -EIO; + sector = bp->bl.blkno << (SCOUTFS_BLOCK_LG_SHIFT - 9); WARN_ON_ONCE(bp->bl.blkno == U64_MAX); @@ -1148,7 +1153,7 @@ static void sm_block_bio_end_io(struct bio *bio, int err) * only layer that sees the full block buffer so we pass the calculated * crc to the caller for them to check in their context. */ -static int sm_block_io(struct block_device *bdev, int rw, u64 blkno, +static int sm_block_io(struct super_block *sb, struct block_device *bdev, int rw, u64 blkno, struct scoutfs_block_header *hdr, size_t len, __le32 *blk_crc) { @@ -1160,6 +1165,9 @@ static int sm_block_io(struct block_device *bdev, int rw, u64 blkno, BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_BLOCK_SM_SIZE); + if (scoutfs_forcing_unmount(sb)) + return -EIO; + if (WARN_ON_ONCE(len > SCOUTFS_BLOCK_SM_SIZE) || WARN_ON_ONCE(!(rw & WRITE) && !blk_crc)) return -EINVAL; @@ -1212,14 +1220,14 @@ int scoutfs_block_read_sm(struct super_block *sb, struct scoutfs_block_header *hdr, size_t len, __le32 *blk_crc) { - return sm_block_io(bdev, READ, blkno, hdr, len, blk_crc); + return sm_block_io(sb, bdev, READ, blkno, hdr, len, blk_crc); } int scoutfs_block_write_sm(struct super_block *sb, struct block_device *bdev, u64 blkno, struct scoutfs_block_header *hdr, size_t len) { - return sm_block_io(bdev, WRITE, blkno, hdr, len, NULL); + return sm_block_io(sb, bdev, WRITE, blkno, hdr, len, NULL); } int scoutfs_block_setup(struct super_block *sb) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 97778352..b9b02696 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1101,6 +1101,10 @@ static int btree_walk(struct super_block *sb, if (WARN_ON_ONCE((flags & BTW_DIRTY) && (!alloc || !wri))) return -EINVAL; + /* all ops come through walk and walk calls all reads */ + if (scoutfs_forcing_unmount(sb)) + return -EIO; + scoutfs_inc_counter(sb, btree_walk); restart: diff --git a/kmod/src/client.c b/kmod/src/client.c index fe50b45f..f36b72ab 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -466,9 +466,11 @@ static void scoutfs_client_connect_worker(struct work_struct *work) out: /* always have a small delay before retrying to avoid storms */ - if (ret && !atomic_read(&client->shutting_down)) + if (ret && !atomic_read(&client->shutting_down) && + !scoutfs_forcing_unmount(sb)) { queue_delayed_work(client->workq, &client->connect_dwork, msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS)); + } } static scoutfs_net_request_t client_req_funcs[] = { @@ -580,7 +582,7 @@ void scoutfs_client_destroy(struct super_block *sb) if (client == NULL) return; - if (client->server_term != 0) { + if (client->server_term != 0 && !scoutfs_forcing_unmount(sb)) { client->sending_farewell = true; ret = scoutfs_net_submit_request(sb, client->conn, SCOUTFS_NET_CMD_FAREWELL, diff --git a/kmod/src/net.c b/kmod/src/net.c index 9d1fedef..2837307c 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -330,6 +330,9 @@ static int submit_send(struct super_block *sb, WARN_ON_ONCE(id == 0 && (flags & SCOUTFS_NET_FLAG_RESPONSE))) return -EINVAL; + if (scoutfs_forcing_unmount(sb)) + return -EIO; + msend = kmalloc(offsetof(struct message_send, nh.data[data_len]), GFP_NOFS); if (!msend) @@ -420,6 +423,16 @@ static int process_request(struct scoutfs_net_connection *conn, mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); } +static int call_resp_func(struct super_block *sb, struct scoutfs_net_connection *conn, + scoutfs_net_response_t resp_func, void *resp_data, + void *resp, unsigned int resp_len, int error) +{ + if (resp_func) + return resp_func(sb, conn, resp, resp_len, error, resp_data); + else + return 0; +} + /* * An incoming response finds the queued request and calls its response * function. The response function for a given request will only be @@ -434,7 +447,6 @@ static int process_response(struct scoutfs_net_connection *conn, struct message_send *msend; scoutfs_net_response_t resp_func = NULL; void *resp_data; - int ret = 0; spin_lock(&conn->lock); @@ -449,11 +461,8 @@ static int process_response(struct scoutfs_net_connection *conn, spin_unlock(&conn->lock); - if (resp_func) - ret = resp_func(sb, conn, mrecv->nh.data, - le16_to_cpu(mrecv->nh.data_len), - net_err_to_host(mrecv->nh.error), resp_data); - return ret; + return call_resp_func(sb, conn, resp_func, resp_data, mrecv->nh.data, + le16_to_cpu(mrecv->nh.data_len), net_err_to_host(mrecv->nh.error)); } /* @@ -823,9 +832,15 @@ static void scoutfs_net_destroy_worker(struct work_struct *work) if (conn->listening_conn && conn->notify_down) conn->notify_down(sb, conn, conn->info, conn->rid); - /* free all messages, refactor and complete for forced unmount? */ + /* + * Usually networking is idle and we destroy pending sends, but when forcing unmount + * we can have to wake up waiters by failing pending sends. + */ list_splice_init(&conn->resend_queue, &conn->send_queue); list_for_each_entry_safe(msend, tmp, &conn->send_queue, head) { + if (scoutfs_forcing_unmount(sb)) + call_resp_func(sb, conn, msend->resp_func, msend->resp_data, + NULL, 0, -ECONNABORTED); free_msend(ninf, msend); } diff --git a/kmod/src/server.c b/kmod/src/server.c index 884ec50d..fb353b25 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -284,6 +284,11 @@ static void scoutfs_server_commit_func(struct work_struct *work) down_write(&server->commit_rwsem); + if (scoutfs_forcing_unmount(sb)) { + ret = -EIO; + goto out; + } + /* make sure next avail has sufficient blocks */ ret = scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri, server->other_avail, diff --git a/kmod/src/super.c b/kmod/src/super.c index d5d1063f..dbeb3015 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -283,6 +283,21 @@ static void scoutfs_put_super(struct super_block *sb) sb->s_fs_info = NULL; } +/* + * Record that we're performing a forced unmount. As put_super drives + * destruction of the filesystem we won't issue more network or storage + * operations because we assume that they'll hang. Pending operations + * can return errors when it's possible to do so. We may be racing with + * pending operations which can't be canceled. + */ +static void scoutfs_umount_begin(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + scoutfs_warn(sb, "forcing unmount, can return errors and lose unsynced data"); + sbi->forced_unmount = true; +} + static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, .drop_inode = scoutfs_drop_inode, @@ -292,6 +307,7 @@ static const struct super_operations scoutfs_super_ops = { .statfs = scoutfs_statfs, .show_options = scoutfs_show_options, .put_super = scoutfs_put_super, + .umount_begin = scoutfs_umount_begin, }; /* diff --git a/kmod/src/super.h b/kmod/src/super.h index 820ee6b8..cbbbc3a0 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -87,6 +87,8 @@ struct scoutfs_sb_info { struct dentry *debug_root; + bool forced_unmount; + unsigned long corruption_messages_once[SC_NR_LONGS]; }; @@ -107,6 +109,13 @@ static inline bool SCOUTFS_IS_META_BDEV(struct scoutfs_super_block *super_block) #define SCOUTFS_META_BDEV_MODE (FMODE_READ | FMODE_WRITE | FMODE_EXCL) +static inline bool scoutfs_forcing_unmount(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + return sbi->forced_unmount; +} + /* * A small string embedded in messages that's used to identify a * specific mount. It's the three most significant bytes of the fsid diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 186239d4..07eea0fa 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -185,6 +185,11 @@ void scoutfs_trans_write_func(struct work_struct *work) wait_event(sbi->trans_hold_wq, drained_holders(tri)); + if (scoutfs_forcing_unmount(sb)) { + ret = -EIO; + goto out; + } + trace_scoutfs_trans_write_func(sb, scoutfs_block_writer_dirty_bytes(sb, &tri->wri)); @@ -202,7 +207,7 @@ void scoutfs_trans_write_func(struct work_struct *work) if (ret < 0) s = "clean advance seq"; } - goto out; + goto err; } if (sbi->trans_deadline_expired) @@ -222,11 +227,12 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_item_write_done(sb) ?: (s = "advance seq", scoutfs_client_advance_seq(sb, &trans_seq)) ?: (s = "get log trees", scoutfs_trans_get_log_trees(sb)); -out: +err: if (ret < 0) scoutfs_err(sb, "critical transaction commit failure: %s, %d", s, ret); +out: spin_lock(&sbi->trans_write_lock); sbi->trans_write_count++; sbi->trans_write_ret = ret; From ccb7c0bf4b34ff85d1983e6551f5b30feed7dcab Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 23 Aug 2019 14:43:38 -0700 Subject: [PATCH 02/22] Add rw sysfs attr wrapper Add a wrapper around __ATTR_RW so that callers can add attributes with a _store function. Signed-off-by: Zach Brown --- kmod/src/sysfs.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/sysfs.h b/kmod/src/sysfs.h index 73788c00..861258df 100644 --- a/kmod/src/sysfs.h +++ b/kmod/src/sysfs.h @@ -10,6 +10,8 @@ #define SCOUTFS_ATTR_RO(_name) \ static struct kobj_attribute scoutfs_attr_##_name = __ATTR_RO(_name) +#define SCOUTFS_ATTR_RW(_name) \ + static struct kobj_attribute scoutfs_attr_##_name = __ATTR_RW(_name) #define SCOUTFS_ATTR_PTR(_name) \ &scoutfs_attr_##_name.attr From 2dde7297910a2e647fc7028645f824c87e502dc9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 23 Aug 2019 14:44:23 -0700 Subject: [PATCH 03/22] Add sysfs create attr w/ parent Add sysfs attribute creation that can provide the parent dir kobject instead of always creating the sysfs object dir off of the main per-mount dir. Signed-off-by: Zach Brown --- kmod/src/sysfs.c | 11 ++++++----- kmod/src/sysfs.h | 11 ++++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/kmod/src/sysfs.c b/kmod/src/sysfs.c index bea667d0..5f2f024c 100644 --- a/kmod/src/sysfs.c +++ b/kmod/src/sysfs.c @@ -131,9 +131,10 @@ void scoutfs_sysfs_init_attrs(struct super_block *sb, * If this returns success then the file will be visible and show can * be called until unmount. */ -int scoutfs_sysfs_create_attrs(struct super_block *sb, - struct scoutfs_sysfs_attrs *ssa, - struct attribute **attrs, char *fmt, ...) +int scoutfs_sysfs_create_attrs_parent(struct super_block *sb, + struct kobject *parent, + struct scoutfs_sysfs_attrs *ssa, + struct attribute **attrs, char *fmt, ...) { va_list args; size_t name_len; @@ -174,8 +175,8 @@ int scoutfs_sysfs_create_attrs(struct super_block *sb, goto out; } - ret = kobject_init_and_add(&ssa->kobj, &ssa->ktype, - scoutfs_sysfs_sb_dir(sb), "%s", ssa->name); + ret = kobject_init_and_add(&ssa->kobj, &ssa->ktype, parent, + "%s", ssa->name); out: if (ret) { kfree(ssa->name); diff --git a/kmod/src/sysfs.h b/kmod/src/sysfs.h index 861258df..2d218bba 100644 --- a/kmod/src/sysfs.h +++ b/kmod/src/sysfs.h @@ -36,9 +36,14 @@ struct scoutfs_sysfs_attrs { void scoutfs_sysfs_init_attrs(struct super_block *sb, struct scoutfs_sysfs_attrs *ssa); -int scoutfs_sysfs_create_attrs(struct super_block *sb, - struct scoutfs_sysfs_attrs *ssa, - struct attribute **attrs, char *fmt, ...); +int scoutfs_sysfs_create_attrs_parent(struct super_block *sb, + struct kobject *parent, + struct scoutfs_sysfs_attrs *ssa, + struct attribute **attrs, char *fmt, ...); +#define scoutfs_sysfs_create_attrs(sb, ssa, attrs, fmt, args...) \ + scoutfs_sysfs_create_attrs_parent(sb, scoutfs_sysfs_sb_dir(sb), \ + ssa, attrs, fmt, ##args) + void scoutfs_sysfs_destroy_attrs(struct super_block *sb, struct scoutfs_sysfs_attrs *ssa); From b060eb4f5d8d530885ff6a14f420dbcb43e0ebec Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 25 Feb 2021 14:31:53 -0800 Subject: [PATCH 04/22] Add fencing subsystem Add the subsystem which tracks pending fence requests and exposes them to userspace for processing. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/fence.c | 460 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/fence.h | 16 ++ kmod/src/super.c | 3 + kmod/src/super.h | 2 + 5 files changed, 482 insertions(+) create mode 100644 kmod/src/fence.c create mode 100644 kmod/src/fence.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index c211d6e4..975ec319 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -18,6 +18,7 @@ scoutfs-y += \ dir.o \ export.o \ ext.o \ + fence.o \ file.o \ forest.o \ inode.o \ diff --git a/kmod/src/fence.c b/kmod/src/fence.c new file mode 100644 index 00000000..cf5da8bd --- /dev/null +++ b/kmod/src/fence.c @@ -0,0 +1,460 @@ +/* + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "msg.h" +#include "sysfs.h" +#include "server.h" +#include "fence.h" + +/* + * Fencing ensures that a given mount can no longer write to the + * metadata or data devices. It's necessary to ensure that it's safe to + * give another mount access to a resource that is currently owned by a + * mount that has stopped responding. + * + * Fencing is performed in collaboration between the currently elected + * quorum leader mount and userspace running on its host. The kernel + * creates fencing requests as it notices that mounts have stopped + * participating. The fence requests are published as directories in + * sysfs. Userspace agents watch for directories, take action, and + * write to files in the directory to indicate that the mount has been + * fenced. Once the mount is fenced the server can reclaim the + * resources previously held by the fenced mount. + * + * The fence requests contain metadata identifying the specific instance + * of the mount that needs to be fenced. This lets a fencing agent + * ensure that a specific mount has been fenced without necessarily + * destroying the node that was hosting it. Maybe the node had rebooted + * and the mount is no longer there, maybe the mount can be force + * unmounted, maybe the node can be configured to isolate the mount from + * the devices. + * + * The fencing mechanism is asynchronous and can fail but the server + * cannot make progress until it completes. If a fence request times + * out the server shuts down in the hope that another instance of a + * server might have more luck fencing a non-responsive mount. + * + * Sources of fencing are fundamentally anchored in shared persistent + * state. It is possible, though unlikely, that servers can fence a + * node and then themselves fail, leaving the next server to try and + * fence the mount again. + */ + +struct fence_info { + struct kset *kset; + struct kobject fence_dir_kobj; + struct workqueue_struct *wq; + wait_queue_head_t waitq; + spinlock_t lock; + struct list_head list; +}; + +#define DECLARE_FENCE_INFO(sb, name) \ + struct fence_info *name = SCOUTFS_SB(sb)->fence_info + +struct pending_fence { + struct super_block *sb; + struct scoutfs_sysfs_attrs ssa; + struct list_head entry; + struct timer_list timer; + + ktime_t start_kt; + __be32 ipv4_addr; + bool fenced; + bool error; + int reason; + u64 rid; +}; + +#define FENCE_FROM_KOBJ(kobj) \ + container_of(SCOUTFS_SYSFS_ATTRS(kobj), struct pending_fence, ssa) +#define DECLARE_FENCE_FROM_KOBJ(name, kobj) \ + struct pending_fence *name = FENCE_FROM_KOBJ(kobj) + +static void destroy_fence(struct pending_fence *fence) +{ + struct super_block *sb = fence->sb; + + scoutfs_sysfs_destroy_attrs(sb, &fence->ssa); + del_timer_sync(&fence->timer); + kfree(fence); +} + +static ssize_t elapsed_secs_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + ktime_t now = ktime_get(); + struct timeval tv = { 0, }; + + if (ktime_after(now, fence->start_kt)) + tv = ktime_to_timeval(ktime_sub(now, fence->start_kt)); + + return snprintf(buf, PAGE_SIZE, "%llu", (long long)tv.tv_sec); +} +SCOUTFS_ATTR_RO(elapsed_secs); + +static ssize_t fenced_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + + return snprintf(buf, PAGE_SIZE, "%u", !!fence->fenced); +} + +/* + * any write to the fenced file from userspace indicates that the mount + * has been safely fenced and can no longer write to the shared device. + */ +static ssize_t fenced_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + DECLARE_FENCE_INFO(fence->sb, fi); + + if (!fence->fenced) { + del_timer_sync(&fence->timer); + fence->fenced = true; + wake_up(&fi->waitq); + } + + return count; +} +SCOUTFS_ATTR_RW(fenced); + +static ssize_t error_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + + return snprintf(buf, PAGE_SIZE, "%u", !!fence->error); +} + +/* + * Fencing can tell us that they were unable to fence the given mount. + * We can't continue if the mount can't be isolated so we shut down the + * server. + */ +static ssize_t error_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, + size_t count) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + struct super_block *sb = fence->sb; + DECLARE_FENCE_INFO(fence->sb, fi); + + if (!fence->error) { + fence->error = true; + scoutfs_err(sb, "error indicated by fence action for rid %016llx", fence->rid); + wake_up(&fi->waitq); + } + + return count; +} +SCOUTFS_ATTR_RW(error); + +static ssize_t ipv4_addr_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + + return snprintf(buf, PAGE_SIZE, "%pI4", &fence->ipv4_addr); +} +SCOUTFS_ATTR_RO(ipv4_addr); + +static ssize_t reason_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + unsigned r = fence->reason; + char *str = "unknown"; + static char *reasons[] = { + [SCOUTFS_FENCE_TEST] = "test", + }; + + if (r < ARRAY_SIZE(reasons) && reasons[r]) + str = reasons[r]; + + return snprintf(buf, PAGE_SIZE, "%s", str); +} +SCOUTFS_ATTR_RO(reason); + +static ssize_t rid_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + + return snprintf(buf, PAGE_SIZE, "%016llx", fence->rid); +} +SCOUTFS_ATTR_RO(rid); + +static struct attribute *fence_attrs[] = { + SCOUTFS_ATTR_PTR(elapsed_secs), + SCOUTFS_ATTR_PTR(fenced), + SCOUTFS_ATTR_PTR(error), + SCOUTFS_ATTR_PTR(ipv4_addr), + SCOUTFS_ATTR_PTR(reason), + SCOUTFS_ATTR_PTR(rid), + NULL, +}; + +#define FENCE_TIMEOUT_MS (MSEC_PER_SEC * 30) + +static void fence_timeout(struct timer_list *timer) +{ + struct pending_fence *fence = from_timer(fence, timer, timer); + struct super_block *sb = fence->sb; + DECLARE_FENCE_INFO(sb, fi); + + fence->error = true; + scoutfs_err(sb, "fence request for rid %016llx was not serviced in %lums, raising error", + fence->rid, FENCE_TIMEOUT_MS); + wake_up(&fi->waitq); +} + +int scoutfs_fence_start(struct super_block *sb, u64 rid, __be32 ipv4_addr, int reason) +{ + DECLARE_FENCE_INFO(sb, fi); + struct pending_fence *fence; + int ret; + + fence = kzalloc(sizeof(struct pending_fence), GFP_NOFS); + if (!fence) { + ret = -ENOMEM; + goto out; + } + + fence->sb = sb; + scoutfs_sysfs_init_attrs(sb, &fence->ssa); + + fence->start_kt = ktime_get(); + fence->ipv4_addr = ipv4_addr; + fence->fenced = false; + fence->error = false; + fence->reason = reason; + fence->rid = rid; + + ret = scoutfs_sysfs_create_attrs_parent(sb, &fi->kset->kobj, + &fence->ssa, fence_attrs, + "%016llx", rid); + if (ret < 0) { + kfree(fence); + goto out; + } + + timer_setup(&fence->timer, fence_timeout, 0); + fence->timer.expires = jiffies + msecs_to_jiffies(FENCE_TIMEOUT_MS); + add_timer(&fence->timer); + + spin_lock(&fi->lock); + list_add_tail(&fence->entry, &fi->list); + spin_unlock(&fi->lock); +out: + return ret; +} + +/* + * Give the caller the rid of the next fence request which has been + * fenced. This doesn't have a position from which to return the next + * because the caller either frees the fence request it's given or shuts + * down. + */ +int scoutfs_fence_next(struct super_block *sb, u64 *rid, int *reason, bool *error) +{ + DECLARE_FENCE_INFO(sb, fi); + struct pending_fence *fence; + int ret = -ENOENT; + + spin_lock(&fi->lock); + list_for_each_entry(fence, &fi->list, entry) { + if (fence->fenced || fence->error) { + *rid = fence->rid; + *reason = fence->reason; + *error = fence->error; + ret = 0; + break; + } + } + spin_unlock(&fi->lock); + + return ret; +} + +int scoutfs_fence_free(struct super_block *sb, u64 rid) +{ + DECLARE_FENCE_INFO(sb, fi); + struct pending_fence *fence; + int ret = -ENOENT; + + spin_lock(&fi->lock); + list_for_each_entry(fence, &fi->list, entry) { + if (fence->rid == rid) { + list_del_init(&fence->entry); + ret = 0; + break; + } + } + spin_unlock(&fi->lock); + + if (ret == 0) { + destroy_fence(fence); + wake_up(&fi->waitq); + } + + return ret; +} + +static bool all_fenced(struct fence_info *fi, bool *error) +{ + struct pending_fence *fence; + bool all = true; + + *error = false; + + spin_lock(&fi->lock); + list_for_each_entry(fence, &fi->list, entry) { + if (fence->error) { + *error = true; + all = true; + break; + } + if (!fence->fenced) { + all = false; + break; + } + } + spin_unlock(&fi->lock); + + return all; +} + +/* + * The caller waits for all the current requests to be fenced, but not + * necessarily reclaimed. + */ +int scoutfs_fence_wait_fenced(struct super_block *sb, long timeout_jiffies) +{ + DECLARE_FENCE_INFO(sb, fi); + bool error; + long ret; + + ret = wait_event_interruptible_timeout(fi->waitq, all_fenced(fi, &error), timeout_jiffies); + if (ret == 0) + ret = -ETIMEDOUT; + else if (ret > 0) + ret = 0; + else if (error) + ret = -EIO; + + return ret; +} + +/* + * This must be called early during startup so that it is guaranteed that + * no other subsystems will try and call fence_start while we're waiting + * for testing fence requests to complete. + */ +int scoutfs_fence_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct mount_options *opts = &sbi->opts; + struct fence_info *fi; + int ret; + + /* can only fence if we can be elected by quorum */ + if (opts->quorum_slot_nr == -1) { + ret = 0; + goto out; + } + + fi = kzalloc(sizeof(struct fence_info), GFP_KERNEL); + if (!fi) { + ret = -ENOMEM; + goto out; + } + + init_waitqueue_head(&fi->waitq); + spin_lock_init(&fi->lock); + INIT_LIST_HEAD(&fi->list); + + sbi->fence_info = fi; + + fi->kset = kset_create_and_add("fence", NULL, scoutfs_sysfs_sb_dir(sb)); + if (!fi->kset) { + ret = -ENOMEM; + goto out; + } + + fi->wq = alloc_workqueue("scoutfs_fence", + WQ_UNBOUND | WQ_NON_REENTRANT, 0); + if (!fi->wq) { + ret = -ENOMEM; + goto out; + } + + ret = 0; +out: + if (ret) + scoutfs_fence_destroy(sb); + + return ret; +} + +/* + * Tear down all pending fence requests because the server is shutting down. + */ +void scoutfs_fence_stop(struct super_block *sb) +{ + DECLARE_FENCE_INFO(sb, fi); + struct pending_fence *fence; + + do { + spin_lock(&fi->lock); + fence = list_first_entry_or_null(&fi->list, struct pending_fence, entry); + if (fence) + list_del_init(&fence->entry); + spin_unlock(&fi->lock); + + if (fence) { + destroy_fence(fence); + wake_up(&fi->waitq); + } + } while (fence); +} + +void scoutfs_fence_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct fence_info *fi = SCOUTFS_SB(sb)->fence_info; + struct pending_fence *fence; + struct pending_fence *tmp; + + if (fi) { + if (fi->wq) + destroy_workqueue(fi->wq); + list_for_each_entry_safe(fence, tmp, &fi->list, entry) + destroy_fence(fence); + if (fi->kset) + kset_unregister(fi->kset); + kfree(fi); + sbi->fence_info = NULL; + } +} diff --git a/kmod/src/fence.h b/kmod/src/fence.h new file mode 100644 index 00000000..0cd8db26 --- /dev/null +++ b/kmod/src/fence.h @@ -0,0 +1,16 @@ +#ifndef _SCOUTFS_FENCE_H_ +#define _SCOUTFS_FENCE_H_ + +enum { +}; + +int scoutfs_fence_start(struct super_block *sb, u64 rid, __be32 ipv4_addr, int reason); +int scoutfs_fence_next(struct super_block *sb, u64 *rid, int *reason, bool *error); +int scoutfs_fence_free(struct super_block *sb, u64 rid); +int scoutfs_fence_wait_fenced(struct super_block *sb, long timeout_jiffies); + +int scoutfs_fence_setup(struct super_block *sb); +void scoutfs_fence_stop(struct super_block *sb); +void scoutfs_fence_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index dbeb3015..19e1503f 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -47,6 +47,7 @@ #include "recov.h" #include "omap.h" #include "volopt.h" +#include "fence.h" #include "scoutfs_trace.h" static struct dentry *scoutfs_debugfs_root; @@ -270,6 +271,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_block_destroy(sb); scoutfs_destroy_triggers(sb); + scoutfs_fence_destroy(sb); scoutfs_options_destroy(sb); scoutfs_sysfs_destroy_attrs(sb, &sbi->mopts_ssa); debugfs_remove(sbi->debug_root); @@ -606,6 +608,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_sysfs_create_attrs(sb, &sbi->mopts_ssa, mount_options_attrs, "mount_options") ?: scoutfs_setup_triggers(sb) ?: + scoutfs_fence_setup(sb) ?: scoutfs_block_setup(sb) ?: scoutfs_forest_setup(sb) ?: scoutfs_item_setup(sb) ?: diff --git a/kmod/src/super.h b/kmod/src/super.h index cbbbc3a0..e44d6575 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -29,6 +29,7 @@ struct srch_info; struct recov_info; struct omap_info; struct volopt_info; +struct fence_info; struct scoutfs_sb_info { struct super_block *sb; @@ -54,6 +55,7 @@ struct scoutfs_sb_info { struct omap_info *omap_info; struct volopt_info *volopt_info; struct item_cache_info *item_cache_info; + struct fence_info *fence_info; wait_queue_head_t trans_hold_wq; struct task_struct *trans_task; From 943351944aa33d9c86977c2a477a0313b0025ed0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 23 Sep 2019 16:25:26 -0700 Subject: [PATCH 05/22] Call fencing from the server The server is responsible for calling the fencing subsystem. It is the source of fencing requests as it decides that previous mounts are unresponsive. It is responsible for reclaiming resources for fenced mounts and freeing their associated fence request. Signed-off-by: Zach Brown --- kmod/src/fence.c | 4 +- kmod/src/fence.h | 3 ++ kmod/src/format.h | 6 +++ kmod/src/net.c | 16 +++++-- kmod/src/quorum.c | 94 ++++++++++++++++++++++++++++++--------- kmod/src/quorum.h | 3 ++ kmod/src/server.c | 111 ++++++++++++++++++++++++++++++++++++++++------ 7 files changed, 199 insertions(+), 38 deletions(-) diff --git a/kmod/src/fence.c b/kmod/src/fence.c index cf5da8bd..c7bb5aa4 100644 --- a/kmod/src/fence.c +++ b/kmod/src/fence.c @@ -187,7 +187,9 @@ static ssize_t reason_show(struct kobject *kobj, struct kobj_attribute *attr, unsigned r = fence->reason; char *str = "unknown"; static char *reasons[] = { - [SCOUTFS_FENCE_TEST] = "test", + [SCOUTFS_FENCE_CLIENT_RECOVERY] = "client_recovery", + [SCOUTFS_FENCE_CLIENT_RECONNECT] = "client_reconnect", + [SCOUTFS_FENCE_QUORUM_BLOCK_LEADER] = "quorum_block_leader", }; if (r < ARRAY_SIZE(reasons) && reasons[r]) diff --git a/kmod/src/fence.h b/kmod/src/fence.h index 0cd8db26..96263d7d 100644 --- a/kmod/src/fence.h +++ b/kmod/src/fence.h @@ -2,6 +2,9 @@ #define _SCOUTFS_FENCE_H_ enum { + SCOUTFS_FENCE_CLIENT_RECOVERY, + SCOUTFS_FENCE_CLIENT_RECONNECT, + SCOUTFS_FENCE_QUORUM_BLOCK_LEADER, }; int scoutfs_fence_start(struct super_block *sb, u64 rid, __be32 ipv4_addr, int reason); diff --git a/kmod/src/format.h b/kmod/src/format.h index 3c0d789f..c2b938d7 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -586,6 +586,12 @@ struct scoutfs_xattr { #define SCOUTFS_QUORUM_HB_IVAL_MS 100 #define SCOUTFS_QUORUM_HB_TIMEO_MS (5 * MSEC_PER_SEC) +/* + * A newly elected leader will give fencing some time before giving up and + * shutting down. + */ +#define SCOUTFS_QUORUM_FENCE_TO_MS (15 * MSEC_PER_SEC) + struct scoutfs_quorum_message { __le64 fsid; __le64 version; diff --git a/kmod/src/net.c b/kmod/src/net.c index 2837307c..0199b8e9 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -30,6 +30,7 @@ #include "net.h" #include "endian_swap.h" #include "tseq.h" +#include "fence.h" /* * scoutfs networking delivers requests and responses between nodes. @@ -1220,6 +1221,7 @@ static void scoutfs_net_reconn_free_worker(struct work_struct *work) unsigned long now = jiffies; unsigned long deadline = 0; bool requeue = false; + int ret; trace_scoutfs_net_reconn_free_work_enter(sb, 0, 0); @@ -1233,10 +1235,18 @@ restart: time_after_eq(now, acc->reconn_deadline))) { set_conn_fl(acc, reconn_freeing); spin_unlock(&conn->lock); - if (!test_conn_fl(conn, shutting_down)) - scoutfs_info(sb, "client timed out "SIN_FMT" -> "SIN_FMT", can not reconnect", - SIN_ARG(&acc->sockname), + if (!test_conn_fl(conn, shutting_down)) { + scoutfs_info(sb, "client "SIN_FMT" reconnect timed out, fencing", SIN_ARG(&acc->peername)); + ret = scoutfs_fence_start(sb, acc->rid, + acc->peername.sin_addr.s_addr, + SCOUTFS_FENCE_CLIENT_RECONNECT); + if (ret) { + scoutfs_err(sb, "client fence returned err %d, shutting down server", + ret); + scoutfs_server_abort(sb); + } + } destroy_conn(acc); goto restart; } diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index c7d50cab..2a6d877e 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -32,6 +32,7 @@ #include "block.h" #include "net.h" #include "sysfs.h" +#include "fence.h" #include "scoutfs_trace.h" /* @@ -461,28 +462,78 @@ static int update_quorum_block(struct super_block *sb, u64 blkno, return ret; } +/* + * The calling server had fenced previous leaders before starting up, + * now that it's up it has reclaimed their resources and can clear their + * leader flags. + */ +int scoutfs_quorum_clear_rid_leader(struct super_block *sb, u64 rid) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct mount_options *opts = &sbi->opts; + struct scoutfs_quorum_block blk; + int ret = 0; + u64 blkno; + int i; + + for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { + if (i == opts->quorum_slot_nr || !quorum_slot_present(super, i)) + continue; + + blkno = SCOUTFS_QUORUM_BLKNO + i; + ret = read_quorum_block(sb, blkno, &blk, NULL); + if (ret < 0) + break; + + if (le64_to_cpu(blk.set_leader.rid) == rid) { + blk.flags &= ~cpu_to_le64(SCOUTFS_QUORUM_BLOCK_LEADER); + set_quorum_block_event(sb, &blk, &blk.fenced); + + ret = write_quorum_block(sb, blkno, &blk, NULL); + break; + } + } + + if (ret < 0) + scoutfs_err(sb, "error %d clearing leader block for rid %016llx", ret, rid); + + return ret; +} /* - * The calling server has been elected and updated their block, but - * can't yet assume that it has exclusive access to the metadata device. - * We read all the quorum blocks looking for previously elected leaders - * to fence so that we're the only leader running. + * The calling server has been elected, had its block updated, and has + * started running but can't yet assume that it has exclusive access to + * the metadata device. We read all the quorum blocks looking for + * previously elected leaders to fence so that we're the only leader + * running. + * + * We only wait for the previous leaders to be fenced. We don't clear + * the leader bits because the server is going to reclaim their + * resources once its up and running. Only then will the leader bits be + * cleared. + * + * Quorum will be sending heartbeats while we wait for fencing. That + * keeps us from being fenced while we allow userspace fencing to take a + * reasonably long time. We still want to timeout eventually. */ -static int fence_leader_blocks(struct super_block *sb) +int scoutfs_quorum_fence_leader_blocks(struct super_block *sb, u64 term) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct mount_options *opts = &sbi->opts; struct scoutfs_quorum_block blk; struct sockaddr_in sin; + bool fence_started = false; u64 blkno; int ret = 0; + int err; int i; BUILD_BUG_ON(SCOUTFS_QUORUM_BLOCKS < SCOUTFS_QUORUM_MAX_SLOTS); for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - if (i == opts->quorum_slot_nr) + if (i == opts->quorum_slot_nr || !quorum_slot_present(super, i)) continue; blkno = SCOUTFS_QUORUM_BLKNO + i; @@ -490,27 +541,31 @@ static int fence_leader_blocks(struct super_block *sb) if (ret < 0) goto out; - if (!(le64_to_cpu(blk.flags) & SCOUTFS_QUORUM_BLOCK_LEADER)) + if (!(le64_to_cpu(blk.flags) & SCOUTFS_QUORUM_BLOCK_LEADER) || + le64_to_cpu(blk.term) > term) continue; scoutfs_inc_counter(sb, quorum_fence_leader); scoutfs_quorum_slot_sin(super, i, &sin); - scoutfs_err(sb, "fencing "SCSBF" at "SIN_FMT, - SCSB_LEFR_ARGS(super->hdr.fsid, blk.set_leader.rid), - SIN_ARG(&sin)); - - blk.flags &= ~cpu_to_le64(SCOUTFS_QUORUM_BLOCK_LEADER); - set_quorum_block_event(sb, &blk, &blk.fenced); - - ret = write_quorum_block(sb, blkno, &blk, NULL); + scoutfs_info(sb, "fencing previous leader "SCSBF" in slot %u with address "SIN_FMT, + SCSB_LEFR_ARGS(super->hdr.fsid, blk.set_leader.rid), i, SIN_ARG(&sin)); + ret = scoutfs_fence_start(sb, le64_to_cpu(blk.set_leader.rid), sin.sin_addr.s_addr, + SCOUTFS_FENCE_QUORUM_BLOCK_LEADER); if (ret < 0) goto out; + fence_started = true; + } out: + if (fence_started) { + err = scoutfs_fence_wait_fenced(sb, msecs_to_jiffies(SCOUTFS_QUORUM_FENCE_TO_MS)); + if (ret == 0) + ret = err; + } if (ret < 0) { - scoutfs_err(sb, "error %d fencing active", ret); + scoutfs_err(sb, "error %d fencing leader blocks", ret); scoutfs_inc_counter(sb, quorum_fence_error); } @@ -670,10 +725,8 @@ static void scoutfs_quorum_worker(struct work_struct *work) qst.term); qst.timeout = heartbeat_interval(); - /* set our leader flag and fence */ - ret = update_quorum_block(sb, blkno, &mark, - qst.role, qst.term) ?: - fence_leader_blocks(sb); + /* set our leader flag before starting server */ + ret = update_quorum_block(sb, blkno, &mark, qst.role, qst.term); if (ret < 0) goto out; @@ -727,6 +780,7 @@ static void scoutfs_quorum_worker(struct work_struct *work) /* always try to stop a running server as we stop */ if (test_bit(QINF_FLAG_SERVER, &qinf->flags)) { scoutfs_server_stop(sb); + scoutfs_fence_stop(sb); send_msg_others(sb, SCOUTFS_QUORUM_MSG_RESIGNATION, qst.term); } diff --git a/kmod/src/quorum.h b/kmod/src/quorum.h index f0994871..3f41fd65 100644 --- a/kmod/src/quorum.h +++ b/kmod/src/quorum.h @@ -8,6 +8,9 @@ u8 scoutfs_quorum_votes_needed(struct super_block *sb); void scoutfs_quorum_slot_sin(struct scoutfs_super_block *super, int i, struct sockaddr_in *sin); +int scoutfs_quorum_fence_leader_blocks(struct super_block *sb, u64 term); +int scoutfs_quorum_clear_rid_leader(struct super_block *sb, u64 rid); + int scoutfs_quorum_setup(struct super_block *sb); void scoutfs_quorum_shutdown(struct super_block *sb); void scoutfs_quorum_destroy(struct super_block *sb); diff --git a/kmod/src/server.c b/kmod/src/server.c index fb353b25..fbe602a9 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -40,6 +40,7 @@ #include "forest.h" #include "recov.h" #include "omap.h" +#include "fence.h" /* * Every active mount can act as the server that listens on a net @@ -106,6 +107,8 @@ struct server_info { /* recovery timeout fences from work */ struct work_struct fence_pending_recov_work; + /* while running we check for fenced mounts to reclaim */ + struct delayed_work reclaim_dwork; }; #define DECLARE_SERVER_INFO(sb, name) \ @@ -1672,7 +1675,15 @@ static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) sizeof(struct scoutfs_mounted_client_btree_val)); } -static int reclaim_rid(struct super_block *sb, u64 rid) +/* + * Reclaim all the resources for a mount which has gone away. It's sent + * us a farewell promising to leave or we actively fenced it. + * + * It's safe to call this multiple times for a given rid. Each + * individual action knows to recognize that it's already been performed + * and return success. + */ +static int reclaim_rid(struct super_block *sb, u64 rid, bool clear_leader) { int ret; @@ -1680,13 +1691,14 @@ static int reclaim_rid(struct super_block *sb, u64 rid) if (ret < 0) return ret; - /* delete mounted client last, client reconnect looks for it */ + /* delete mounted client last, recovery looks for it */ ret = scoutfs_lock_server_farewell(sb, rid) ?: remove_trans_seq(sb, rid) ?: reclaim_log_trees(sb, rid) ?: cancel_srch_compact(sb, rid) ?: - delete_mounted_client(sb, rid) ?: - scoutfs_omap_remove_rid(sb, rid); + scoutfs_omap_remove_rid(sb, rid) ?: + (clear_leader ? scoutfs_quorum_clear_rid_leader(sb, rid) : 0) ?: + delete_mounted_client(sb, rid); return scoutfs_server_apply_commit(sb, ret); } @@ -1816,9 +1828,9 @@ static void farewell_worker(struct work_struct *work) } } - /* process and send farewell responses */ + /* clean up resources for mounts before sending responses */ list_for_each_entry_safe(fw, tmp, &send, entry) { - ret = reclaim_rid(sb, fw->rid); + ret = reclaim_rid(sb, fw->rid, false); if (ret) goto out; } @@ -2017,21 +2029,19 @@ static void fence_pending_recov_worker(struct work_struct *work) struct server_info *server = container_of(work, struct server_info, fence_pending_recov_work); struct super_block *sb = server->sb; - u64 rid; + u64 rid = 0; int ret; - while ((rid = scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_ALL)) > 0) { + while ((rid = scoutfs_recov_next_pending(sb, rid, SCOUTFS_RECOV_ALL)) > 0) { scoutfs_err(sb, "%lu ms recovery timeout expired for client rid %016llx, fencing", SERVER_RECOV_TIMEOUT_MS, rid); - ret = reclaim_rid(sb, rid); - if (ret < 0) { - scoutfs_err(sb, "error %d reclaiming rid %016llx, shutting down", ret, rid); - stop_server(server); + ret = scoutfs_fence_start(sb, rid, 0, SCOUTFS_FENCE_CLIENT_RECOVERY); + if (ret) { + scoutfs_err(sb, "fence returned err %d, shutting down server", ret); + scoutfs_server_abort(sb); break; } - - scoutfs_server_recov_finish(sb, rid, SCOUTFS_RECOV_ALL); } } @@ -2102,6 +2112,69 @@ out: return ret; } +static void queue_reclaim_work(struct server_info *server, unsigned long delay) +{ + if (!server->shutting_down) + queue_delayed_work(server->wq, &server->reclaim_dwork, delay); +} + +#define RECLAIM_WORK_DELAY_MS MSEC_PER_SEC + +/* + * Fencing is performed by userspace and can happen as we're elected + * leader before the server is running. Once we're running we want to + * reclaim resources from any mounts that may have been fenced. + * + * The reclaim worker runs regularly in the background and reclaims the + * resources for mounts that have been fenced. Once the fenced rid has + * been reclaimed the fence request can be removed. + * + * This is queued by the server work as it starts up, requeues itself + * until shutdown, and is then canceled by the server work as it shuts + * down. + */ +static void reclaim_worker(struct work_struct *work) +{ + struct server_info *server = container_of(work, struct server_info, reclaim_dwork.work); + struct super_block *sb = server->sb; + bool error; + int reason; + u64 rid; + int ret; + + ret = scoutfs_fence_next(sb, &rid, &reason, &error); + if (ret < 0) + goto out; + + if (error == true) { + scoutfs_err(sb, "saw error indicator on fence request for rid %016llx, shutting down server", + rid); + scoutfs_server_abort(sb); + ret = -ESHUTDOWN; + goto out; + } + + ret = reclaim_rid(sb, rid, reason == SCOUTFS_FENCE_QUORUM_BLOCK_LEADER); + if (ret < 0) { + scoutfs_err(sb, "failure to reclaim fenced rid %016llx: err %d, shutting down server", + rid, ret); + scoutfs_server_abort(sb); + goto out; + } + + scoutfs_info(sb, "successfully reclaimed resources for fenced rid %016llx", rid); + scoutfs_fence_free(sb, rid); + scoutfs_server_recov_finish(sb, rid, SCOUTFS_RECOV_ALL); + ret = 0; + +out: + /* queue next reclaim immediately if we're making progress */ + if (ret == 0) + queue_reclaim_work(server, 0); + else + queue_reclaim_work(server, msecs_to_jiffies(RECLAIM_WORK_DELAY_MS)); +} + static void scoutfs_server_worker(struct work_struct *work) { struct server_info *server = container_of(work, struct server_info, @@ -2118,6 +2191,11 @@ static void scoutfs_server_worker(struct work_struct *work) trace_scoutfs_server_work_enter(sb, 0, 0); + /* first make sure no other servers are still running */ + ret = scoutfs_quorum_fence_leader_blocks(sb, server->term); + if (ret < 0) + goto out; + scoutfs_quorum_slot_sin(super, opts->quorum_slot_nr, &sin); scoutfs_info(sb, "server setting up at "SIN_FMT, SIN_ARG(&sin)); @@ -2189,6 +2267,8 @@ static void scoutfs_server_worker(struct work_struct *work) scoutfs_info(sb, "server ready at "SIN_FMT, SIN_ARG(&sin)); complete(&server->start_comp); + queue_reclaim_work(server, 0); + /* wait_event/wake_up provide barriers */ wait_event_interruptible(server->waitq, server->shutting_down); @@ -2197,6 +2277,7 @@ shutdown: /* wait for farewell to finish sending messages */ flush_work(&server->farewell_work); + cancel_delayed_work_sync(&server->reclaim_dwork); /* wait for requests to finish, no more requests */ scoutfs_net_shutdown(sb, conn); @@ -2209,6 +2290,7 @@ shutdown: /* wait for extra queues by requests, won't find waiters */ flush_work(&server->commit_work); + scoutfs_fence_stop(sb); scoutfs_lock_server_destroy(sb); scoutfs_omap_server_shutdown(sb); @@ -2299,6 +2381,7 @@ int scoutfs_server_setup(struct super_block *sb) seqcount_init(&server->volopt_seqcount); mutex_init(&server->volopt_mutex); INIT_WORK(&server->fence_pending_recov_work, fence_pending_recov_worker); + INIT_DELAYED_WORK(&server->reclaim_dwork, reclaim_worker); server->wq = alloc_workqueue("scoutfs_server", WQ_UNBOUND | WQ_NON_REENTRANT, 0); From 1f1f40f07992823f121a6b74bfc635d74ba31a91 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 3 Mar 2021 14:27:50 -0800 Subject: [PATCH 06/22] Add fence agent that processes fence requests Signed-off-by: Zach Brown --- utils/fenced/local-force-unmount | 35 ++++++++++++ utils/fenced/scoutfs-fenced | 94 ++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100755 utils/fenced/local-force-unmount create mode 100755 utils/fenced/scoutfs-fenced diff --git a/utils/fenced/local-force-unmount b/utils/fenced/local-force-unmount new file mode 100755 index 00000000..9d97a79b --- /dev/null +++ b/utils/fenced/local-force-unmount @@ -0,0 +1,35 @@ +#!/usr/bin/bash + +echo_fail() { + echo "$@" > /dev/stderr + exit 1 +} + +rid="$SCOUTFS_FENCED_REQ_RID" + +# +# Look for a local mount with the rid to fence. Typically we'll at +# least find the mount with the server that requested the fence that +# we're processing. But it's possible that mounts are unmounted +# before, or while, we're running. +# +mnts=$(findmnt -l -n -t scoutfs -o TARGET) || \ + echo_fail "findmnt -t scoutfs failed" > /dev/stderr + +for mnt in $mnts; do + mnt_rid=$(scoutfs statfs -p "$mnt" -s rid) || \ + echo_fail "scoutfs statfs $mnt failed" + + if [ "$mnt_rid" == "$rid" ]; then + umount -f "$mnt" || \ + echo_fail "umout -f $mnt" + + exit 0 + fi +done + +# +# If the mount doesn't exist on this host then it can't access the +# devices by definition and can be considered fenced. +# +exit 0 diff --git a/utils/fenced/scoutfs-fenced b/utils/fenced/scoutfs-fenced new file mode 100755 index 00000000..2037de38 --- /dev/null +++ b/utils/fenced/scoutfs-fenced @@ -0,0 +1,94 @@ +#!/usr/bin/bash + +message_output() +{ + printf "[%s] %s\n" "$(date '+%F %T.%N')" "$@" +} + +error_message() +{ + message_output "$@" >> /dev/stderr +} + +error_exit() +{ + error_message "$@, exiting" + exit 1 +} + +log_message() +{ + message_output "$@" >> /dev/stdout +} + +# restart if we catch hup to re-read the config +hup_restart() +{ + log_message "caught SIGHUP, restarting" + exec "$@" +} +trap hup_restart SIGHUP + +# defaults +SCOUTFS_FENCED_CONFIG_FILE=${SCOUTFS_FENCED_CONFIG_FILE:-/etc/scoutfs/scoutfs-fenced.conf} +SCOUTFS_FENCED_DELAY=2 +#SCOUTFS_FENCED_RUN +#SCOUTFS_FENCED_RUN_ARGS + +test -n "$SCOUTFS_FENCED_CONFIG_FILE" || \ + error_exit "SCOUTFS_FENCED_CONFIG_FILE isn't set" +test -r "$SCOUTFS_FENCED_CONFIG_FILE" || \ + error_exit "SCOUTFS_FENCED_CONFIG_FILE isn't readable file" + +log_message "reading config file $SCOUTFS_FENCED_CONFIG_FILE" + +. "$SCOUTFS_FENCED_CONFIG_FILE" || \ + error_exit "error sourcing $SCOUTFS_FENCED_CONFIG_FILE as bash script" + +for conf in "${!SCOUTFS_FENCED_@}"; do + log_message " config var $conf=${!conf}" +done + +test -n "$SCOUTFS_FENCED_RUN" || \ + error_exit "SCOUTFS_FENCED_RUN must be set" +test -x "$SCOUTFS_FENCED_RUN" || \ + error_exit "SCOUTFS_FENCED_RUN '$SCOUTFS_FENCED_RUN' isn't executable" + +# +# main loop watching for fence request across all filesystems +# + +while sleep $SCOUTFS_FENCED_DELAY; do + for fence in /sys/fs/scoutfs/*/fence/*; do + # catches unmatched regex when no dirs + if [ ! -d "$fence" ]; then + continue + fi + + # skip requests that have been handled + if [ $(cat "$fence/fenced") == 1 -o $(cat "$fence/error") == 1 ]; then + continue + fi + + srv=$(basename $(dirname $(dirname $fence))) + rid="$(cat $fence/rid)" + ip="$(cat $fence/ipv4_addr)" + reason="$(cat $fence/reason)" + + log_message "server $srv fencing rid $rid at IP $ip for $reason" + + # export _REQ_ vars for run to use + export SCOUTFS_FENCED_REQ_RID="$rid" + export SCOUTFS_FENCED_REQ_IP="$ip" + + $run $SCOUTFS_FENCED_RUN_ARGS + rc=$? + if [ "$rc" != 0 ]; then + log_message "server $srv fencing rid $rid saw error status $rc from $run" + echo 1 > "$fence/error" + continue + fi + + echo 1 > "$fence/fenced" + done +done From 8b78f701a1fcbb99fcb65a283610b037ed2c969f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 26 Feb 2021 10:07:01 -0800 Subject: [PATCH 07/22] Add fence-and-reclaim test Add a test which exercises the various reasons for fencing mounts and checks that we reclaim the resources that they had. Signed-off-by: Zach Brown --- tests/funcs/fs.sh | 85 +++++++++++++++++++++ tests/golden/fence-and-reclaim | 5 ++ tests/sequence | 1 + tests/tests/fence-and-reclaim.sh | 127 +++++++++++++++++++++++++++++++ 4 files changed, 218 insertions(+) create mode 100644 tests/golden/fence-and-reclaim create mode 100644 tests/tests/fence-and-reclaim.sh diff --git a/tests/funcs/fs.sh b/tests/funcs/fs.sh index 03fb9888..b68bcfe7 100644 --- a/tests/funcs/fs.sh +++ b/tests/funcs/fs.sh @@ -17,6 +17,17 @@ t_sync_seq_index() t_quiet sync } +t_mount_rid() +{ + local nr="${1:-0}" + local mnt="$(eval echo \$T_M$nr)" + local rid + + rid=$(scoutfs statfs -s rid -p "$mnt") + + echo "$rid" +} + # # Output the "f.$fsid.r.$rid" identifier string for the given mount # number, 0 is used by default if none is specified. @@ -132,6 +143,16 @@ t_umount() eval t_quiet umount \$T_M$nr } +t_force_umount() +{ + local nr="$1" + + test "$nr" -lt "$T_NR_MOUNTS" || \ + t_fail "fs nr $nr invalid" + + eval t_quiet umount -f \$T_M$nr +} + # # Attempt to mount all the configured mounts, assuming that they're # not already mounted. @@ -277,3 +298,67 @@ t_counter_diff_changed() { echo "counter $which didn't change" || echo "counter $which changed" } + +# +# See if we can find a local mount with the caller's rid. +# +t_rid_is_mounted() { + local rid="$1" + local fr="$1" + + for fr in /sys/fs/scoutfs/*; do + if [ "$(cat $fr/rid)" == "$rid" ]; then + return 0 + fi + done + + return 1 +} + +# +# A given mount is being fenced if any mount has a fence request pending +# for it which hasn't finished and been removed. +# +t_rid_is_fencing() { + local rid="$1" + local fr + + for fr in /sys/fs/scoutfs/*; do + if [ -d "$fr/fence/$rid" ]; then + return 0 + fi + done + + return 1 +} + +# +# Wait until the mount identified by the first rid arg is not in any +# states specified by the remaining state description word args. +# +t_wait_if_rid_is() { + local rid="$1" + + while ( [[ $* =~ mounted ]] && t_rid_is_mounted $rid ) || + ( [[ $* =~ fencing ]] && t_rid_is_fencing $rid ) ; do + sleep .5 + done +} + +# +# Wait until any mount identifies itself as the elected leader. We can +# be waiting while tests mount and unmount so mounts may not be mounted +# at the test's expected mount points. +# +t_wait_for_leader() { + local i + + while sleep .25; do + for i in $(t_fs_nrs); do + local ldr="$(t_sysfs_path $i 2>/dev/null)/quorum/is_leader" + if [ "$(cat $ldr 2>/dev/null)" == "1" ]; then + return + fi + done + done +} diff --git a/tests/golden/fence-and-reclaim b/tests/golden/fence-and-reclaim new file mode 100644 index 00000000..dadc5edd --- /dev/null +++ b/tests/golden/fence-and-reclaim @@ -0,0 +1,5 @@ +== make sure all mounts can see each other +== force unmount one client, connection timeout, fence nop, mount +== force unmount all non-server, connection timeout, fence nop, mount +== force unmount server, quorum elects new leader, fence nop, mount +== force unmount everything, new server fences all previous diff --git a/tests/sequence b/tests/sequence index 972bebe2..0d709c1d 100644 --- a/tests/sequence +++ b/tests/sequence @@ -28,6 +28,7 @@ lock-conflicting-batch-commit.sh cross-mount-data-free.sh persistent-item-vers.sh setup-error-teardown.sh +fence-and-reclaim.sh mount-unmount-race.sh createmany-parallel-mounts.sh archive-light-cycle.sh diff --git a/tests/tests/fence-and-reclaim.sh b/tests/tests/fence-and-reclaim.sh new file mode 100644 index 00000000..1ce52048 --- /dev/null +++ b/tests/tests/fence-and-reclaim.sh @@ -0,0 +1,127 @@ +# +# Fence nodes and reclaim their resources. +# + +t_require_commands sleep touch grep sync scoutfs +t_require_mounts 2 + +# +# Make sure that all mounts can read the results of a write from each +# mount. And make sure that the greatest of all the written seqs is +# visible after the writes were commited by remote reads. +# +check_read_write() +{ + local expected + local greatest=0 + local seq + local path + local saw + local w + local r + + for w in $(t_fs_nrs); do + expected="$w wrote at $(date --rfc-3339=ns)" + eval path="\$T_D${w}/written" + echo "$expected" > "$path" + + seq=$(scoutfs stat -s meta_seq $path) + if [ "$seq" -gt "$greatest" ]; then + greatest=$seq + fi + + for r in $(t_fs_nrs); do + eval path="\$T_D${r}/written" + saw=$(cat "$path") + if [ "$saw" != "$expected" ]; then + echo "mount $r read '$saw' after mount $w wrote '$expected'" + fi + done + done + + seq=$(scoutfs statfs -s committed_seq -p $T_D0) + if [ "$seq" -lt "$greatest" ]; then + echo "committed_seq $seq less than greatest $greatest" + fi +} + +echo "== make sure all mounts can see each other" +check_read_write + +echo "== force unmount one client, connection timeout, fence nop, mount" +cl=$(t_first_client_nr) +sv=$(t_server_nr) +rid=$(t_mount_rid $cl) +echo "cl $cl sv $sv rid $rid" >> "$T_TMP.log" +sync +t_force_umount $cl +# wait for client reconnection to timeout +while grep -q $rid $(t_debugfs_path $sv)/connections; do + sleep .5 +done +while t_rid_is_fencing $rid; do + sleep .5 +done +t_mount $cl +check_read_write + +echo "== force unmount all non-server, connection timeout, fence nop, mount" +sv=$(t_server_nr) +pattern="nonsense" +sync +for cl in $(t_fs_nrs); do + if [ $cl == $sv ]; then + continue; + fi + + rid=$(t_mount_rid $cl) + pattern="$pattern|$rid" + echo "cl $cl sv $sv rid $rid" >> "$T_TMP.log" + + t_force_umount $cl +done + +# wait for all client reconnections to timeout +while egrep -q "($pattern)" $(t_debugfs_path $sv)/connections; do + sleep .5 +done +# wait for all fence requests to complete +while test -d $(echo /sys/fs/scoutfs/*/fence/* | cut -d " " -f 1); do + sleep .5 +done +# remount all the clients +for cl in $(t_fs_nrs); do + if [ $cl == $sv ]; then + continue; + fi + t_mount $cl +done +check_read_write + +echo "== force unmount server, quorum elects new leader, fence nop, mount" +sv=$(t_server_nr) +rid=$(t_mount_rid $sv) +echo "sv $sv rid $rid" >> "$T_TMP.log" +sync +t_force_umount $sv +t_wait_for_leader +# wait until new server is done fencing unmounted leader rid +while t_rid_is_fencing $rid; do + sleep .5 +done +t_mount $sv +check_read_write + +echo "== force unmount everything, new server fences all previous" +sync +for nr in $(t_fs_nrs); do + t_force_umount $nr +done +t_mount_all +# wait for all fence requests to complete +while test -d $(echo /sys/fs/scoutfs/*/fence/* | cut -d " " -f 1); do + sleep .5 +done +check_read_write + +t_pass From 9ebc9d0f66ee6481853a2a5d8cbb00e8d7efc013 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 3 Mar 2021 10:33:46 -0800 Subject: [PATCH 08/22] Manage client reconnect delay The client currently always queues immediate connect work if it's nodify_down is called. It was assuming that notify_down is only called from a healthy established connection. But it's also called for unsuccessful conneect attempts that might not have timed out. Say the host is up but the port isn't listening. This results in spamming connection attempts while an old stale leader block until a new server is elected, fences the previous leader, and updates their quorum block. The fix is to explicitly manage the connection work queueing delay. We only set it to immediately queue on mount and when we see a greeting reply from the server. We always set it to a longer timeout as we start a connection attempt. This means we'll always have a long reconnect delay unless we really connected to a server. Signed-off-by: Zach Brown --- kmod/src/client.c | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index f36b72ab..7a4b4322 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -48,6 +48,7 @@ struct client_info { struct workqueue_struct *workq; struct delayed_work connect_dwork; + unsigned long connect_delay_jiffies; u64 server_term; @@ -349,6 +350,7 @@ static int client_greeting(struct super_block *sb, scoutfs_net_client_greeting(sb, conn, new_server); client->server_term = le64_to_cpu(gr->server_term); + client->connect_delay_jiffies = 0; ret = 0; out: return ret; @@ -398,6 +400,20 @@ out: return ret; } +/* + * If we're not seeing successful connections we want to back off. Each + * connection attempt starts by setting a long connection work delay. + * We only set a shorter delay if we see a greeting response from the + * server. At that point we'll try to immediately reconnect if the + * connection is broken. + */ +static void queue_connect_dwork(struct super_block *sb, struct client_info *client) +{ + if (!atomic_read(&client->shutting_down) && !scoutfs_forcing_unmount(sb)) + queue_delayed_work(client->workq, &client->connect_dwork, + client->connect_delay_jiffies); +} + /* * This work is responsible for maintaining a connection from the client * to the server. It's queued on mount and disconnect and we requeue @@ -437,6 +453,9 @@ static void scoutfs_client_connect_worker(struct work_struct *work) goto out; } + /* always wait a bit until a greeting response sets a lower delay */ + client->connect_delay_jiffies = msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS); + ret = scoutfs_quorum_server_sin(sb, &sin); if (ret < 0) goto out; @@ -464,13 +483,8 @@ static void scoutfs_client_connect_worker(struct work_struct *work) if (ret) scoutfs_net_shutdown(sb, client->conn); out: - - /* always have a small delay before retrying to avoid storms */ - if (ret && !atomic_read(&client->shutting_down) && - !scoutfs_forcing_unmount(sb)) { - queue_delayed_work(client->workq, &client->connect_dwork, - msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS)); - } + if (ret) + queue_connect_dwork(sb, client); } static scoutfs_net_request_t client_req_funcs[] = { @@ -489,8 +503,7 @@ static void client_notify_down(struct super_block *sb, { struct client_info *client = SCOUTFS_SB(sb)->client_info; - if (!atomic_read(&client->shutting_down)) - queue_delayed_work(client->workq, &client->connect_dwork, 0); + queue_connect_dwork(sb, client); } int scoutfs_client_setup(struct super_block *sb) @@ -525,7 +538,7 @@ int scoutfs_client_setup(struct super_block *sb) goto out; } - queue_delayed_work(client->workq, &client->connect_dwork, 0); + queue_connect_dwork(sb, client); ret = 0; out: From f3764b873b1b95f4cd028c8fbbdd0f801fdcfe6a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 15 Mar 2021 10:50:05 -0700 Subject: [PATCH 09/22] Save previous connected client address Our connection state spans sockets that can disconnect and reconnect. While sockets are connected we store the socket's remote address in the connection's peername and we clear it as sockets disconnect. Fencing wants to know the last connected address of the mount. It's a bit of metadata we know about the mount that can be used to find it and fence it. As we store the peer address we also stash it away as the last known peer address for the socket. Fencing can then use that instead of the current socket peer address which is guaranteed to be uninitialized because there's no socket connected. Signed-off-by: Zach Brown --- kmod/src/net.c | 7 +++++-- kmod/src/net.h | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index 0199b8e9..593a62e7 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -941,6 +941,8 @@ static int sock_opts_and_names(struct scoutfs_net_connection *conn, ret = -EAFNOSUPPORT; if (ret) goto out; + + conn->last_peername = conn->peername; out: return ret; } @@ -1237,9 +1239,9 @@ restart: spin_unlock(&conn->lock); if (!test_conn_fl(conn, shutting_down)) { scoutfs_info(sb, "client "SIN_FMT" reconnect timed out, fencing", - SIN_ARG(&acc->peername)); + SIN_ARG(&acc->last_peername)); ret = scoutfs_fence_start(sb, acc->rid, - acc->peername.sin_addr.s_addr, + acc->last_peername.sin_addr.s_addr, SCOUTFS_FENCE_CLIENT_RECONNECT); if (ret) { scoutfs_err(sb, "client fence returned err %d, shutting down server", @@ -1317,6 +1319,7 @@ scoutfs_net_alloc_conn(struct super_block *sb, init_waitqueue_head(&conn->waitq); conn->sockname.sin_family = AF_INET; conn->peername.sin_family = AF_INET; + conn->last_peername.sin_family = AF_INET; INIT_LIST_HEAD(&conn->accepted_head); INIT_LIST_HEAD(&conn->accepted_list); conn->next_send_seq = 1; diff --git a/kmod/src/net.h b/kmod/src/net.h index 0b18b6ff..e16ec524 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -49,6 +49,7 @@ struct scoutfs_net_connection { u64 greeting_id; struct sockaddr_in sockname; struct sockaddr_in peername; + struct sockaddr_in last_peername; struct list_head accepted_head; struct scoutfs_net_connection *listening_conn; From ab5466a771e13237741174297fa3f2f27c818e6f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 16 Mar 2021 13:14:45 -0700 Subject: [PATCH 10/22] Protect server shutting down with smp barriers I saw a confusing hang that looked like a lack of ordering between a waker setting shutting_down and a wait event testing it after being woken up. Let's see if more barriers help. Signed-off-by: Zach Brown --- kmod/src/server.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index fbe602a9..cbb23088 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -160,10 +160,21 @@ struct commit_waiter { int ret; }; +static bool test_shutting_down(struct server_info *server) +{ + smp_rmb(); + return server->shutting_down; +} + +static void set_shutting_down(struct server_info *server, bool val) +{ + server->shutting_down = val; + smp_rmb(); +} + static void stop_server(struct server_info *server) { - /* wait_event/wake_up provide barriers */ - server->shutting_down = true; + set_shutting_down(server, true); wake_up(&server->waitq); } @@ -1550,7 +1561,7 @@ static int cancel_srch_compact(struct super_block *sb, u64 rid) */ static void queue_farewell_work(struct server_info *server) { - if (!server->shutting_down) + if (!test_shutting_down(server)) queue_work(server->wq, &server->farewell_work); } @@ -2049,7 +2060,7 @@ static void recovery_timeout(struct super_block *sb) { DECLARE_SERVER_INFO(sb, server); - if (!server->shutting_down) + if (!test_shutting_down(server)) queue_work(server->wq, &server->fence_pending_recov_work); } @@ -2114,7 +2125,7 @@ out: static void queue_reclaim_work(struct server_info *server, unsigned long delay) { - if (!server->shutting_down) + if (!test_shutting_down(server)) queue_delayed_work(server->wq, &server->reclaim_dwork, delay); } @@ -2270,7 +2281,7 @@ static void scoutfs_server_worker(struct work_struct *work) queue_reclaim_work(server, 0); /* wait_event/wake_up provide barriers */ - wait_event_interruptible(server->waitq, server->shutting_down); + wait_event_interruptible(server->waitq, test_shutting_down(server)); shutdown: scoutfs_info(sb, "server shutting down at "SIN_FMT, SIN_ARG(&sin)); @@ -2317,7 +2328,7 @@ int scoutfs_server_start(struct super_block *sb, u64 term) DECLARE_SERVER_INFO(sb, server); server->err = 0; - server->shutting_down = false; + set_shutting_down(server, false); server->term = term; init_completion(&server->start_comp); From 66630342958d10d5c6609d2fe939c2074ab436d1 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 16 Mar 2021 14:08:15 -0700 Subject: [PATCH 11/22] Run the fence agent in the background of tests Signed-off-by: Zach Brown --- tests/run-tests.sh | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/run-tests.sh b/tests/run-tests.sh index 1caeeb44..d60865c1 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -18,10 +18,15 @@ die() { exit 1 } +timestamp() +{ + date '+%F %T.%N' +} + # output a message with a timestamp to the run.log log() { - echo "[$(date '+%F %T.%N')] $*" >> "$T_RESULTS/run.log" + echo "[$(timestamp)] $*" >> "$T_RESULTS/run.log" } # run a logged command, exiting if it fails @@ -367,6 +372,39 @@ cmd cat /sys/kernel/debug/tracing/set_event cmd grep . /sys/kernel/debug/tracing/options/trace_printk \ /proc/sys/kernel/ftrace_dump_on_oops +# +# Build a fenced config that runs scripts out of the repository rather +# than the default system directory +# +conf="$T_RESULTS/scoutfs-fencd.conf" +cat > $conf << EOF +SCOUTFS_FENCED_DELAY=1 +SCOUTFS_FENCED_RUN=$T_UTILS/fenced/local-force-unmount +SCOUTFS_FENCED_RUN_ARGS="" +EOF +export SCOUTFS_FENCED_CONFIG_FILE="$conf" + +# +# Run the agent in the background, log its output, an kill it if we +# exit +# +fenced_log() +{ + echo "[$(timestamp)] $*" >> "$T_RESULTS/fenced.stdout.log" +} +fenced_pid="" +kill_fenced() +{ + if test -n "$fenced_pid" -a -d "/proc/$fenced_pid" ; then + fenced_log "killing fenced pid $fenced_pid" + kill "$fenced_pid" + fi +} +trap kill_fenced EXIT +$T_UTILS/fenced/scoutfs-fenced > "$T_RESULTS/fenced.stdout.log" 2> "$T_RESULTS/fenced.stderr.log" & +fenced_pid=$! +fenced_log "started fenced pid $fenced_pid in the background" + # # mount concurrently so that a quorum is present to elect the leader and # start a server. From 933fc687c392041e98f52128a3b450fec2947451 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 4 May 2021 11:18:28 -0700 Subject: [PATCH 12/22] omap remove_rid might not find entry Client recovery in the server doesn't add the omap rid for all the clients that it's waiting for. It only adds the rid as they connect. A client whose recovery timeout expires and is evicted will try to have its omap rid removed without being added. Today this triggers a warning and returns an error from a time when the omap rid lifecycle was more rigid. Now that it's being called by the server's reclaim_rid, along with a bunch of other functions that succeed if called for non-existant clients, let's have the omap remove_rid do the same. Signed-off-by: Zach Brown --- kmod/src/omap.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/kmod/src/omap.c b/kmod/src/omap.c index bb3ac8c4..37893914 100644 --- a/kmod/src/omap.c +++ b/kmod/src/omap.c @@ -485,6 +485,10 @@ static int remove_rid_from_reqs(struct omap_info *ominf, u64 rid, u64 *resp_rid, * response if it was the last rid waiting for a response. * * If this returns an error then the server will shut down. + * + * This can be called multiple times by different servers if there are + * errors reclaiming an evicted mount, so we allow asking to remove a + * rid that hasn't been added. */ int scoutfs_omap_remove_rid(struct super_block *sb, u64 rid) { @@ -495,21 +499,20 @@ int scoutfs_omap_remove_rid(struct super_block *sb, u64 rid) u64 resp_id = 0; int ret; - map = kmalloc(sizeof(struct scoutfs_open_ino_map), GFP_NOFS); - if (!map) { - ret = -ENOMEM; - goto out; - } - spin_lock(&ominf->lock); entry = find_rid(&ominf->rids, rid); if (entry) free_rid(&ominf->rids, entry); spin_unlock(&ominf->lock); - /* the server really shouldn't be removing a rid it never added */ - if (WARN_ON_ONCE(!entry)) { - ret = -ENOENT; + if (!entry) { + ret = 0; + goto out; + } + + map = kmalloc(sizeof(struct scoutfs_open_ino_map), GFP_NOFS); + if (!map) { + ret = -ENOMEM; goto out; } From aad2d3db59509c3d8c3b39be6aaffd446480519d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 4 May 2021 13:27:49 -0700 Subject: [PATCH 13/22] Add stage_tmpfile to .gitignore We missed adding this newly added binary to .gitignore. Signed-off-by: Zach Brown --- tests/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/.gitignore b/tests/.gitignore index b6f5b65c..d8268d17 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -4,3 +4,4 @@ src/dumb_setxattr src/handle_cat src/bulk_create_paths src/find_xattrs +src/stage_tmpfile From 76cef6fdfcb96bfeddcb257ae2c8817ddbb8ff2c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 7 May 2021 11:39:18 -0700 Subject: [PATCH 14/22] Let _recov_next_pending iterate over rids Currently the server's recovery timeout work synchronously reclaims resources for each client whose recovery timed out. scoutfs_recov_next_pending() can always return the head of the pending list because its caller will always remove it from the list as it iterates. As we move to real fencing the server will be creating fence requests for all the timed out clients concurrently. It will need to iterate over all the rids for clients in recovery. So we sort recovery's pending list by rid and change _recov_next_pending to return the next pending rid after a rid argument. This lets the server iterate over all the pending rids at once. Signed-off-by: Zach Brown --- kmod/src/lock_server.c | 2 +- kmod/src/omap.c | 2 +- kmod/src/recov.c | 49 +++++++++++++++++++++++++++++++----------- kmod/src/recov.h | 2 +- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index fb33fd80..09ce48d7 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -486,7 +486,7 @@ static int process_waiting_requests(struct super_block *sb, /* processing waits for all invalidation responses or recovery */ if (!list_empty(&snode->invalidated) || - scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_LOCKS) != 0) { + scoutfs_recov_next_pending(sb, 0, SCOUTFS_RECOV_LOCKS) != 0) { ret = 0; goto out; } diff --git a/kmod/src/omap.c b/kmod/src/omap.c index 37893914..3dfcbea8 100644 --- a/kmod/src/omap.c +++ b/kmod/src/omap.c @@ -619,7 +619,7 @@ static int handle_requests(struct super_block *sb) int ret; int err; - if (scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_GREETING)) + if (scoutfs_recov_next_pending(sb, 0, SCOUTFS_RECOV_GREETING)) return 0; ret = 0; diff --git a/kmod/src/recov.c b/kmod/src/recov.c index b0d894c2..3829760b 100644 --- a/kmod/src/recov.c +++ b/kmod/src/recov.c @@ -16,9 +16,11 @@ #include #include #include +#include #include "super.h" #include "recov.h" +#include "cmp.h" /* * There are a few server messages which can't be processed until they @@ -47,18 +49,41 @@ struct recov_pending { int which; }; -static struct recov_pending *find_pending(struct recov_info *recinf, u64 rid, int which) +static struct recov_pending *next_pending(struct recov_info *recinf, u64 rid, int which) { struct recov_pending *pend; list_for_each_entry(pend, &recinf->pending, head) { - if ((rid == 0 || pend->rid == rid) && (pend->which & which)) + if (pend->rid > rid && pend->which & which) return pend; } return NULL; } +static struct recov_pending *lookup_pending(struct recov_info *recinf, u64 rid, int which) +{ + struct recov_pending *pend; + + pend = next_pending(recinf, rid - 1, which); + if (pend && pend->rid == rid) + return pend; + + return NULL; +} + +/* + * We keep the pending list sorted by rid so that we can iterate over + * them. The list should be small and shouldn't be used often. + */ +static int cmp_pending_rid(void *priv, struct list_head *A, struct list_head *B) +{ + struct recov_pending *a = list_entry(A, struct recov_pending, head); + struct recov_pending *b = list_entry(B, struct recov_pending, head); + + return scoutfs_cmp_u64s(a->rid, b->rid); +} + /* * Record that we'll be waiting for a client to recover something. * _finished will eventually be called for every _prepare, either @@ -80,14 +105,15 @@ int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which) spin_lock(&recinf->lock); - pend = find_pending(recinf, rid, SCOUTFS_RECOV_ALL); + pend = lookup_pending(recinf, rid, SCOUTFS_RECOV_ALL); if (pend) { pend->which |= which; } else { swap(pend, alloc); pend->rid = rid; pend->which = which; - list_add(&pend->head, &recinf->pending); + list_add_tail(&pend->head, &recinf->pending); + list_sort(NULL, &recinf->pending, cmp_pending_rid); } spin_unlock(&recinf->lock); @@ -159,7 +185,7 @@ int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which) spin_lock(&recinf->lock); - pend = find_pending(recinf, rid, which); + pend = lookup_pending(recinf, rid, which); if (pend) { pend->which &= ~which; if (pend->which) { @@ -190,29 +216,28 @@ bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which) bool is_pending; spin_lock(&recinf->lock); - is_pending = find_pending(recinf, rid, which) != NULL; + is_pending = lookup_pending(recinf, rid, which) != NULL; spin_unlock(&recinf->lock); return is_pending; } /* - * Returns 0 if there are no rids waiting for the given state to be - * recovered. Returns the rid of a client still waiting if there are - * any, in no specified order. + * Return the next rid after the given rid of a client waiting for the + * given state to be recovered. Start with rid 0, returns 0 when there + * are no more clients waiting for recovery. * * This is inherently racey. Callers are responsible for resolving any * actions taken based on pending with the recovery finishing, perhaps * before we return. */ -u64 scoutfs_recov_next_pending(struct super_block *sb, int which) +u64 scoutfs_recov_next_pending(struct super_block *sb, u64 rid, int which) { DECLARE_RECOV_INFO(sb, recinf); struct recov_pending *pend; - u64 rid; spin_lock(&recinf->lock); - pend = find_pending(recinf, 0, which); + pend = next_pending(recinf, rid, which); rid = pend ? pend->rid : 0; spin_unlock(&recinf->lock); diff --git a/kmod/src/recov.h b/kmod/src/recov.h index cfdca30e..9be58b3a 100644 --- a/kmod/src/recov.h +++ b/kmod/src/recov.h @@ -14,7 +14,7 @@ int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_ unsigned int timeout_ms); int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which); bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which); -u64 scoutfs_recov_next_pending(struct super_block *sb, int which); +u64 scoutfs_recov_next_pending(struct super_block *sb, u64 rid, int which); void scoutfs_recov_shutdown(struct super_block *sb); int scoutfs_recov_setup(struct super_block *sb); From 07066690470aef1281188b0c68b5b792c91e8140 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 7 May 2021 12:19:40 -0700 Subject: [PATCH 15/22] Clean up quorum block read error messages The error messages from reading quorum blocks were confusing. The mark was being checked when the block had already seen an error, and we got multiple messages for some errors. This cleans it up a bit so we only get one error message for each error source and each message contains relevant context. Signed-off-by: Zach Brown --- kmod/src/quorum.c | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index 2a6d877e..fcba1b28 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -346,6 +346,7 @@ static int read_quorum_block(struct super_block *sb, u64 blkno, { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; + char msg[150]; __le32 crc; int ret; @@ -356,25 +357,37 @@ static int read_quorum_block(struct super_block *sb, u64 blkno, ret = scoutfs_block_read_sm(sb, sbi->meta_bdev, blkno, &blk->hdr, sizeof(*blk), &crc); + if (ret < 0) { + scoutfs_err(sb, "quorum block read error %d", ret); + goto out; + } /* detect invalid blocks */ - if (ret == 0 && - ((blk->hdr.crc != crc) || - (le32_to_cpu(blk->hdr.magic) != SCOUTFS_BLOCK_MAGIC_QUORUM) || - (blk->hdr.fsid != super->hdr.fsid) || - (le64_to_cpu(blk->hdr.blkno) != blkno))) { - scoutfs_inc_counter(sb, quorum_read_invalid_block); + if (blk->hdr.crc != crc) + snprintf(msg, sizeof(msg), "blk crc %08x != %08x", + le32_to_cpu(blk->hdr.crc), le32_to_cpu(crc)); + else if (le32_to_cpu(blk->hdr.magic) != SCOUTFS_BLOCK_MAGIC_QUORUM) + snprintf(msg, sizeof(msg), "blk magic %08x != %08x", + le32_to_cpu(blk->hdr.magic), SCOUTFS_BLOCK_MAGIC_QUORUM); + else if (blk->hdr.fsid != super->hdr.fsid) + snprintf(msg, sizeof(msg), "blk fsid %016llx != %016llx", + le64_to_cpu(blk->hdr.fsid), le64_to_cpu(super->hdr.fsid)); + else if (le64_to_cpu(blk->hdr.blkno) != blkno) + snprintf(msg, sizeof(msg), "blk blkno %llu != %llu", + le64_to_cpu(blk->hdr.blkno), blkno); + else if (mark && *mark != 0 && blk->random_write_mark != *mark) + snprintf(msg, sizeof(msg), "blk mark %016llx != %016llx, are multiple mounts configured with the same slot?", + le64_to_cpu(blk->random_write_mark), le64_to_cpu(*mark)); + else + msg[0] = '\0'; + + if (msg[0] != '\0') { + scoutfs_err(sb, "read invalid quorum block, %s", msg); ret = -EIO; + goto out; } - if (mark && *mark != 0 && blk->random_write_mark != *mark) { - scoutfs_err(sb, "read unexpected quorum block write mark, are multiple mounts configured with the same slot?"); - ret = -EIO; - } - - if (ret < 0) - scoutfs_err(sb, "quorum block read error %d", ret); - +out: return ret; } From a972e42fbaccad942bdbd6e180f8134b6c0c9ab6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 26 Feb 2021 10:26:33 -0800 Subject: [PATCH 16/22] Update dmesg filters for fencing and reclaim Add regexes for the messages that come from fencing and reclaiming resources from fenced mounts. Signed-off-by: Zach Brown --- tests/funcs/filter.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/funcs/filter.sh b/tests/funcs/filter.sh index 5c793be4..0d0850c3 100644 --- a/tests/funcs/filter.sh +++ b/tests/funcs/filter.sh @@ -62,5 +62,14 @@ t_filter_dmesg() # in debugging kernels we can slow things down a bit re="$re|hrtimer: interrupt took .*" + # fencing tests force unmounts and trigger timeouts + re="$re|scoutfs .* forcing unmount" + re="$re|scoutfs .* reconnect timed out" + re="$re|scoutfs .* recovery timeout expired" + re="$re|scoutfs .* fencing previous leader" + re="$re|scoutfs .* reclaimed resources" + re="$re|scoutfs .* quorum .* error" + re="$re|scoutfs .* error reading quorum block" + egrep -v "($re)" } From 877e30d60f1fd33c1c9b0d7886ae9b632aac6f97 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 7 May 2021 15:34:05 -0700 Subject: [PATCH 17/22] Add client address to mounted_client item Add the peername of the client's connected socket to its mounted_client item as it mounts. If the client doesn't recover then fencing can use the IP to find the host to fence. Signed-off-by: Zach Brown --- kmod/src/format.h | 2 ++ kmod/src/net.h | 10 ++++++++ kmod/src/server.c | 65 +++++++++++++++++++++++++++++++++++++---------- utils/src/print.c | 8 ++++-- 4 files changed, 70 insertions(+), 15 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index c2b938d7..300a1a46 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -327,7 +327,9 @@ struct scoutfs_alloc_root { #define SCOUTFS_ALLOC_OWNER_SRCH 3 struct scoutfs_mounted_client_btree_val { + union scoutfs_inet_addr addr; __u8 flags; + __u8 __pad[7]; }; #define SCOUTFS_MOUNTED_CLIENT_QUORUM (1 << 0) diff --git a/kmod/src/net.h b/kmod/src/net.h index e16ec524..847e9204 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -100,6 +100,16 @@ static inline void scoutfs_addr_to_sin(struct sockaddr_in *sin, sin->sin_port = cpu_to_be16(le16_to_cpu(addr->v4.port)); } +static inline void scoutfs_sin_to_addr(union scoutfs_inet_addr *addr, struct sockaddr_in *sin) +{ + BUG_ON(sin->sin_family != AF_INET); + + memset(addr, 0, sizeof(union scoutfs_inet_addr)); + addr->v4.family = cpu_to_le16(SCOUTFS_AF_IPV4); + addr->v4.addr = be32_to_le32(sin->sin_addr.s_addr); + addr->v4.port = be16_to_le16(sin->sin_port); +} + struct scoutfs_net_connection * scoutfs_net_alloc_conn(struct super_block *sb, scoutfs_net_notify_t notify_up, diff --git a/kmod/src/server.c b/kmod/src/server.c index cbb23088..872d5339 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -1453,14 +1453,19 @@ static void init_mounted_client_key(struct scoutfs_key *key, u64 rid) }; } +static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) +{ + return (iref->val_len != sizeof(struct scoutfs_mounted_client_btree_val)); +} + /* * Insert a new mounted client item for a client that is sending us a * greeting that hasn't yet seen a response. The greeting can be * retransmitted to a new server after the previous inserted the item so * it's acceptable to see -EEXIST. */ -static int insert_mounted_client(struct super_block *sb, u64 rid, - u64 gr_flags) +static int insert_mounted_client(struct super_block *sb, u64 rid, u64 gr_flags, + struct sockaddr_in *sin) { DECLARE_SERVER_INFO(sb, server); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; @@ -1469,6 +1474,7 @@ static int insert_mounted_client(struct super_block *sb, u64 rid, int ret; init_mounted_client_key(&key, rid); + scoutfs_sin_to_addr(&mcv.addr, sin); mcv.flags = 0; if (gr_flags & SCOUTFS_NET_GREETING_FLAG_QUORUM) mcv.flags |= SCOUTFS_MOUNTED_CLIENT_QUORUM; @@ -1484,6 +1490,34 @@ static int insert_mounted_client(struct super_block *sb, u64 rid, return ret; } +static int lookup_mounted_client_addr(struct super_block *sb, u64 rid, + union scoutfs_inet_addr *addr) +{ + DECLARE_SERVER_INFO(sb, server); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_mounted_client_btree_val *mcv; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + int ret; + + init_mounted_client_key(&key, rid); + + mutex_lock(&server->mounted_clients_mutex); + ret = scoutfs_btree_lookup(sb, &super->mounted_clients, &key, &iref); + if (ret == 0) { + if (invalid_mounted_client_item(&iref)) { + ret = -EIO; + } else { + mcv = iref.val; + *addr = mcv->addr; + } + scoutfs_btree_put_iref(&iref); + } + mutex_unlock(&server->mounted_clients_mutex); + + return ret; +} + /* * Remove the record of a mounted client. The record can already be * removed if we're processing a farewell on behalf of a client that @@ -1622,8 +1656,8 @@ static int server_greeting(struct super_block *sb, if (ret < 0) goto send_err; - ret = insert_mounted_client(sb, le64_to_cpu(gr->rid), - le64_to_cpu(gr->flags)); + ret = insert_mounted_client(sb, le64_to_cpu(gr->rid), le64_to_cpu(gr->flags), + &conn->peername); ret = scoutfs_server_apply_commit(sb, ret); queue_work(server->wq, &server->farewell_work); @@ -1680,11 +1714,6 @@ struct farewell_request { u64 rid; }; -static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) -{ - return (iref->val_len != - sizeof(struct scoutfs_mounted_client_btree_val)); -} /* * Reclaim all the resources for a mount which has gone away. It's sent @@ -2040,20 +2069,30 @@ static void fence_pending_recov_worker(struct work_struct *work) struct server_info *server = container_of(work, struct server_info, fence_pending_recov_work); struct super_block *sb = server->sb; + union scoutfs_inet_addr addr; u64 rid = 0; - int ret; + int ret = 0; while ((rid = scoutfs_recov_next_pending(sb, rid, SCOUTFS_RECOV_ALL)) > 0) { scoutfs_err(sb, "%lu ms recovery timeout expired for client rid %016llx, fencing", SERVER_RECOV_TIMEOUT_MS, rid); - ret = scoutfs_fence_start(sb, rid, 0, SCOUTFS_FENCE_CLIENT_RECOVERY); - if (ret) { + ret = lookup_mounted_client_addr(sb, rid, &addr); + if (ret < 0) { + scoutfs_err(sb, "client rid addr lookup err %d, shutting down server", ret); + break; + } + + ret = scoutfs_fence_start(sb, rid, le32_to_be32(addr.v4.addr), + SCOUTFS_FENCE_CLIENT_RECOVERY); + if (ret < 0) { scoutfs_err(sb, "fence returned err %d, shutting down server", ret); - scoutfs_server_abort(sb); break; } } + + if (ret < 0) + scoutfs_server_abort(sb); } static void recovery_timeout(struct super_block *sb) diff --git a/utils/src/print.c b/utils/src/print.c index 9ddc3e4b..e61361bd 100644 --- a/utils/src/print.c +++ b/utils/src/print.c @@ -372,9 +372,13 @@ static int print_mounted_client_entry(struct scoutfs_key *key, void *val, unsigned val_len, void *arg) { struct scoutfs_mounted_client_btree_val *mcv = val; + struct in_addr in; - printf(" rid %016llx flags 0x%x\n", - le64_to_cpu(key->skmc_rid), mcv->flags); + memset(&in, 0, sizeof(in)); + in.s_addr = htonl(le32_to_cpu(mcv->addr.v4.addr)); + + printf(" rid %016llx ipv4_addr %s flags 0x%x\n", + le64_to_cpu(key->skmc_rid), inet_ntoa(in), mcv->flags); return 0; } From 1e460e5cb00b9d6d45d6c4301602568580a50cfe Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 20 May 2021 11:58:49 -0700 Subject: [PATCH 18/22] Add scoutfs-fenced and its run scripts to spec Install the scoutfs-fenced daemon and its run scripts in the rpm spec file. Signed-off-by: Zach Brown --- utils/scoutfs-utils.spec.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/scoutfs-utils.spec.in b/utils/scoutfs-utils.spec.in index 42bb86c5..859419da 100644 --- a/utils/scoutfs-utils.spec.in +++ b/utils/scoutfs-utils.spec.in @@ -54,12 +54,15 @@ cp man/*.8.gz $RPM_BUILD_ROOT%{_mandir}/man8/. install -m 755 -D src/scoutfs $RPM_BUILD_ROOT%{_sbindir}/scoutfs install -m 644 -D src/ioctl.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/ioctl.h install -m 644 -D src/format.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/format.h +install -m 755 -D fenced/scoutfs-fenced $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/scoutfs-fenced +install -m 755 -D fenced/local-force-unmount $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/local-force-unmount %files %defattr(644,root,root,755) %{_mandir}/man*/scoutfs*.gz %defattr(755,root,root,755) %{_sbindir}/scoutfs +%{_libexecdir}/scoutfs-fenced %files -n scoutfs-devel %defattr(644,root,root,755) From 1199bac91d8e44d24fa32aeb870e378bbdb841db Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 25 May 2021 12:55:19 -0700 Subject: [PATCH 19/22] Fix quorum server shutdown If the server shuts down it calls into quorum to tell it that the server has exited. This stops quorum from sending heartbeats that suppress other leader elections. The function that did this got the logic wrong. It was setting the bit instead of clearing it, having been initially written to set a bit when the server exited. Signed-off-by: Zach Brown --- kmod/src/quorum.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index fcba1b28..a69e5270 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -813,15 +813,15 @@ out: } /* - * Set a flag for the quorum work's next iteration to indicate that the - * server has shutdown and that it should step down as leader, update - * quorum blocks, and stop sending heartbeats. + * Clear the server flag for the quorum work's next iteration to + * indicate that the server has shutdown and that it should step down as + * leader, update quorum blocks, and stop sending heartbeats. */ void scoutfs_quorum_server_shutdown(struct super_block *sb) { DECLARE_QUORUM_INFO(sb, qinf); - set_bit(QINF_FLAG_SERVER, &qinf->flags); + clear_bit(QINF_FLAG_SERVER, &qinf->flags); } /* From bdc0282fa784137b234c8e9d9373d06850cdfd38 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 26 May 2021 13:39:46 -0700 Subject: [PATCH 20/22] Describe fencing in the scoutfs.5 man page Signed-off-by: Zach Brown --- utils/man/scoutfs.5 | 79 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/utils/man/scoutfs.5 b/utils/man/scoutfs.5 index c489a5d5..150eb1fb 100644 --- a/utils/man/scoutfs.5 +++ b/utils/man/scoutfs.5 @@ -1,6 +1,6 @@ .TH scoutfs 5 .SH NAME -scoutfs \- overview and mount options for the scoutfs filesystem +scoutfs \- high level overview of the scoutfs filesystem .SH DESCRIPTION A scoutfs filesystem is stored on two block devices. Multiple mounts of the filesystem are supported between hosts that share access to the @@ -68,7 +68,82 @@ in that zone. When the mount is not in a zone, or its zone has no more free extents, the server will try and find free extents in a zone that no other mount currently occupies. The result is to try and produce write streams where only one mount is writing into each zone. -.SH FURTHER READING +.SH FENCING +.B scoutfs +mounts coordinate exclusive access to shared resources through +comminication with the mount that was elected leader. +A mount can malfunction and stop participating at which point it needs +to be safely isolated ("fenced off") from shared resources before other mounts can +have their turn at exclusive access. +.sp +Only the elected leader can fence mounts. As the leader decides that a +mount must be fenced, typically by timeouts expiring without +comminication from the mount, it creates a fence request. Fence +requests are visible as directories in the leader mount's sysfs +directory. The fence request directory is named for the RID of the +mount being fenced. The directory contains the following files: + +.RS +.TP +.B elapsec_secs +Reading this file gives the number of seconds that have passed since +this fence request was created. +.TP +.B error +This file contains 0 when the fence request is created. Userspace +fencing agents write 1 into this file if they are unable to fence the +mount. The volume can not make progress until the mount is fenced so +this will cause the server to stop and another mount will be elected +leader. +.TP +.B fenced +This file contains 0 when the fence request is created. Userspace +fencing agents write 1 into this file once the mount has been fenced. +.TP +.B ipv4_addr +This file contains the dotted quad IPv4 peer address of the last +connected socket from the mount. Userspace fencing agents can use this +to find the host that contains the mount. +.TP +.B reason +This file contains a text string that indicates the reason that the +mount is being fenced: + +.B client_recovery +- During startup the server found persistent items recording the presence +of a mount that didn't reconnect to the server in time. +.sp +.B client_reconnect +- A mount disconnected from the server and didn't reconnect in time. +.sp +.B quorum_block_leader +- As a leader was elected it read persistent blocks that indicated that +a previous leader had not shut down and cleared their quorum block. +.TP +.B rid +This file contains the hex string of the RID of the mount to be fenced. +.RE + +The request directories enable userspace processes to gather the +information to find the host with the mount to fence, isolate the mount +by whatever means are appropriate (f.e. cut off network and storage +communication, force unmount the mount, isolate storage fabric ports, +reboot the host) and write to the +.I fenced +file. +.sp +Once the +.I fenced +file is written to the server reclaims the resources +associated with the fenced mount and resumes normal operations. +.sp +If the +.I error +file is written to then the server cannot make forward progress and +shuts down. The request can similarly enter an errored state if enough +time passes before userspace completes the request. + +.SH CORRUPTION DETECTION A .B scoutfs filesystem can detect corruption at runtime. A catalog of kernel log From 76076011a2bd9a9285c894dc832d14abd855133b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 26 May 2021 13:59:28 -0700 Subject: [PATCH 21/22] Add scoutfs-fenced man page Signed-off-by: Zach Brown --- utils/man/scoutfs-fenced.8 | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 utils/man/scoutfs-fenced.8 diff --git a/utils/man/scoutfs-fenced.8 b/utils/man/scoutfs-fenced.8 new file mode 100644 index 00000000..4be5348e --- /dev/null +++ b/utils/man/scoutfs-fenced.8 @@ -0,0 +1,66 @@ +.TH scoutfs-fenced 8 +.SH NAME +scoutfs-fenced \- scoutfs fence request monitoring and dispatch daemon +.SH DESCRIPTION +The +.B scoutfs-fenced +daemon runs on hosts with mounts that are configured as quorum members +and could create fence requests. It watches sysfs directories of +mounted scoutfs volumes for the directories store requests +to fence a mount. + +.SH ENVIRONMENT +scoutfs-fenced reads the +.I SCOUTFS_FENCED_CONFIG_FILE +environment variable for the path to the config file that contains its +configuration. The file must be readable and is sourced as a bash +script and is expected to set the following configuration variables. + +.SH CONFIGURATION + +.TP +.B SCOUTFS_FENCED_DELAY +The number of seconds to wait beteween checking for fence request +directories in the sysfs directories of all mounts on the host. + +.TP +.B SCOUTFS_FENCED_RUN +The path to the command to execute for each fence request. The file at +the path must be executable. + +.TP +.B SCOUTFS_FENCED_RUN_ARGS +The arguments that are unconditionally passed through to the run +command. + +.SH DAEMONIZING AND LOGGING + +scoutfs-fenced runs in the foreground and writes to stderr and stdout. +Disconnecting it from parents and redirecting its output are the +responsibility of the host environment. + +.SH RUN COMMAND INTERFACE + +scoutfs-fenced sets enviroment variables for the run command with +information about the mount that must be fenced: + +.TP +.B SCOUTFS_FENCED_REQ_RID +The RID of the mount to be fenced. +.TP +.B SCOUTFS_FENCED_REQ_IP +The dotted quad IPv4 address of the last connection from the mount. + +.RE +The return status of the run command indicates if the mount was +fenced, or not. If the mount was successfully fenced then the command +should return a 0 success status. If the run command returns a non-zero +failure status then the request will be set as errored and the server +will shut down. The next server that starts will create another fence +request for the mount. + +.SH SEE ALSO +.BR scoutfs (5), + +.SH AUTHORS +Zach Brown From 38a4a56741aec33b5445bccf37f9e1c18c8c0b1e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 31 May 2021 10:12:45 -0700 Subject: [PATCH 22/22] Stop writing to other quorum slot blocks The core quorum work loop assumes that it has exclusive access to its slot's quorum block. It uniquely marks blocks it writes and verifies the marks on read to discover if another mount has written to its slot under the assumption that this must be a configuration error that put two mounts in the same slot. But the design of the leader bit in the block violates the invariant that only a slot will write to its block. As the server comes up and fences previous leaders it writes to their block to clear their leader bit. The final hole in the design is that because we're fencing mounts, not slots, each slot can have two mounts in play. An active mount can be using the slot and there can still be a persistent record of a previous mount in the slot that crashed that needs to be fenced. All this comes together to have the server fence an old mount in a slot while a new mount is coming up. The new mount sees the mark change and freaks out and stops participating in quorum. The fix is to rework the quorum blocks so that each slot only writes to its own block. Instead of the server writing to each fenced mount's slot, it writes a fence event to its block once all previous mounts have been fenced. We add a bit of bookkeeping so that the server can discover when all block leader fence operations have completed. Each event gets its own term so we can compare events to discover live servers. We get rid of the write marks and instead have an event that is written as a quorum agent starts up and is then checked on every read to make sure it still matches. Signed-off-by: Zach Brown --- kmod/src/fence.c | 18 +++ kmod/src/fence.h | 1 + kmod/src/format.h | 26 ++-- kmod/src/quorum.c | 332 ++++++++++++++++++++---------------------- kmod/src/quorum.h | 6 +- kmod/src/server.c | 20 ++- tests/funcs/filter.sh | 1 + utils/src/print.c | 32 ++-- 8 files changed, 225 insertions(+), 211 deletions(-) diff --git a/kmod/src/fence.c b/kmod/src/fence.c index c7bb5aa4..1b039b2e 100644 --- a/kmod/src/fence.c +++ b/kmod/src/fence.c @@ -300,6 +300,24 @@ int scoutfs_fence_next(struct super_block *sb, u64 *rid, int *reason, bool *erro return ret; } +int scoutfs_fence_reason_pending(struct super_block *sb, int reason) +{ + DECLARE_FENCE_INFO(sb, fi); + struct pending_fence *fence; + bool pending = false; + + spin_lock(&fi->lock); + list_for_each_entry(fence, &fi->list, entry) { + if (fence->reason == reason) { + pending = true; + break; + } + } + spin_unlock(&fi->lock); + + return pending; +} + int scoutfs_fence_free(struct super_block *sb, u64 rid) { DECLARE_FENCE_INFO(sb, fi); diff --git a/kmod/src/fence.h b/kmod/src/fence.h index 96263d7d..f9139001 100644 --- a/kmod/src/fence.h +++ b/kmod/src/fence.h @@ -9,6 +9,7 @@ enum { int scoutfs_fence_start(struct super_block *sb, u64 rid, __be32 ipv4_addr, int reason); int scoutfs_fence_next(struct super_block *sb, u64 *rid, int *reason, bool *error); +int scoutfs_fence_reason_pending(struct super_block *sb, int reason); int scoutfs_fence_free(struct super_block *sb, u64 rid); int scoutfs_fence_wait_fenced(struct super_block *sb, long timeout_jiffies); diff --git a/kmod/src/format.h b/kmod/src/format.h index 300a1a46..924a1842 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -625,18 +625,24 @@ struct scoutfs_quorum_config { } slots[SCOUTFS_QUORUM_MAX_SLOTS]; }; -struct scoutfs_quorum_block { - struct scoutfs_block_header hdr; - __le64 term; - __le64 random_write_mark; - __le64 flags; - struct scoutfs_quorum_block_event { - __le64 rid; - struct scoutfs_timespec ts; - } write, update_term, set_leader, clear_leader, fenced; +enum { + SCOUTFS_QUORUM_EVENT_BEGIN, /* quorum service starting up */ + SCOUTFS_QUORUM_EVENT_TERM, /* updated persistent term */ + SCOUTFS_QUORUM_EVENT_ELECT, /* won election */ + SCOUTFS_QUORUM_EVENT_FENCE, /* server fenced others */ + SCOUTFS_QUORUM_EVENT_STOP, /* server stopped */ + SCOUTFS_QUORUM_EVENT_END, /* quorum service shutting down */ + SCOUTFS_QUORUM_EVENT_NR, }; -#define SCOUTFS_QUORUM_BLOCK_LEADER (1 << 0) +struct scoutfs_quorum_block { + struct scoutfs_block_header hdr; + struct scoutfs_quorum_block_event { + __le64 rid; + __le64 term; + struct scoutfs_timespec ts; + } events[SCOUTFS_QUORUM_EVENT_NR]; +}; /* * Tunable options that apply to the entire system. They can be set in diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index a69e5270..8017642f 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -61,10 +61,9 @@ * running (maybe they've deadlocked, or lost network communications). * In addition to a configuration slot in the super block, each quorum * member also has a known block location that represents their slot. - * They set a flag in their block indicating that they've been elected - * leader, then read slots for all the other blocks looking for - * previously active leaders to fence. After that it can start the - * server. + * The block contains an array of events which are updated during the life + * time of the quorum agent. The elected leader set its elected event + * and can then start the server. * * It's critical to raft elections that a participant's term not go * backwards in time so each mount also uses its quorum block to store @@ -335,17 +334,17 @@ static int recv_msg(struct super_block *sb, struct quorum_host_msg *msg, } /* - * The caller can provide a mark that they're using to track their - * written blocks. It's updated as they write the block and we can - * compare it with what we read to see if there have been unexpected - * intervening writes to the block -- the caller is supposed to have - * exclusive access to the block (or was fenced). + * Read and verify block fields before giving it to the caller. We + * should have exclusive write access to the block. We know that + * something has gone horribly wrong if we don't see our rid in the + * begin event after we've written it as we started up. */ -static int read_quorum_block(struct super_block *sb, u64 blkno, - struct scoutfs_quorum_block *blk, __le64 *mark) +static int read_quorum_block(struct super_block *sb, u64 blkno, struct scoutfs_quorum_block *blk, + bool check_rid) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; + const u64 rid = sbi->rid; char msg[150]; __le32 crc; int ret; @@ -375,9 +374,9 @@ static int read_quorum_block(struct super_block *sb, u64 blkno, else if (le64_to_cpu(blk->hdr.blkno) != blkno) snprintf(msg, sizeof(msg), "blk blkno %llu != %llu", le64_to_cpu(blk->hdr.blkno), blkno); - else if (mark && *mark != 0 && blk->random_write_mark != *mark) - snprintf(msg, sizeof(msg), "blk mark %016llx != %016llx, are multiple mounts configured with the same slot?", - le64_to_cpu(blk->random_write_mark), le64_to_cpu(*mark)); + else if (check_rid && le64_to_cpu(blk->events[SCOUTFS_QUORUM_EVENT_BEGIN].rid) != rid) + snprintf(msg, sizeof(msg), "quorum block begin rid %016llx != our rid %016llx, are multiple mounts configured with this slot?", + le64_to_cpu(blk->events[SCOUTFS_QUORUM_EVENT_BEGIN].rid), rid); else msg[0] = '\0'; @@ -391,184 +390,159 @@ out: return ret; } -static void set_quorum_block_event(struct super_block *sb, - struct scoutfs_quorum_block *blk, - struct scoutfs_quorum_block_event *ev) +static void set_quorum_block_event(struct super_block *sb, struct scoutfs_quorum_block *blk, + int event, u64 term) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_quorum_block_event *ev; struct timespec64 ts; + if (WARN_ON_ONCE(event < 0 || event >= SCOUTFS_QUORUM_EVENT_NR)) + return; + getnstimeofday64(&ts); + ev = &blk->events[event]; ev->rid = cpu_to_le64(sbi->rid); + ev->term = cpu_to_le64(term); ev->ts.sec = cpu_to_le64(ts.tv_sec); ev->ts.nsec = cpu_to_le32(ts.tv_nsec); } -/* - * Every time we write a block we update the write stamp and random - * write mark so readers can see our write. - */ -static int write_quorum_block(struct super_block *sb, u64 blkno, - struct scoutfs_quorum_block *blk, __le64 *mark) +static int write_quorum_block(struct super_block *sb, u64 blkno, struct scoutfs_quorum_block *blk) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - int ret; if (WARN_ON_ONCE(blkno < SCOUTFS_QUORUM_BLKNO) || WARN_ON_ONCE(blkno >= (SCOUTFS_QUORUM_BLKNO + SCOUTFS_QUORUM_BLOCKS))) return -EINVAL; - do { - get_random_bytes(&blk->random_write_mark, - sizeof(blk->random_write_mark)); - } while (blk->random_write_mark == 0); - - if (mark) - *mark = blk->random_write_mark; - - set_quorum_block_event(sb, blk, &blk->write); - - ret = scoutfs_block_write_sm(sb, sbi->meta_bdev, blkno, - &blk->hdr, sizeof(*blk)); - if (ret < 0) - scoutfs_err(sb, "quorum block write error %d", ret); - - return ret; + return scoutfs_block_write_sm(sb, sbi->meta_bdev, blkno, &blk->hdr, sizeof(*blk)); } /* - * Read the caller's slot's current quorum block, make a change, and - * write it back out. If the caller provides a mark it can cause read - * errors if we read a mark that doesn't match the last mark that the - * caller wrote. + * Read the caller's slot's quorum block, make a change, and write it + * back out. */ -static int update_quorum_block(struct super_block *sb, u64 blkno, - __le64 *mark, int role, u64 term) +static int update_quorum_block(struct super_block *sb, int event, u64 term, bool check_rid) { + struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + u64 blkno = SCOUTFS_QUORUM_BLKNO + opts->quorum_slot_nr; struct scoutfs_quorum_block blk; - u64 flags; - u64 bits; - u64 set; int ret; - ret = read_quorum_block(sb, blkno, &blk, mark); + ret = read_quorum_block(sb, blkno, &blk, check_rid); if (ret == 0) { - if (blk.term != cpu_to_le64(term)) { - blk.term = cpu_to_le64(term); - set_quorum_block_event(sb, &blk, &blk.update_term); - } - - flags = le64_to_cpu(blk.flags); - bits = SCOUTFS_QUORUM_BLOCK_LEADER; - set = role == LEADER ? SCOUTFS_QUORUM_BLOCK_LEADER : 0; - if ((flags & bits) != set) - set_quorum_block_event(sb, &blk, - set ? &blk.set_leader : - &blk.clear_leader); - blk.flags = cpu_to_le64((flags & ~bits) | set); - - ret = write_quorum_block(sb, blkno, &blk, mark); - } - - return ret; -} - -/* - * The calling server had fenced previous leaders before starting up, - * now that it's up it has reclaimed their resources and can clear their - * leader flags. - */ -int scoutfs_quorum_clear_rid_leader(struct super_block *sb, u64 rid) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct mount_options *opts = &sbi->opts; - struct scoutfs_quorum_block blk; - int ret = 0; - u64 blkno; - int i; - - for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - if (i == opts->quorum_slot_nr || !quorum_slot_present(super, i)) - continue; - - blkno = SCOUTFS_QUORUM_BLKNO + i; - ret = read_quorum_block(sb, blkno, &blk, NULL); + set_quorum_block_event(sb, &blk, event, term); + ret = write_quorum_block(sb, blkno, &blk); if (ret < 0) - break; - - if (le64_to_cpu(blk.set_leader.rid) == rid) { - blk.flags &= ~cpu_to_le64(SCOUTFS_QUORUM_BLOCK_LEADER); - set_quorum_block_event(sb, &blk, &blk.fenced); - - ret = write_quorum_block(sb, blkno, &blk, NULL); - break; - } + scoutfs_err(sb, "error %d reading quorum block %llu to update event %d term %llu", + ret, blkno, event, term); + } else { + scoutfs_err(sb, "error %d writing quorum block %llu after updating event %d term %llu", + ret, blkno, event, term); } - if (ret < 0) - scoutfs_err(sb, "error %d clearing leader block for rid %016llx", ret, rid); - return ret; } /* - * The calling server has been elected, had its block updated, and has - * started running but can't yet assume that it has exclusive access to - * the metadata device. We read all the quorum blocks looking for - * previously elected leaders to fence so that we're the only leader - * running. + * The calling server has fenced previous leaders and reclaimed their + * resources. We can now update our fence event with a greater term to + * stop future leaders from doing the same. + */ +int scoutfs_quorum_fence_complete(struct super_block *sb, u64 term) +{ + return update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_FENCE, term, true); +} + +/* + * The calling server has been elected and has started running but can't + * yet assume that it has exclusive access to the metadata device. We + * read all the quorum blocks looking for previously elected leaders to + * fence so that we're the only leader running. * - * We only wait for the previous leaders to be fenced. We don't clear - * the leader bits because the server is going to reclaim their - * resources once its up and running. Only then will the leader bits be - * cleared. + * We're relying on the invariant that there can't be two mounts running + * with the same slot nr at the same time. With this constraint there + * can be at most two previous leaders per slot that need to be fenced: + * a persistent record of an old mount on the slot, and an active mount. + * + * If we start fence requests then we only wait for them to complete + * before returning. The server will reclaim their resources once it is + * up and running and will call us to update the fence event. If we + * don't start fence requests then we update the fence event + * immediately, the server has nothing more to do. * * Quorum will be sending heartbeats while we wait for fencing. That * keeps us from being fenced while we allow userspace fencing to take a * reasonably long time. We still want to timeout eventually. */ -int scoutfs_quorum_fence_leader_blocks(struct super_block *sb, u64 term) +int scoutfs_quorum_fence_leaders(struct super_block *sb, u64 term) { +#define NR_OLD 2 + struct scoutfs_quorum_block_event old[SCOUTFS_QUORUM_MAX_SLOTS][NR_OLD] = {{{0,}}}; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct mount_options *opts = &sbi->opts; struct scoutfs_quorum_block blk; struct sockaddr_in sin; + const u64 rid = sbi->rid; bool fence_started = false; - u64 blkno; + u64 fenced = 0; + __le64 fence_rid; int ret = 0; int err; int i; + int j; BUILD_BUG_ON(SCOUTFS_QUORUM_BLOCKS < SCOUTFS_QUORUM_MAX_SLOTS); for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - if (i == opts->quorum_slot_nr || !quorum_slot_present(super, i)) + if (!quorum_slot_present(super, i)) continue; - blkno = SCOUTFS_QUORUM_BLKNO + i; - ret = read_quorum_block(sb, blkno, &blk, NULL); + ret = read_quorum_block(sb, SCOUTFS_QUORUM_BLKNO + i, &blk, false); if (ret < 0) goto out; - if (!(le64_to_cpu(blk.flags) & SCOUTFS_QUORUM_BLOCK_LEADER) || - le64_to_cpu(blk.term) > term) - continue; + /* elected leader still running */ + if (le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_ELECT].term) > + le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_STOP].term)) + old[i][0] = blk.events[SCOUTFS_QUORUM_EVENT_ELECT]; - scoutfs_inc_counter(sb, quorum_fence_leader); - scoutfs_quorum_slot_sin(super, i, &sin); + /* persistent record of previous server before elected */ + if ((le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_FENCE].term) > + le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_STOP].term)) && + (le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_FENCE].term) < + le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_ELECT].term))) + old[i][1] = blk.events[SCOUTFS_QUORUM_EVENT_FENCE]; - scoutfs_info(sb, "fencing previous leader "SCSBF" in slot %u with address "SIN_FMT, - SCSB_LEFR_ARGS(super->hdr.fsid, blk.set_leader.rid), i, SIN_ARG(&sin)); - ret = scoutfs_fence_start(sb, le64_to_cpu(blk.set_leader.rid), sin.sin_addr.s_addr, - SCOUTFS_FENCE_QUORUM_BLOCK_LEADER); - if (ret < 0) - goto out; - fence_started = true; + /* find greatest term that has fenced everything before it */ + fenced = max(fenced, le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_FENCE].term)); + } + /* now actually fence any old leaders which haven't been fenced yet */ + for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { + for (j = 0; j < NR_OLD; j++) { + if (le64_to_cpu(old[i][j].term) == 0 || /* uninitialized */ + le64_to_cpu(old[i][j].term) < fenced || /* already fenced */ + le64_to_cpu(old[i][j].term) > term || /* newer than us */ + le64_to_cpu(old[i][j].rid) == rid) /* us */ + continue; + + scoutfs_inc_counter(sb, quorum_fence_leader); + scoutfs_quorum_slot_sin(super, i, &sin); + fence_rid = old[i][j].rid; + + scoutfs_info(sb, "fencing previous leader "SCSBF" at term %llu in slot %u with address "SIN_FMT, + SCSB_LEFR_ARGS(super->hdr.fsid, fence_rid), + le64_to_cpu(old[i][j].term), i, SIN_ARG(&sin)); + ret = scoutfs_fence_start(sb, le64_to_cpu(fence_rid), sin.sin_addr.s_addr, + SCOUTFS_FENCE_QUORUM_BLOCK_LEADER); + if (ret < 0) + goto out; + fence_started = true; + } } out: @@ -576,9 +550,14 @@ out: err = scoutfs_fence_wait_fenced(sb, msecs_to_jiffies(SCOUTFS_QUORUM_FENCE_TO_MS)); if (ret == 0) ret = err; + } else { + err = scoutfs_quorum_fence_complete(sb, term); + if (ret == 0) + ret = err; } + if (ret < 0) { - scoutfs_err(sb, "error %d fencing leader blocks", ret); + scoutfs_err(sb, "error %d attempting to find and fence previous leaders", ret); scoutfs_inc_counter(sb, quorum_fence_error); } @@ -601,23 +580,22 @@ static void scoutfs_quorum_worker(struct work_struct *work) struct sockaddr_in unused; struct quorum_host_msg msg; struct quorum_status qst; - __le64 mark; u64 blkno; int ret; + int err; /* recording votes from slots as native single word bitmap */ BUILD_BUG_ON(SCOUTFS_QUORUM_MAX_SLOTS > BITS_PER_LONG); /* get our starting term from our persistent block */ - mark = 0; blkno = SCOUTFS_QUORUM_BLKNO + opts->quorum_slot_nr; - ret = read_quorum_block(sb, blkno, &blk, &mark); + ret = read_quorum_block(sb, blkno, &blk, false); if (ret < 0) goto out; /* start out as a follower */ qst.role = FOLLOWER; - qst.term = le64_to_cpu(blk.term); + qst.term = le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_TERM].term); qst.vote_for = -1; qst.vote_bits = 0; @@ -627,6 +605,11 @@ static void scoutfs_quorum_worker(struct work_struct *work) else qst.timeout = election_timeout(); + /* record that we're up and running, readers check that it isn't updated */ + ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_BEGIN, qst.term, false); + if (ret < 0) + goto out; + while (!qinf->shutdown) { ret = recv_msg(sb, &msg, qst.timeout); @@ -657,11 +640,6 @@ static void scoutfs_quorum_worker(struct work_struct *work) send_msg_others(sb, SCOUTFS_QUORUM_MSG_RESIGNATION, qst.term); scoutfs_inc_counter(sb, quorum_send_resignation); - - ret = update_quorum_block(sb, blkno, &mark, - qst.role, qst.term); - if (ret < 0) - goto out; } spin_lock(&qinf->show_lock); @@ -692,8 +670,7 @@ static void scoutfs_quorum_worker(struct work_struct *work) qst.timeout = election_timeout(); /* store our increased term */ - ret = update_quorum_block(sb, blkno, &mark, - qst.role, qst.term); + ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_TERM, qst.term, true); if (ret < 0) goto out; } @@ -710,6 +687,11 @@ static void scoutfs_quorum_worker(struct work_struct *work) qst.term); qst.timeout = election_timeout(); scoutfs_inc_counter(sb, quorum_send_request); + + /* store our increased term */ + ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_TERM, qst.term, true); + if (ret < 0) + goto out; } /* candidates count votes in their term */ @@ -738,8 +720,8 @@ static void scoutfs_quorum_worker(struct work_struct *work) qst.term); qst.timeout = heartbeat_interval(); - /* set our leader flag before starting server */ - ret = update_quorum_block(sb, blkno, &mark, qst.role, qst.term); + /* record that we've been elected before starting up server */ + ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_ELECT, qst.term, true); if (ret < 0) goto out; @@ -750,8 +732,13 @@ static void scoutfs_quorum_worker(struct work_struct *work) ret = scoutfs_server_start(sb, qst.term); if (ret < 0) { - scoutfs_err(sb, "server startup failed with %d", - ret); + clear_bit(QINF_FLAG_SERVER, &qinf->flags); + scoutfs_err(sb, "server startup failed with %d", ret); + /* store our increased term */ + err = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_STOP, qst.term, + true); + if (err < 0 && ret == 0) + ret = err; goto out; } } @@ -798,13 +785,8 @@ static void scoutfs_quorum_worker(struct work_struct *work) qst.term); } - /* always try to clear leader block as we stop to avoid fencing */ - if (qst.role == LEADER) { - ret = update_quorum_block(sb, blkno, &mark, - FOLLOWER, qst.term); - if (ret < 0) - goto out; - } + /* informational event that we're shutting down, nothing relies on it */ + update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_END, qst.term, true); out: if (ret < 0) { scoutfs_err(sb, "quorum service saw error %d, shutting down. Cluster will be degraded until this slot is remounted to restart the quorum service", @@ -813,58 +795,60 @@ out: } /* - * Clear the server flag for the quorum work's next iteration to - * indicate that the server has shutdown and that it should step down as - * leader, update quorum blocks, and stop sending heartbeats. + * The calling server has shutdown and is no longer using shared + * resources. Clear the bit so that we stop sending heartbeats and + * allow the next server to be elected. Update the stop event so that + * it won't be considered available by clients or fenced by the next + * leader. */ -void scoutfs_quorum_server_shutdown(struct super_block *sb) +void scoutfs_quorum_server_shutdown(struct super_block *sb, u64 term) { DECLARE_QUORUM_INFO(sb, qinf); clear_bit(QINF_FLAG_SERVER, &qinf->flags); + update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_STOP, term, true); } /* * Clients read quorum blocks looking for the leader with a server whose * address it can try and connect to. * - * There can be multiple running servers if a client checks before a - * server has had a chance to fence any old servers. We try to use the - * block with the most recent timestamp. If we get it wrong the - * connection will timeout and the client will try again, presumably - * finding a single server block. + * There can be records of multiple previous elected leaders if the + * current server hasn't yet fenced any old servers. We use the elected + * leader with the greatest elected term. If we get it wrong the + * connection will timeout and the client will try again. */ int scoutfs_quorum_server_sin(struct super_block *sb, struct sockaddr_in *sin) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_quorum_block blk; - struct timespec64 recent = {0,}; - struct timespec64 ts; - int ret; + u64 elect_term; + u64 term = 0; + int ret = 0; int i; for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - ret = read_quorum_block(sb, SCOUTFS_QUORUM_BLKNO + i, &blk, - NULL); + if (!quorum_slot_present(super, i)) + continue; + + ret = read_quorum_block(sb, SCOUTFS_QUORUM_BLKNO + i, &blk, false); if (ret < 0) { scoutfs_err(sb, "error reading quorum block nr %u: %d", i, ret); goto out; } - ts.tv_sec = le64_to_cpu(blk.set_leader.ts.sec); - ts.tv_nsec = le32_to_cpu(blk.set_leader.ts.nsec); - - if ((le64_to_cpu(blk.flags) & SCOUTFS_QUORUM_BLOCK_LEADER) && - (timespec64_to_ns(&ts) > timespec64_to_ns(&recent))) { - recent = ts; + elect_term = le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_ELECT].term); + if (elect_term > term && + elect_term > le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_STOP].term)) { + term = elect_term; scoutfs_quorum_slot_sin(super, i, sin); continue; } } - if (timespec64_to_ns(&recent) == 0) + if (term == 0) ret = -ENOENT; out: diff --git a/kmod/src/quorum.h b/kmod/src/quorum.h index 3f41fd65..1c2b6315 100644 --- a/kmod/src/quorum.h +++ b/kmod/src/quorum.h @@ -2,14 +2,14 @@ #define _SCOUTFS_QUORUM_H_ int scoutfs_quorum_server_sin(struct super_block *sb, struct sockaddr_in *sin); -void scoutfs_quorum_server_shutdown(struct super_block *sb); +void scoutfs_quorum_server_shutdown(struct super_block *sb, u64 term); u8 scoutfs_quorum_votes_needed(struct super_block *sb); void scoutfs_quorum_slot_sin(struct scoutfs_super_block *super, int i, struct sockaddr_in *sin); -int scoutfs_quorum_fence_leader_blocks(struct super_block *sb, u64 term); -int scoutfs_quorum_clear_rid_leader(struct super_block *sb, u64 rid); +int scoutfs_quorum_fence_leaders(struct super_block *sb, u64 term); +int scoutfs_quorum_fence_complete(struct super_block *sb, u64 term); int scoutfs_quorum_setup(struct super_block *sb); void scoutfs_quorum_shutdown(struct super_block *sb); diff --git a/kmod/src/server.c b/kmod/src/server.c index 872d5339..4eeefccd 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -1723,7 +1723,7 @@ struct farewell_request { * individual action knows to recognize that it's already been performed * and return success. */ -static int reclaim_rid(struct super_block *sb, u64 rid, bool clear_leader) +static int reclaim_rid(struct super_block *sb, u64 rid) { int ret; @@ -1737,7 +1737,6 @@ static int reclaim_rid(struct super_block *sb, u64 rid, bool clear_leader) reclaim_log_trees(sb, rid) ?: cancel_srch_compact(sb, rid) ?: scoutfs_omap_remove_rid(sb, rid) ?: - (clear_leader ? scoutfs_quorum_clear_rid_leader(sb, rid) : 0) ?: delete_mounted_client(sb, rid); return scoutfs_server_apply_commit(sb, ret); @@ -1870,7 +1869,7 @@ static void farewell_worker(struct work_struct *work) /* clean up resources for mounts before sending responses */ list_for_each_entry_safe(fw, tmp, &send, entry) { - ret = reclaim_rid(sb, fw->rid, false); + ret = reclaim_rid(sb, fw->rid); if (ret) goto out; } @@ -2204,7 +2203,7 @@ static void reclaim_worker(struct work_struct *work) goto out; } - ret = reclaim_rid(sb, rid, reason == SCOUTFS_FENCE_QUORUM_BLOCK_LEADER); + ret = reclaim_rid(sb, rid); if (ret < 0) { scoutfs_err(sb, "failure to reclaim fenced rid %016llx: err %d, shutting down server", rid, ret); @@ -2215,6 +2214,15 @@ static void reclaim_worker(struct work_struct *work) scoutfs_info(sb, "successfully reclaimed resources for fenced rid %016llx", rid); scoutfs_fence_free(sb, rid); scoutfs_server_recov_finish(sb, rid, SCOUTFS_RECOV_ALL); + + /* tell quorum we've finished fencing all previous leaders */ + if (reason == SCOUTFS_FENCE_QUORUM_BLOCK_LEADER && + !scoutfs_fence_reason_pending(sb, reason)) { + ret = scoutfs_quorum_fence_complete(sb, server->term); + if (ret < 0) + goto out; + } + ret = 0; out: @@ -2242,7 +2250,7 @@ static void scoutfs_server_worker(struct work_struct *work) trace_scoutfs_server_work_enter(sb, 0, 0); /* first make sure no other servers are still running */ - ret = scoutfs_quorum_fence_leader_blocks(sb, server->term); + ret = scoutfs_quorum_fence_leaders(sb, server->term); if (ret < 0) goto out; @@ -2348,7 +2356,7 @@ out: scoutfs_net_free_conn(sb, conn); /* let quorum know that we've shutdown */ - scoutfs_quorum_server_shutdown(sb); + scoutfs_quorum_server_shutdown(sb, server->term); scoutfs_info(sb, "server stopped at "SIN_FMT, SIN_ARG(&sin)); trace_scoutfs_server_work_exit(sb, 0, ret); diff --git a/tests/funcs/filter.sh b/tests/funcs/filter.sh index 0d0850c3..8f146e34 100644 --- a/tests/funcs/filter.sh +++ b/tests/funcs/filter.sh @@ -70,6 +70,7 @@ t_filter_dmesg() re="$re|scoutfs .* reclaimed resources" re="$re|scoutfs .* quorum .* error" re="$re|scoutfs .* error reading quorum block" + re="$re|scoutfs .* error .* writing quorum block" egrep -v "($re)" } diff --git a/utils/src/print.c b/utils/src/print.c index e61361bd..5fa57bdb 100644 --- a/utils/src/print.c +++ b/utils/src/print.c @@ -816,16 +816,16 @@ static char *alloc_addr_str(union scoutfs_inet_addr *ia) static int print_quorum_blocks(int fd, struct scoutfs_super_block *super) { - struct print_events { - size_t offset; - char *name; - } events[] = { - OFF_NAME(write), OFF_NAME(update_term), OFF_NAME(set_leader), - OFF_NAME(clear_leader), OFF_NAME(fenced), + const static char *event_names[] = { + [SCOUTFS_QUORUM_EVENT_BEGIN] = "begin", + [SCOUTFS_QUORUM_EVENT_TERM] = "term", + [SCOUTFS_QUORUM_EVENT_ELECT] = "elect", + [SCOUTFS_QUORUM_EVENT_FENCE] = "fence", + [SCOUTFS_QUORUM_EVENT_STOP] = "stop", + [SCOUTFS_QUORUM_EVENT_END] = "end", }; struct scoutfs_quorum_block *blk = NULL; struct scoutfs_quorum_block_event *ev; - char *log_addr = NULL; u64 blkno; int ret; int i; @@ -834,6 +834,7 @@ static int print_quorum_blocks(int fd, struct scoutfs_super_block *super) for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { blkno = SCOUTFS_QUORUM_BLKNO + i; free(blk); + blk = NULL; ret = read_block(fd, blkno, SCOUTFS_BLOCK_SM_SHIFT, (void **)&blk); if (ret) goto out; @@ -841,24 +842,19 @@ static int print_quorum_blocks(int fd, struct scoutfs_super_block *super) printf("quorum blkno %llu (slot %llu)\n", blkno, blkno - SCOUTFS_QUORUM_BLKNO); print_block_header(&blk->hdr, SCOUTFS_BLOCK_SM_SIZE); - printf(" term %llu random_write_mark 0x%llx flags 0x%llx\n", - le64_to_cpu(blk->term), - le64_to_cpu(blk->random_write_mark), - le64_to_cpu(blk->flags)); - for (e = 0; e < array_size(events); e++) { - ev = (void *)blk + events[e].offset; + for (e = 0; e < array_size(event_names); e++) { + ev = &blk->events[e]; - printf(" %12s: rid %016llx ts %llu.%08u\n", - events[e].name, le64_to_cpu(ev->rid), - le64_to_cpu(ev->ts.sec), - le32_to_cpu(ev->ts.nsec)); + printf(" %12s: rid %016llx term %llu ts %llu.%08u\n", + event_names[e], le64_to_cpu(ev->rid), le64_to_cpu(ev->term), + le64_to_cpu(ev->ts.sec), le32_to_cpu(ev->ts.nsec)); } } ret = 0; out: - free(log_addr); + free(blk); return ret; }