diff --git a/kmod/src/Makefile b/kmod/src/Makefile index c211d6e4..975ec319 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -18,6 +18,7 @@ scoutfs-y += \ dir.o \ export.o \ ext.o \ + fence.o \ file.o \ forest.o \ inode.o \ diff --git a/kmod/src/block.c b/kmod/src/block.c index 53578725..58c12d5b 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -200,7 +200,9 @@ static void block_free(struct super_block *sb, struct block_private *bp) else BUG(); - WARN_ON_ONCE(!list_empty(&bp->dirty_entry)); + /* ok to tear down dirty blocks when forcing unmount */ + WARN_ON_ONCE(!scoutfs_forcing_unmount(sb) && !list_empty(&bp->dirty_entry)); + WARN_ON_ONCE(atomic_read(&bp->refcount)); WARN_ON_ONCE(atomic_read(&bp->io_count)); kfree(bp); @@ -485,6 +487,9 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp, sector_t sector; int ret = 0; + if (scoutfs_forcing_unmount(sb)) + return -EIO; + sector = bp->bl.blkno << (SCOUTFS_BLOCK_LG_SHIFT - 9); WARN_ON_ONCE(bp->bl.blkno == U64_MAX); @@ -1148,7 +1153,7 @@ static void sm_block_bio_end_io(struct bio *bio, int err) * only layer that sees the full block buffer so we pass the calculated * crc to the caller for them to check in their context. */ -static int sm_block_io(struct block_device *bdev, int rw, u64 blkno, +static int sm_block_io(struct super_block *sb, struct block_device *bdev, int rw, u64 blkno, struct scoutfs_block_header *hdr, size_t len, __le32 *blk_crc) { @@ -1160,6 +1165,9 @@ static int sm_block_io(struct block_device *bdev, int rw, u64 blkno, BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_BLOCK_SM_SIZE); + if (scoutfs_forcing_unmount(sb)) + return -EIO; + if (WARN_ON_ONCE(len > SCOUTFS_BLOCK_SM_SIZE) || WARN_ON_ONCE(!(rw & WRITE) && !blk_crc)) return -EINVAL; @@ -1212,14 +1220,14 @@ int scoutfs_block_read_sm(struct super_block *sb, struct scoutfs_block_header *hdr, size_t len, __le32 *blk_crc) { - return sm_block_io(bdev, READ, blkno, hdr, len, blk_crc); + return sm_block_io(sb, bdev, READ, blkno, hdr, len, blk_crc); } int scoutfs_block_write_sm(struct super_block *sb, struct block_device *bdev, u64 blkno, struct scoutfs_block_header *hdr, size_t len) { - return sm_block_io(bdev, WRITE, blkno, hdr, len, NULL); + return sm_block_io(sb, bdev, WRITE, blkno, hdr, len, NULL); } int scoutfs_block_setup(struct super_block *sb) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 97778352..b9b02696 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1101,6 +1101,10 @@ static int btree_walk(struct super_block *sb, if (WARN_ON_ONCE((flags & BTW_DIRTY) && (!alloc || !wri))) return -EINVAL; + /* all ops come through walk and walk calls all reads */ + if (scoutfs_forcing_unmount(sb)) + return -EIO; + scoutfs_inc_counter(sb, btree_walk); restart: diff --git a/kmod/src/client.c b/kmod/src/client.c index fe50b45f..7a4b4322 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -48,6 +48,7 @@ struct client_info { struct workqueue_struct *workq; struct delayed_work connect_dwork; + unsigned long connect_delay_jiffies; u64 server_term; @@ -349,6 +350,7 @@ static int client_greeting(struct super_block *sb, scoutfs_net_client_greeting(sb, conn, new_server); client->server_term = le64_to_cpu(gr->server_term); + client->connect_delay_jiffies = 0; ret = 0; out: return ret; @@ -398,6 +400,20 @@ out: return ret; } +/* + * If we're not seeing successful connections we want to back off. Each + * connection attempt starts by setting a long connection work delay. + * We only set a shorter delay if we see a greeting response from the + * server. At that point we'll try to immediately reconnect if the + * connection is broken. + */ +static void queue_connect_dwork(struct super_block *sb, struct client_info *client) +{ + if (!atomic_read(&client->shutting_down) && !scoutfs_forcing_unmount(sb)) + queue_delayed_work(client->workq, &client->connect_dwork, + client->connect_delay_jiffies); +} + /* * This work is responsible for maintaining a connection from the client * to the server. It's queued on mount and disconnect and we requeue @@ -437,6 +453,9 @@ static void scoutfs_client_connect_worker(struct work_struct *work) goto out; } + /* always wait a bit until a greeting response sets a lower delay */ + client->connect_delay_jiffies = msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS); + ret = scoutfs_quorum_server_sin(sb, &sin); if (ret < 0) goto out; @@ -464,11 +483,8 @@ static void scoutfs_client_connect_worker(struct work_struct *work) if (ret) scoutfs_net_shutdown(sb, client->conn); out: - - /* always have a small delay before retrying to avoid storms */ - if (ret && !atomic_read(&client->shutting_down)) - queue_delayed_work(client->workq, &client->connect_dwork, - msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS)); + if (ret) + queue_connect_dwork(sb, client); } static scoutfs_net_request_t client_req_funcs[] = { @@ -487,8 +503,7 @@ static void client_notify_down(struct super_block *sb, { struct client_info *client = SCOUTFS_SB(sb)->client_info; - if (!atomic_read(&client->shutting_down)) - queue_delayed_work(client->workq, &client->connect_dwork, 0); + queue_connect_dwork(sb, client); } int scoutfs_client_setup(struct super_block *sb) @@ -523,7 +538,7 @@ int scoutfs_client_setup(struct super_block *sb) goto out; } - queue_delayed_work(client->workq, &client->connect_dwork, 0); + queue_connect_dwork(sb, client); ret = 0; out: @@ -580,7 +595,7 @@ void scoutfs_client_destroy(struct super_block *sb) if (client == NULL) return; - if (client->server_term != 0) { + if (client->server_term != 0 && !scoutfs_forcing_unmount(sb)) { client->sending_farewell = true; ret = scoutfs_net_submit_request(sb, client->conn, SCOUTFS_NET_CMD_FAREWELL, diff --git a/kmod/src/fence.c b/kmod/src/fence.c new file mode 100644 index 00000000..1b039b2e --- /dev/null +++ b/kmod/src/fence.c @@ -0,0 +1,480 @@ +/* + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "msg.h" +#include "sysfs.h" +#include "server.h" +#include "fence.h" + +/* + * Fencing ensures that a given mount can no longer write to the + * metadata or data devices. It's necessary to ensure that it's safe to + * give another mount access to a resource that is currently owned by a + * mount that has stopped responding. + * + * Fencing is performed in collaboration between the currently elected + * quorum leader mount and userspace running on its host. The kernel + * creates fencing requests as it notices that mounts have stopped + * participating. The fence requests are published as directories in + * sysfs. Userspace agents watch for directories, take action, and + * write to files in the directory to indicate that the mount has been + * fenced. Once the mount is fenced the server can reclaim the + * resources previously held by the fenced mount. + * + * The fence requests contain metadata identifying the specific instance + * of the mount that needs to be fenced. This lets a fencing agent + * ensure that a specific mount has been fenced without necessarily + * destroying the node that was hosting it. Maybe the node had rebooted + * and the mount is no longer there, maybe the mount can be force + * unmounted, maybe the node can be configured to isolate the mount from + * the devices. + * + * The fencing mechanism is asynchronous and can fail but the server + * cannot make progress until it completes. If a fence request times + * out the server shuts down in the hope that another instance of a + * server might have more luck fencing a non-responsive mount. + * + * Sources of fencing are fundamentally anchored in shared persistent + * state. It is possible, though unlikely, that servers can fence a + * node and then themselves fail, leaving the next server to try and + * fence the mount again. + */ + +struct fence_info { + struct kset *kset; + struct kobject fence_dir_kobj; + struct workqueue_struct *wq; + wait_queue_head_t waitq; + spinlock_t lock; + struct list_head list; +}; + +#define DECLARE_FENCE_INFO(sb, name) \ + struct fence_info *name = SCOUTFS_SB(sb)->fence_info + +struct pending_fence { + struct super_block *sb; + struct scoutfs_sysfs_attrs ssa; + struct list_head entry; + struct timer_list timer; + + ktime_t start_kt; + __be32 ipv4_addr; + bool fenced; + bool error; + int reason; + u64 rid; +}; + +#define FENCE_FROM_KOBJ(kobj) \ + container_of(SCOUTFS_SYSFS_ATTRS(kobj), struct pending_fence, ssa) +#define DECLARE_FENCE_FROM_KOBJ(name, kobj) \ + struct pending_fence *name = FENCE_FROM_KOBJ(kobj) + +static void destroy_fence(struct pending_fence *fence) +{ + struct super_block *sb = fence->sb; + + scoutfs_sysfs_destroy_attrs(sb, &fence->ssa); + del_timer_sync(&fence->timer); + kfree(fence); +} + +static ssize_t elapsed_secs_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + ktime_t now = ktime_get(); + struct timeval tv = { 0, }; + + if (ktime_after(now, fence->start_kt)) + tv = ktime_to_timeval(ktime_sub(now, fence->start_kt)); + + return snprintf(buf, PAGE_SIZE, "%llu", (long long)tv.tv_sec); +} +SCOUTFS_ATTR_RO(elapsed_secs); + +static ssize_t fenced_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + + return snprintf(buf, PAGE_SIZE, "%u", !!fence->fenced); +} + +/* + * any write to the fenced file from userspace indicates that the mount + * has been safely fenced and can no longer write to the shared device. + */ +static ssize_t fenced_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + DECLARE_FENCE_INFO(fence->sb, fi); + + if (!fence->fenced) { + del_timer_sync(&fence->timer); + fence->fenced = true; + wake_up(&fi->waitq); + } + + return count; +} +SCOUTFS_ATTR_RW(fenced); + +static ssize_t error_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + + return snprintf(buf, PAGE_SIZE, "%u", !!fence->error); +} + +/* + * Fencing can tell us that they were unable to fence the given mount. + * We can't continue if the mount can't be isolated so we shut down the + * server. + */ +static ssize_t error_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, + size_t count) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + struct super_block *sb = fence->sb; + DECLARE_FENCE_INFO(fence->sb, fi); + + if (!fence->error) { + fence->error = true; + scoutfs_err(sb, "error indicated by fence action for rid %016llx", fence->rid); + wake_up(&fi->waitq); + } + + return count; +} +SCOUTFS_ATTR_RW(error); + +static ssize_t ipv4_addr_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + + return snprintf(buf, PAGE_SIZE, "%pI4", &fence->ipv4_addr); +} +SCOUTFS_ATTR_RO(ipv4_addr); + +static ssize_t reason_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + unsigned r = fence->reason; + char *str = "unknown"; + static char *reasons[] = { + [SCOUTFS_FENCE_CLIENT_RECOVERY] = "client_recovery", + [SCOUTFS_FENCE_CLIENT_RECONNECT] = "client_reconnect", + [SCOUTFS_FENCE_QUORUM_BLOCK_LEADER] = "quorum_block_leader", + }; + + if (r < ARRAY_SIZE(reasons) && reasons[r]) + str = reasons[r]; + + return snprintf(buf, PAGE_SIZE, "%s", str); +} +SCOUTFS_ATTR_RO(reason); + +static ssize_t rid_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + DECLARE_FENCE_FROM_KOBJ(fence, kobj); + + return snprintf(buf, PAGE_SIZE, "%016llx", fence->rid); +} +SCOUTFS_ATTR_RO(rid); + +static struct attribute *fence_attrs[] = { + SCOUTFS_ATTR_PTR(elapsed_secs), + SCOUTFS_ATTR_PTR(fenced), + SCOUTFS_ATTR_PTR(error), + SCOUTFS_ATTR_PTR(ipv4_addr), + SCOUTFS_ATTR_PTR(reason), + SCOUTFS_ATTR_PTR(rid), + NULL, +}; + +#define FENCE_TIMEOUT_MS (MSEC_PER_SEC * 30) + +static void fence_timeout(struct timer_list *timer) +{ + struct pending_fence *fence = from_timer(fence, timer, timer); + struct super_block *sb = fence->sb; + DECLARE_FENCE_INFO(sb, fi); + + fence->error = true; + scoutfs_err(sb, "fence request for rid %016llx was not serviced in %lums, raising error", + fence->rid, FENCE_TIMEOUT_MS); + wake_up(&fi->waitq); +} + +int scoutfs_fence_start(struct super_block *sb, u64 rid, __be32 ipv4_addr, int reason) +{ + DECLARE_FENCE_INFO(sb, fi); + struct pending_fence *fence; + int ret; + + fence = kzalloc(sizeof(struct pending_fence), GFP_NOFS); + if (!fence) { + ret = -ENOMEM; + goto out; + } + + fence->sb = sb; + scoutfs_sysfs_init_attrs(sb, &fence->ssa); + + fence->start_kt = ktime_get(); + fence->ipv4_addr = ipv4_addr; + fence->fenced = false; + fence->error = false; + fence->reason = reason; + fence->rid = rid; + + ret = scoutfs_sysfs_create_attrs_parent(sb, &fi->kset->kobj, + &fence->ssa, fence_attrs, + "%016llx", rid); + if (ret < 0) { + kfree(fence); + goto out; + } + + timer_setup(&fence->timer, fence_timeout, 0); + fence->timer.expires = jiffies + msecs_to_jiffies(FENCE_TIMEOUT_MS); + add_timer(&fence->timer); + + spin_lock(&fi->lock); + list_add_tail(&fence->entry, &fi->list); + spin_unlock(&fi->lock); +out: + return ret; +} + +/* + * Give the caller the rid of the next fence request which has been + * fenced. This doesn't have a position from which to return the next + * because the caller either frees the fence request it's given or shuts + * down. + */ +int scoutfs_fence_next(struct super_block *sb, u64 *rid, int *reason, bool *error) +{ + DECLARE_FENCE_INFO(sb, fi); + struct pending_fence *fence; + int ret = -ENOENT; + + spin_lock(&fi->lock); + list_for_each_entry(fence, &fi->list, entry) { + if (fence->fenced || fence->error) { + *rid = fence->rid; + *reason = fence->reason; + *error = fence->error; + ret = 0; + break; + } + } + spin_unlock(&fi->lock); + + return ret; +} + +int scoutfs_fence_reason_pending(struct super_block *sb, int reason) +{ + DECLARE_FENCE_INFO(sb, fi); + struct pending_fence *fence; + bool pending = false; + + spin_lock(&fi->lock); + list_for_each_entry(fence, &fi->list, entry) { + if (fence->reason == reason) { + pending = true; + break; + } + } + spin_unlock(&fi->lock); + + return pending; +} + +int scoutfs_fence_free(struct super_block *sb, u64 rid) +{ + DECLARE_FENCE_INFO(sb, fi); + struct pending_fence *fence; + int ret = -ENOENT; + + spin_lock(&fi->lock); + list_for_each_entry(fence, &fi->list, entry) { + if (fence->rid == rid) { + list_del_init(&fence->entry); + ret = 0; + break; + } + } + spin_unlock(&fi->lock); + + if (ret == 0) { + destroy_fence(fence); + wake_up(&fi->waitq); + } + + return ret; +} + +static bool all_fenced(struct fence_info *fi, bool *error) +{ + struct pending_fence *fence; + bool all = true; + + *error = false; + + spin_lock(&fi->lock); + list_for_each_entry(fence, &fi->list, entry) { + if (fence->error) { + *error = true; + all = true; + break; + } + if (!fence->fenced) { + all = false; + break; + } + } + spin_unlock(&fi->lock); + + return all; +} + +/* + * The caller waits for all the current requests to be fenced, but not + * necessarily reclaimed. + */ +int scoutfs_fence_wait_fenced(struct super_block *sb, long timeout_jiffies) +{ + DECLARE_FENCE_INFO(sb, fi); + bool error; + long ret; + + ret = wait_event_interruptible_timeout(fi->waitq, all_fenced(fi, &error), timeout_jiffies); + if (ret == 0) + ret = -ETIMEDOUT; + else if (ret > 0) + ret = 0; + else if (error) + ret = -EIO; + + return ret; +} + +/* + * This must be called early during startup so that it is guaranteed that + * no other subsystems will try and call fence_start while we're waiting + * for testing fence requests to complete. + */ +int scoutfs_fence_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct mount_options *opts = &sbi->opts; + struct fence_info *fi; + int ret; + + /* can only fence if we can be elected by quorum */ + if (opts->quorum_slot_nr == -1) { + ret = 0; + goto out; + } + + fi = kzalloc(sizeof(struct fence_info), GFP_KERNEL); + if (!fi) { + ret = -ENOMEM; + goto out; + } + + init_waitqueue_head(&fi->waitq); + spin_lock_init(&fi->lock); + INIT_LIST_HEAD(&fi->list); + + sbi->fence_info = fi; + + fi->kset = kset_create_and_add("fence", NULL, scoutfs_sysfs_sb_dir(sb)); + if (!fi->kset) { + ret = -ENOMEM; + goto out; + } + + fi->wq = alloc_workqueue("scoutfs_fence", + WQ_UNBOUND | WQ_NON_REENTRANT, 0); + if (!fi->wq) { + ret = -ENOMEM; + goto out; + } + + ret = 0; +out: + if (ret) + scoutfs_fence_destroy(sb); + + return ret; +} + +/* + * Tear down all pending fence requests because the server is shutting down. + */ +void scoutfs_fence_stop(struct super_block *sb) +{ + DECLARE_FENCE_INFO(sb, fi); + struct pending_fence *fence; + + do { + spin_lock(&fi->lock); + fence = list_first_entry_or_null(&fi->list, struct pending_fence, entry); + if (fence) + list_del_init(&fence->entry); + spin_unlock(&fi->lock); + + if (fence) { + destroy_fence(fence); + wake_up(&fi->waitq); + } + } while (fence); +} + +void scoutfs_fence_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct fence_info *fi = SCOUTFS_SB(sb)->fence_info; + struct pending_fence *fence; + struct pending_fence *tmp; + + if (fi) { + if (fi->wq) + destroy_workqueue(fi->wq); + list_for_each_entry_safe(fence, tmp, &fi->list, entry) + destroy_fence(fence); + if (fi->kset) + kset_unregister(fi->kset); + kfree(fi); + sbi->fence_info = NULL; + } +} diff --git a/kmod/src/fence.h b/kmod/src/fence.h new file mode 100644 index 00000000..f9139001 --- /dev/null +++ b/kmod/src/fence.h @@ -0,0 +1,20 @@ +#ifndef _SCOUTFS_FENCE_H_ +#define _SCOUTFS_FENCE_H_ + +enum { + SCOUTFS_FENCE_CLIENT_RECOVERY, + SCOUTFS_FENCE_CLIENT_RECONNECT, + SCOUTFS_FENCE_QUORUM_BLOCK_LEADER, +}; + +int scoutfs_fence_start(struct super_block *sb, u64 rid, __be32 ipv4_addr, int reason); +int scoutfs_fence_next(struct super_block *sb, u64 *rid, int *reason, bool *error); +int scoutfs_fence_reason_pending(struct super_block *sb, int reason); +int scoutfs_fence_free(struct super_block *sb, u64 rid); +int scoutfs_fence_wait_fenced(struct super_block *sb, long timeout_jiffies); + +int scoutfs_fence_setup(struct super_block *sb); +void scoutfs_fence_stop(struct super_block *sb); +void scoutfs_fence_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 3c0d789f..924a1842 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -327,7 +327,9 @@ struct scoutfs_alloc_root { #define SCOUTFS_ALLOC_OWNER_SRCH 3 struct scoutfs_mounted_client_btree_val { + union scoutfs_inet_addr addr; __u8 flags; + __u8 __pad[7]; }; #define SCOUTFS_MOUNTED_CLIENT_QUORUM (1 << 0) @@ -586,6 +588,12 @@ struct scoutfs_xattr { #define SCOUTFS_QUORUM_HB_IVAL_MS 100 #define SCOUTFS_QUORUM_HB_TIMEO_MS (5 * MSEC_PER_SEC) +/* + * A newly elected leader will give fencing some time before giving up and + * shutting down. + */ +#define SCOUTFS_QUORUM_FENCE_TO_MS (15 * MSEC_PER_SEC) + struct scoutfs_quorum_message { __le64 fsid; __le64 version; @@ -617,18 +625,24 @@ struct scoutfs_quorum_config { } slots[SCOUTFS_QUORUM_MAX_SLOTS]; }; -struct scoutfs_quorum_block { - struct scoutfs_block_header hdr; - __le64 term; - __le64 random_write_mark; - __le64 flags; - struct scoutfs_quorum_block_event { - __le64 rid; - struct scoutfs_timespec ts; - } write, update_term, set_leader, clear_leader, fenced; +enum { + SCOUTFS_QUORUM_EVENT_BEGIN, /* quorum service starting up */ + SCOUTFS_QUORUM_EVENT_TERM, /* updated persistent term */ + SCOUTFS_QUORUM_EVENT_ELECT, /* won election */ + SCOUTFS_QUORUM_EVENT_FENCE, /* server fenced others */ + SCOUTFS_QUORUM_EVENT_STOP, /* server stopped */ + SCOUTFS_QUORUM_EVENT_END, /* quorum service shutting down */ + SCOUTFS_QUORUM_EVENT_NR, }; -#define SCOUTFS_QUORUM_BLOCK_LEADER (1 << 0) +struct scoutfs_quorum_block { + struct scoutfs_block_header hdr; + struct scoutfs_quorum_block_event { + __le64 rid; + __le64 term; + struct scoutfs_timespec ts; + } events[SCOUTFS_QUORUM_EVENT_NR]; +}; /* * Tunable options that apply to the entire system. They can be set in diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index fb33fd80..09ce48d7 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -486,7 +486,7 @@ static int process_waiting_requests(struct super_block *sb, /* processing waits for all invalidation responses or recovery */ if (!list_empty(&snode->invalidated) || - scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_LOCKS) != 0) { + scoutfs_recov_next_pending(sb, 0, SCOUTFS_RECOV_LOCKS) != 0) { ret = 0; goto out; } diff --git a/kmod/src/net.c b/kmod/src/net.c index 9d1fedef..593a62e7 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -30,6 +30,7 @@ #include "net.h" #include "endian_swap.h" #include "tseq.h" +#include "fence.h" /* * scoutfs networking delivers requests and responses between nodes. @@ -330,6 +331,9 @@ static int submit_send(struct super_block *sb, WARN_ON_ONCE(id == 0 && (flags & SCOUTFS_NET_FLAG_RESPONSE))) return -EINVAL; + if (scoutfs_forcing_unmount(sb)) + return -EIO; + msend = kmalloc(offsetof(struct message_send, nh.data[data_len]), GFP_NOFS); if (!msend) @@ -420,6 +424,16 @@ static int process_request(struct scoutfs_net_connection *conn, mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); } +static int call_resp_func(struct super_block *sb, struct scoutfs_net_connection *conn, + scoutfs_net_response_t resp_func, void *resp_data, + void *resp, unsigned int resp_len, int error) +{ + if (resp_func) + return resp_func(sb, conn, resp, resp_len, error, resp_data); + else + return 0; +} + /* * An incoming response finds the queued request and calls its response * function. The response function for a given request will only be @@ -434,7 +448,6 @@ static int process_response(struct scoutfs_net_connection *conn, struct message_send *msend; scoutfs_net_response_t resp_func = NULL; void *resp_data; - int ret = 0; spin_lock(&conn->lock); @@ -449,11 +462,8 @@ static int process_response(struct scoutfs_net_connection *conn, spin_unlock(&conn->lock); - if (resp_func) - ret = resp_func(sb, conn, mrecv->nh.data, - le16_to_cpu(mrecv->nh.data_len), - net_err_to_host(mrecv->nh.error), resp_data); - return ret; + return call_resp_func(sb, conn, resp_func, resp_data, mrecv->nh.data, + le16_to_cpu(mrecv->nh.data_len), net_err_to_host(mrecv->nh.error)); } /* @@ -823,9 +833,15 @@ static void scoutfs_net_destroy_worker(struct work_struct *work) if (conn->listening_conn && conn->notify_down) conn->notify_down(sb, conn, conn->info, conn->rid); - /* free all messages, refactor and complete for forced unmount? */ + /* + * Usually networking is idle and we destroy pending sends, but when forcing unmount + * we can have to wake up waiters by failing pending sends. + */ list_splice_init(&conn->resend_queue, &conn->send_queue); list_for_each_entry_safe(msend, tmp, &conn->send_queue, head) { + if (scoutfs_forcing_unmount(sb)) + call_resp_func(sb, conn, msend->resp_func, msend->resp_data, + NULL, 0, -ECONNABORTED); free_msend(ninf, msend); } @@ -925,6 +941,8 @@ static int sock_opts_and_names(struct scoutfs_net_connection *conn, ret = -EAFNOSUPPORT; if (ret) goto out; + + conn->last_peername = conn->peername; out: return ret; } @@ -1205,6 +1223,7 @@ static void scoutfs_net_reconn_free_worker(struct work_struct *work) unsigned long now = jiffies; unsigned long deadline = 0; bool requeue = false; + int ret; trace_scoutfs_net_reconn_free_work_enter(sb, 0, 0); @@ -1218,10 +1237,18 @@ restart: time_after_eq(now, acc->reconn_deadline))) { set_conn_fl(acc, reconn_freeing); spin_unlock(&conn->lock); - if (!test_conn_fl(conn, shutting_down)) - scoutfs_info(sb, "client timed out "SIN_FMT" -> "SIN_FMT", can not reconnect", - SIN_ARG(&acc->sockname), - SIN_ARG(&acc->peername)); + if (!test_conn_fl(conn, shutting_down)) { + scoutfs_info(sb, "client "SIN_FMT" reconnect timed out, fencing", + SIN_ARG(&acc->last_peername)); + ret = scoutfs_fence_start(sb, acc->rid, + acc->last_peername.sin_addr.s_addr, + SCOUTFS_FENCE_CLIENT_RECONNECT); + if (ret) { + scoutfs_err(sb, "client fence returned err %d, shutting down server", + ret); + scoutfs_server_abort(sb); + } + } destroy_conn(acc); goto restart; } @@ -1292,6 +1319,7 @@ scoutfs_net_alloc_conn(struct super_block *sb, init_waitqueue_head(&conn->waitq); conn->sockname.sin_family = AF_INET; conn->peername.sin_family = AF_INET; + conn->last_peername.sin_family = AF_INET; INIT_LIST_HEAD(&conn->accepted_head); INIT_LIST_HEAD(&conn->accepted_list); conn->next_send_seq = 1; diff --git a/kmod/src/net.h b/kmod/src/net.h index 0b18b6ff..847e9204 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -49,6 +49,7 @@ struct scoutfs_net_connection { u64 greeting_id; struct sockaddr_in sockname; struct sockaddr_in peername; + struct sockaddr_in last_peername; struct list_head accepted_head; struct scoutfs_net_connection *listening_conn; @@ -99,6 +100,16 @@ static inline void scoutfs_addr_to_sin(struct sockaddr_in *sin, sin->sin_port = cpu_to_be16(le16_to_cpu(addr->v4.port)); } +static inline void scoutfs_sin_to_addr(union scoutfs_inet_addr *addr, struct sockaddr_in *sin) +{ + BUG_ON(sin->sin_family != AF_INET); + + memset(addr, 0, sizeof(union scoutfs_inet_addr)); + addr->v4.family = cpu_to_le16(SCOUTFS_AF_IPV4); + addr->v4.addr = be32_to_le32(sin->sin_addr.s_addr); + addr->v4.port = be16_to_le16(sin->sin_port); +} + struct scoutfs_net_connection * scoutfs_net_alloc_conn(struct super_block *sb, scoutfs_net_notify_t notify_up, diff --git a/kmod/src/omap.c b/kmod/src/omap.c index bb3ac8c4..3dfcbea8 100644 --- a/kmod/src/omap.c +++ b/kmod/src/omap.c @@ -485,6 +485,10 @@ static int remove_rid_from_reqs(struct omap_info *ominf, u64 rid, u64 *resp_rid, * response if it was the last rid waiting for a response. * * If this returns an error then the server will shut down. + * + * This can be called multiple times by different servers if there are + * errors reclaiming an evicted mount, so we allow asking to remove a + * rid that hasn't been added. */ int scoutfs_omap_remove_rid(struct super_block *sb, u64 rid) { @@ -495,21 +499,20 @@ int scoutfs_omap_remove_rid(struct super_block *sb, u64 rid) u64 resp_id = 0; int ret; - map = kmalloc(sizeof(struct scoutfs_open_ino_map), GFP_NOFS); - if (!map) { - ret = -ENOMEM; - goto out; - } - spin_lock(&ominf->lock); entry = find_rid(&ominf->rids, rid); if (entry) free_rid(&ominf->rids, entry); spin_unlock(&ominf->lock); - /* the server really shouldn't be removing a rid it never added */ - if (WARN_ON_ONCE(!entry)) { - ret = -ENOENT; + if (!entry) { + ret = 0; + goto out; + } + + map = kmalloc(sizeof(struct scoutfs_open_ino_map), GFP_NOFS); + if (!map) { + ret = -ENOMEM; goto out; } @@ -616,7 +619,7 @@ static int handle_requests(struct super_block *sb) int ret; int err; - if (scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_GREETING)) + if (scoutfs_recov_next_pending(sb, 0, SCOUTFS_RECOV_GREETING)) return 0; ret = 0; diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index c7d50cab..8017642f 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -32,6 +32,7 @@ #include "block.h" #include "net.h" #include "sysfs.h" +#include "fence.h" #include "scoutfs_trace.h" /* @@ -60,10 +61,9 @@ * running (maybe they've deadlocked, or lost network communications). * In addition to a configuration slot in the super block, each quorum * member also has a known block location that represents their slot. - * They set a flag in their block indicating that they've been elected - * leader, then read slots for all the other blocks looking for - * previously active leaders to fence. After that it can start the - * server. + * The block contains an array of events which are updated during the life + * time of the quorum agent. The elected leader set its elected event + * and can then start the server. * * It's critical to raft elections that a participant's term not go * backwards in time so each mount also uses its quorum block to store @@ -334,17 +334,18 @@ static int recv_msg(struct super_block *sb, struct quorum_host_msg *msg, } /* - * The caller can provide a mark that they're using to track their - * written blocks. It's updated as they write the block and we can - * compare it with what we read to see if there have been unexpected - * intervening writes to the block -- the caller is supposed to have - * exclusive access to the block (or was fenced). + * Read and verify block fields before giving it to the caller. We + * should have exclusive write access to the block. We know that + * something has gone horribly wrong if we don't see our rid in the + * begin event after we've written it as we started up. */ -static int read_quorum_block(struct super_block *sb, u64 blkno, - struct scoutfs_quorum_block *blk, __le64 *mark) +static int read_quorum_block(struct super_block *sb, u64 blkno, struct scoutfs_quorum_block *blk, + bool check_rid) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; + const u64 rid = sbi->rid; + char msg[150]; __le32 crc; int ret; @@ -355,162 +356,208 @@ static int read_quorum_block(struct super_block *sb, u64 blkno, ret = scoutfs_block_read_sm(sb, sbi->meta_bdev, blkno, &blk->hdr, sizeof(*blk), &crc); + if (ret < 0) { + scoutfs_err(sb, "quorum block read error %d", ret); + goto out; + } /* detect invalid blocks */ - if (ret == 0 && - ((blk->hdr.crc != crc) || - (le32_to_cpu(blk->hdr.magic) != SCOUTFS_BLOCK_MAGIC_QUORUM) || - (blk->hdr.fsid != super->hdr.fsid) || - (le64_to_cpu(blk->hdr.blkno) != blkno))) { - scoutfs_inc_counter(sb, quorum_read_invalid_block); + if (blk->hdr.crc != crc) + snprintf(msg, sizeof(msg), "blk crc %08x != %08x", + le32_to_cpu(blk->hdr.crc), le32_to_cpu(crc)); + else if (le32_to_cpu(blk->hdr.magic) != SCOUTFS_BLOCK_MAGIC_QUORUM) + snprintf(msg, sizeof(msg), "blk magic %08x != %08x", + le32_to_cpu(blk->hdr.magic), SCOUTFS_BLOCK_MAGIC_QUORUM); + else if (blk->hdr.fsid != super->hdr.fsid) + snprintf(msg, sizeof(msg), "blk fsid %016llx != %016llx", + le64_to_cpu(blk->hdr.fsid), le64_to_cpu(super->hdr.fsid)); + else if (le64_to_cpu(blk->hdr.blkno) != blkno) + snprintf(msg, sizeof(msg), "blk blkno %llu != %llu", + le64_to_cpu(blk->hdr.blkno), blkno); + else if (check_rid && le64_to_cpu(blk->events[SCOUTFS_QUORUM_EVENT_BEGIN].rid) != rid) + snprintf(msg, sizeof(msg), "quorum block begin rid %016llx != our rid %016llx, are multiple mounts configured with this slot?", + le64_to_cpu(blk->events[SCOUTFS_QUORUM_EVENT_BEGIN].rid), rid); + else + msg[0] = '\0'; + + if (msg[0] != '\0') { + scoutfs_err(sb, "read invalid quorum block, %s", msg); ret = -EIO; + goto out; } - if (mark && *mark != 0 && blk->random_write_mark != *mark) { - scoutfs_err(sb, "read unexpected quorum block write mark, are multiple mounts configured with the same slot?"); - ret = -EIO; - } - - if (ret < 0) - scoutfs_err(sb, "quorum block read error %d", ret); - +out: return ret; } -static void set_quorum_block_event(struct super_block *sb, - struct scoutfs_quorum_block *blk, - struct scoutfs_quorum_block_event *ev) +static void set_quorum_block_event(struct super_block *sb, struct scoutfs_quorum_block *blk, + int event, u64 term) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_quorum_block_event *ev; struct timespec64 ts; + if (WARN_ON_ONCE(event < 0 || event >= SCOUTFS_QUORUM_EVENT_NR)) + return; + getnstimeofday64(&ts); + ev = &blk->events[event]; ev->rid = cpu_to_le64(sbi->rid); + ev->term = cpu_to_le64(term); ev->ts.sec = cpu_to_le64(ts.tv_sec); ev->ts.nsec = cpu_to_le32(ts.tv_nsec); } -/* - * Every time we write a block we update the write stamp and random - * write mark so readers can see our write. - */ -static int write_quorum_block(struct super_block *sb, u64 blkno, - struct scoutfs_quorum_block *blk, __le64 *mark) +static int write_quorum_block(struct super_block *sb, u64 blkno, struct scoutfs_quorum_block *blk) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - int ret; if (WARN_ON_ONCE(blkno < SCOUTFS_QUORUM_BLKNO) || WARN_ON_ONCE(blkno >= (SCOUTFS_QUORUM_BLKNO + SCOUTFS_QUORUM_BLOCKS))) return -EINVAL; - do { - get_random_bytes(&blk->random_write_mark, - sizeof(blk->random_write_mark)); - } while (blk->random_write_mark == 0); - - if (mark) - *mark = blk->random_write_mark; - - set_quorum_block_event(sb, blk, &blk->write); - - ret = scoutfs_block_write_sm(sb, sbi->meta_bdev, blkno, - &blk->hdr, sizeof(*blk)); - if (ret < 0) - scoutfs_err(sb, "quorum block write error %d", ret); - - return ret; + return scoutfs_block_write_sm(sb, sbi->meta_bdev, blkno, &blk->hdr, sizeof(*blk)); } /* - * Read the caller's slot's current quorum block, make a change, and - * write it back out. If the caller provides a mark it can cause read - * errors if we read a mark that doesn't match the last mark that the - * caller wrote. + * Read the caller's slot's quorum block, make a change, and write it + * back out. */ -static int update_quorum_block(struct super_block *sb, u64 blkno, - __le64 *mark, int role, u64 term) +static int update_quorum_block(struct super_block *sb, int event, u64 term, bool check_rid) { + struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + u64 blkno = SCOUTFS_QUORUM_BLKNO + opts->quorum_slot_nr; struct scoutfs_quorum_block blk; - u64 flags; - u64 bits; - u64 set; int ret; - ret = read_quorum_block(sb, blkno, &blk, mark); + ret = read_quorum_block(sb, blkno, &blk, check_rid); if (ret == 0) { - if (blk.term != cpu_to_le64(term)) { - blk.term = cpu_to_le64(term); - set_quorum_block_event(sb, &blk, &blk.update_term); - } - - flags = le64_to_cpu(blk.flags); - bits = SCOUTFS_QUORUM_BLOCK_LEADER; - set = role == LEADER ? SCOUTFS_QUORUM_BLOCK_LEADER : 0; - if ((flags & bits) != set) - set_quorum_block_event(sb, &blk, - set ? &blk.set_leader : - &blk.clear_leader); - blk.flags = cpu_to_le64((flags & ~bits) | set); - - ret = write_quorum_block(sb, blkno, &blk, mark); + set_quorum_block_event(sb, &blk, event, term); + ret = write_quorum_block(sb, blkno, &blk); + if (ret < 0) + scoutfs_err(sb, "error %d reading quorum block %llu to update event %d term %llu", + ret, blkno, event, term); + } else { + scoutfs_err(sb, "error %d writing quorum block %llu after updating event %d term %llu", + ret, blkno, event, term); } return ret; } +/* + * The calling server has fenced previous leaders and reclaimed their + * resources. We can now update our fence event with a greater term to + * stop future leaders from doing the same. + */ +int scoutfs_quorum_fence_complete(struct super_block *sb, u64 term) +{ + return update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_FENCE, term, true); +} /* - * The calling server has been elected and updated their block, but - * can't yet assume that it has exclusive access to the metadata device. - * We read all the quorum blocks looking for previously elected leaders - * to fence so that we're the only leader running. + * The calling server has been elected and has started running but can't + * yet assume that it has exclusive access to the metadata device. We + * read all the quorum blocks looking for previously elected leaders to + * fence so that we're the only leader running. + * + * We're relying on the invariant that there can't be two mounts running + * with the same slot nr at the same time. With this constraint there + * can be at most two previous leaders per slot that need to be fenced: + * a persistent record of an old mount on the slot, and an active mount. + * + * If we start fence requests then we only wait for them to complete + * before returning. The server will reclaim their resources once it is + * up and running and will call us to update the fence event. If we + * don't start fence requests then we update the fence event + * immediately, the server has nothing more to do. + * + * Quorum will be sending heartbeats while we wait for fencing. That + * keeps us from being fenced while we allow userspace fencing to take a + * reasonably long time. We still want to timeout eventually. */ -static int fence_leader_blocks(struct super_block *sb) +int scoutfs_quorum_fence_leaders(struct super_block *sb, u64 term) { +#define NR_OLD 2 + struct scoutfs_quorum_block_event old[SCOUTFS_QUORUM_MAX_SLOTS][NR_OLD] = {{{0,}}}; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct mount_options *opts = &sbi->opts; struct scoutfs_quorum_block blk; struct sockaddr_in sin; - u64 blkno; + const u64 rid = sbi->rid; + bool fence_started = false; + u64 fenced = 0; + __le64 fence_rid; int ret = 0; + int err; int i; + int j; BUILD_BUG_ON(SCOUTFS_QUORUM_BLOCKS < SCOUTFS_QUORUM_MAX_SLOTS); for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - if (i == opts->quorum_slot_nr) + if (!quorum_slot_present(super, i)) continue; - blkno = SCOUTFS_QUORUM_BLKNO + i; - ret = read_quorum_block(sb, blkno, &blk, NULL); + ret = read_quorum_block(sb, SCOUTFS_QUORUM_BLKNO + i, &blk, false); if (ret < 0) goto out; - if (!(le64_to_cpu(blk.flags) & SCOUTFS_QUORUM_BLOCK_LEADER)) - continue; + /* elected leader still running */ + if (le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_ELECT].term) > + le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_STOP].term)) + old[i][0] = blk.events[SCOUTFS_QUORUM_EVENT_ELECT]; - scoutfs_inc_counter(sb, quorum_fence_leader); - scoutfs_quorum_slot_sin(super, i, &sin); + /* persistent record of previous server before elected */ + if ((le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_FENCE].term) > + le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_STOP].term)) && + (le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_FENCE].term) < + le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_ELECT].term))) + old[i][1] = blk.events[SCOUTFS_QUORUM_EVENT_FENCE]; - scoutfs_err(sb, "fencing "SCSBF" at "SIN_FMT, - SCSB_LEFR_ARGS(super->hdr.fsid, blk.set_leader.rid), - SIN_ARG(&sin)); + /* find greatest term that has fenced everything before it */ + fenced = max(fenced, le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_FENCE].term)); + } - blk.flags &= ~cpu_to_le64(SCOUTFS_QUORUM_BLOCK_LEADER); - set_quorum_block_event(sb, &blk, &blk.fenced); + /* now actually fence any old leaders which haven't been fenced yet */ + for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { + for (j = 0; j < NR_OLD; j++) { + if (le64_to_cpu(old[i][j].term) == 0 || /* uninitialized */ + le64_to_cpu(old[i][j].term) < fenced || /* already fenced */ + le64_to_cpu(old[i][j].term) > term || /* newer than us */ + le64_to_cpu(old[i][j].rid) == rid) /* us */ + continue; - ret = write_quorum_block(sb, blkno, &blk, NULL); - if (ret < 0) - goto out; + scoutfs_inc_counter(sb, quorum_fence_leader); + scoutfs_quorum_slot_sin(super, i, &sin); + fence_rid = old[i][j].rid; + + scoutfs_info(sb, "fencing previous leader "SCSBF" at term %llu in slot %u with address "SIN_FMT, + SCSB_LEFR_ARGS(super->hdr.fsid, fence_rid), + le64_to_cpu(old[i][j].term), i, SIN_ARG(&sin)); + ret = scoutfs_fence_start(sb, le64_to_cpu(fence_rid), sin.sin_addr.s_addr, + SCOUTFS_FENCE_QUORUM_BLOCK_LEADER); + if (ret < 0) + goto out; + fence_started = true; + } } out: + if (fence_started) { + err = scoutfs_fence_wait_fenced(sb, msecs_to_jiffies(SCOUTFS_QUORUM_FENCE_TO_MS)); + if (ret == 0) + ret = err; + } else { + err = scoutfs_quorum_fence_complete(sb, term); + if (ret == 0) + ret = err; + } + if (ret < 0) { - scoutfs_err(sb, "error %d fencing active", ret); + scoutfs_err(sb, "error %d attempting to find and fence previous leaders", ret); scoutfs_inc_counter(sb, quorum_fence_error); } @@ -533,23 +580,22 @@ static void scoutfs_quorum_worker(struct work_struct *work) struct sockaddr_in unused; struct quorum_host_msg msg; struct quorum_status qst; - __le64 mark; u64 blkno; int ret; + int err; /* recording votes from slots as native single word bitmap */ BUILD_BUG_ON(SCOUTFS_QUORUM_MAX_SLOTS > BITS_PER_LONG); /* get our starting term from our persistent block */ - mark = 0; blkno = SCOUTFS_QUORUM_BLKNO + opts->quorum_slot_nr; - ret = read_quorum_block(sb, blkno, &blk, &mark); + ret = read_quorum_block(sb, blkno, &blk, false); if (ret < 0) goto out; /* start out as a follower */ qst.role = FOLLOWER; - qst.term = le64_to_cpu(blk.term); + qst.term = le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_TERM].term); qst.vote_for = -1; qst.vote_bits = 0; @@ -559,6 +605,11 @@ static void scoutfs_quorum_worker(struct work_struct *work) else qst.timeout = election_timeout(); + /* record that we're up and running, readers check that it isn't updated */ + ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_BEGIN, qst.term, false); + if (ret < 0) + goto out; + while (!qinf->shutdown) { ret = recv_msg(sb, &msg, qst.timeout); @@ -589,11 +640,6 @@ static void scoutfs_quorum_worker(struct work_struct *work) send_msg_others(sb, SCOUTFS_QUORUM_MSG_RESIGNATION, qst.term); scoutfs_inc_counter(sb, quorum_send_resignation); - - ret = update_quorum_block(sb, blkno, &mark, - qst.role, qst.term); - if (ret < 0) - goto out; } spin_lock(&qinf->show_lock); @@ -624,8 +670,7 @@ static void scoutfs_quorum_worker(struct work_struct *work) qst.timeout = election_timeout(); /* store our increased term */ - ret = update_quorum_block(sb, blkno, &mark, - qst.role, qst.term); + ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_TERM, qst.term, true); if (ret < 0) goto out; } @@ -642,6 +687,11 @@ static void scoutfs_quorum_worker(struct work_struct *work) qst.term); qst.timeout = election_timeout(); scoutfs_inc_counter(sb, quorum_send_request); + + /* store our increased term */ + ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_TERM, qst.term, true); + if (ret < 0) + goto out; } /* candidates count votes in their term */ @@ -670,10 +720,8 @@ static void scoutfs_quorum_worker(struct work_struct *work) qst.term); qst.timeout = heartbeat_interval(); - /* set our leader flag and fence */ - ret = update_quorum_block(sb, blkno, &mark, - qst.role, qst.term) ?: - fence_leader_blocks(sb); + /* record that we've been elected before starting up server */ + ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_ELECT, qst.term, true); if (ret < 0) goto out; @@ -684,8 +732,13 @@ static void scoutfs_quorum_worker(struct work_struct *work) ret = scoutfs_server_start(sb, qst.term); if (ret < 0) { - scoutfs_err(sb, "server startup failed with %d", - ret); + clear_bit(QINF_FLAG_SERVER, &qinf->flags); + scoutfs_err(sb, "server startup failed with %d", ret); + /* store our increased term */ + err = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_STOP, qst.term, + true); + if (err < 0 && ret == 0) + ret = err; goto out; } } @@ -727,17 +780,13 @@ static void scoutfs_quorum_worker(struct work_struct *work) /* always try to stop a running server as we stop */ if (test_bit(QINF_FLAG_SERVER, &qinf->flags)) { scoutfs_server_stop(sb); + scoutfs_fence_stop(sb); send_msg_others(sb, SCOUTFS_QUORUM_MSG_RESIGNATION, qst.term); } - /* always try to clear leader block as we stop to avoid fencing */ - if (qst.role == LEADER) { - ret = update_quorum_block(sb, blkno, &mark, - FOLLOWER, qst.term); - if (ret < 0) - goto out; - } + /* informational event that we're shutting down, nothing relies on it */ + update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_END, qst.term, true); out: if (ret < 0) { scoutfs_err(sb, "quorum service saw error %d, shutting down. Cluster will be degraded until this slot is remounted to restart the quorum service", @@ -746,58 +795,60 @@ out: } /* - * Set a flag for the quorum work's next iteration to indicate that the - * server has shutdown and that it should step down as leader, update - * quorum blocks, and stop sending heartbeats. + * The calling server has shutdown and is no longer using shared + * resources. Clear the bit so that we stop sending heartbeats and + * allow the next server to be elected. Update the stop event so that + * it won't be considered available by clients or fenced by the next + * leader. */ -void scoutfs_quorum_server_shutdown(struct super_block *sb) +void scoutfs_quorum_server_shutdown(struct super_block *sb, u64 term) { DECLARE_QUORUM_INFO(sb, qinf); - set_bit(QINF_FLAG_SERVER, &qinf->flags); + clear_bit(QINF_FLAG_SERVER, &qinf->flags); + update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_STOP, term, true); } /* * Clients read quorum blocks looking for the leader with a server whose * address it can try and connect to. * - * There can be multiple running servers if a client checks before a - * server has had a chance to fence any old servers. We try to use the - * block with the most recent timestamp. If we get it wrong the - * connection will timeout and the client will try again, presumably - * finding a single server block. + * There can be records of multiple previous elected leaders if the + * current server hasn't yet fenced any old servers. We use the elected + * leader with the greatest elected term. If we get it wrong the + * connection will timeout and the client will try again. */ int scoutfs_quorum_server_sin(struct super_block *sb, struct sockaddr_in *sin) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_quorum_block blk; - struct timespec64 recent = {0,}; - struct timespec64 ts; - int ret; + u64 elect_term; + u64 term = 0; + int ret = 0; int i; for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - ret = read_quorum_block(sb, SCOUTFS_QUORUM_BLKNO + i, &blk, - NULL); + if (!quorum_slot_present(super, i)) + continue; + + ret = read_quorum_block(sb, SCOUTFS_QUORUM_BLKNO + i, &blk, false); if (ret < 0) { scoutfs_err(sb, "error reading quorum block nr %u: %d", i, ret); goto out; } - ts.tv_sec = le64_to_cpu(blk.set_leader.ts.sec); - ts.tv_nsec = le32_to_cpu(blk.set_leader.ts.nsec); - - if ((le64_to_cpu(blk.flags) & SCOUTFS_QUORUM_BLOCK_LEADER) && - (timespec64_to_ns(&ts) > timespec64_to_ns(&recent))) { - recent = ts; + elect_term = le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_ELECT].term); + if (elect_term > term && + elect_term > le64_to_cpu(blk.events[SCOUTFS_QUORUM_EVENT_STOP].term)) { + term = elect_term; scoutfs_quorum_slot_sin(super, i, sin); continue; } } - if (timespec64_to_ns(&recent) == 0) + if (term == 0) ret = -ENOENT; out: diff --git a/kmod/src/quorum.h b/kmod/src/quorum.h index f0994871..1c2b6315 100644 --- a/kmod/src/quorum.h +++ b/kmod/src/quorum.h @@ -2,12 +2,15 @@ #define _SCOUTFS_QUORUM_H_ int scoutfs_quorum_server_sin(struct super_block *sb, struct sockaddr_in *sin); -void scoutfs_quorum_server_shutdown(struct super_block *sb); +void scoutfs_quorum_server_shutdown(struct super_block *sb, u64 term); u8 scoutfs_quorum_votes_needed(struct super_block *sb); void scoutfs_quorum_slot_sin(struct scoutfs_super_block *super, int i, struct sockaddr_in *sin); +int scoutfs_quorum_fence_leaders(struct super_block *sb, u64 term); +int scoutfs_quorum_fence_complete(struct super_block *sb, u64 term); + int scoutfs_quorum_setup(struct super_block *sb); void scoutfs_quorum_shutdown(struct super_block *sb); void scoutfs_quorum_destroy(struct super_block *sb); diff --git a/kmod/src/recov.c b/kmod/src/recov.c index b0d894c2..3829760b 100644 --- a/kmod/src/recov.c +++ b/kmod/src/recov.c @@ -16,9 +16,11 @@ #include #include #include +#include #include "super.h" #include "recov.h" +#include "cmp.h" /* * There are a few server messages which can't be processed until they @@ -47,18 +49,41 @@ struct recov_pending { int which; }; -static struct recov_pending *find_pending(struct recov_info *recinf, u64 rid, int which) +static struct recov_pending *next_pending(struct recov_info *recinf, u64 rid, int which) { struct recov_pending *pend; list_for_each_entry(pend, &recinf->pending, head) { - if ((rid == 0 || pend->rid == rid) && (pend->which & which)) + if (pend->rid > rid && pend->which & which) return pend; } return NULL; } +static struct recov_pending *lookup_pending(struct recov_info *recinf, u64 rid, int which) +{ + struct recov_pending *pend; + + pend = next_pending(recinf, rid - 1, which); + if (pend && pend->rid == rid) + return pend; + + return NULL; +} + +/* + * We keep the pending list sorted by rid so that we can iterate over + * them. The list should be small and shouldn't be used often. + */ +static int cmp_pending_rid(void *priv, struct list_head *A, struct list_head *B) +{ + struct recov_pending *a = list_entry(A, struct recov_pending, head); + struct recov_pending *b = list_entry(B, struct recov_pending, head); + + return scoutfs_cmp_u64s(a->rid, b->rid); +} + /* * Record that we'll be waiting for a client to recover something. * _finished will eventually be called for every _prepare, either @@ -80,14 +105,15 @@ int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which) spin_lock(&recinf->lock); - pend = find_pending(recinf, rid, SCOUTFS_RECOV_ALL); + pend = lookup_pending(recinf, rid, SCOUTFS_RECOV_ALL); if (pend) { pend->which |= which; } else { swap(pend, alloc); pend->rid = rid; pend->which = which; - list_add(&pend->head, &recinf->pending); + list_add_tail(&pend->head, &recinf->pending); + list_sort(NULL, &recinf->pending, cmp_pending_rid); } spin_unlock(&recinf->lock); @@ -159,7 +185,7 @@ int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which) spin_lock(&recinf->lock); - pend = find_pending(recinf, rid, which); + pend = lookup_pending(recinf, rid, which); if (pend) { pend->which &= ~which; if (pend->which) { @@ -190,29 +216,28 @@ bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which) bool is_pending; spin_lock(&recinf->lock); - is_pending = find_pending(recinf, rid, which) != NULL; + is_pending = lookup_pending(recinf, rid, which) != NULL; spin_unlock(&recinf->lock); return is_pending; } /* - * Returns 0 if there are no rids waiting for the given state to be - * recovered. Returns the rid of a client still waiting if there are - * any, in no specified order. + * Return the next rid after the given rid of a client waiting for the + * given state to be recovered. Start with rid 0, returns 0 when there + * are no more clients waiting for recovery. * * This is inherently racey. Callers are responsible for resolving any * actions taken based on pending with the recovery finishing, perhaps * before we return. */ -u64 scoutfs_recov_next_pending(struct super_block *sb, int which) +u64 scoutfs_recov_next_pending(struct super_block *sb, u64 rid, int which) { DECLARE_RECOV_INFO(sb, recinf); struct recov_pending *pend; - u64 rid; spin_lock(&recinf->lock); - pend = find_pending(recinf, 0, which); + pend = next_pending(recinf, rid, which); rid = pend ? pend->rid : 0; spin_unlock(&recinf->lock); diff --git a/kmod/src/recov.h b/kmod/src/recov.h index cfdca30e..9be58b3a 100644 --- a/kmod/src/recov.h +++ b/kmod/src/recov.h @@ -14,7 +14,7 @@ int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_ unsigned int timeout_ms); int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which); bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which); -u64 scoutfs_recov_next_pending(struct super_block *sb, int which); +u64 scoutfs_recov_next_pending(struct super_block *sb, u64 rid, int which); void scoutfs_recov_shutdown(struct super_block *sb); int scoutfs_recov_setup(struct super_block *sb); diff --git a/kmod/src/server.c b/kmod/src/server.c index 884ec50d..4eeefccd 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -40,6 +40,7 @@ #include "forest.h" #include "recov.h" #include "omap.h" +#include "fence.h" /* * Every active mount can act as the server that listens on a net @@ -106,6 +107,8 @@ struct server_info { /* recovery timeout fences from work */ struct work_struct fence_pending_recov_work; + /* while running we check for fenced mounts to reclaim */ + struct delayed_work reclaim_dwork; }; #define DECLARE_SERVER_INFO(sb, name) \ @@ -157,10 +160,21 @@ struct commit_waiter { int ret; }; +static bool test_shutting_down(struct server_info *server) +{ + smp_rmb(); + return server->shutting_down; +} + +static void set_shutting_down(struct server_info *server, bool val) +{ + server->shutting_down = val; + smp_rmb(); +} + static void stop_server(struct server_info *server) { - /* wait_event/wake_up provide barriers */ - server->shutting_down = true; + set_shutting_down(server, true); wake_up(&server->waitq); } @@ -284,6 +298,11 @@ static void scoutfs_server_commit_func(struct work_struct *work) down_write(&server->commit_rwsem); + if (scoutfs_forcing_unmount(sb)) { + ret = -EIO; + goto out; + } + /* make sure next avail has sufficient blocks */ ret = scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri, server->other_avail, @@ -1434,14 +1453,19 @@ static void init_mounted_client_key(struct scoutfs_key *key, u64 rid) }; } +static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) +{ + return (iref->val_len != sizeof(struct scoutfs_mounted_client_btree_val)); +} + /* * Insert a new mounted client item for a client that is sending us a * greeting that hasn't yet seen a response. The greeting can be * retransmitted to a new server after the previous inserted the item so * it's acceptable to see -EEXIST. */ -static int insert_mounted_client(struct super_block *sb, u64 rid, - u64 gr_flags) +static int insert_mounted_client(struct super_block *sb, u64 rid, u64 gr_flags, + struct sockaddr_in *sin) { DECLARE_SERVER_INFO(sb, server); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; @@ -1450,6 +1474,7 @@ static int insert_mounted_client(struct super_block *sb, u64 rid, int ret; init_mounted_client_key(&key, rid); + scoutfs_sin_to_addr(&mcv.addr, sin); mcv.flags = 0; if (gr_flags & SCOUTFS_NET_GREETING_FLAG_QUORUM) mcv.flags |= SCOUTFS_MOUNTED_CLIENT_QUORUM; @@ -1465,6 +1490,34 @@ static int insert_mounted_client(struct super_block *sb, u64 rid, return ret; } +static int lookup_mounted_client_addr(struct super_block *sb, u64 rid, + union scoutfs_inet_addr *addr) +{ + DECLARE_SERVER_INFO(sb, server); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_mounted_client_btree_val *mcv; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + int ret; + + init_mounted_client_key(&key, rid); + + mutex_lock(&server->mounted_clients_mutex); + ret = scoutfs_btree_lookup(sb, &super->mounted_clients, &key, &iref); + if (ret == 0) { + if (invalid_mounted_client_item(&iref)) { + ret = -EIO; + } else { + mcv = iref.val; + *addr = mcv->addr; + } + scoutfs_btree_put_iref(&iref); + } + mutex_unlock(&server->mounted_clients_mutex); + + return ret; +} + /* * Remove the record of a mounted client. The record can already be * removed if we're processing a farewell on behalf of a client that @@ -1542,7 +1595,7 @@ static int cancel_srch_compact(struct super_block *sb, u64 rid) */ static void queue_farewell_work(struct server_info *server) { - if (!server->shutting_down) + if (!test_shutting_down(server)) queue_work(server->wq, &server->farewell_work); } @@ -1603,8 +1656,8 @@ static int server_greeting(struct super_block *sb, if (ret < 0) goto send_err; - ret = insert_mounted_client(sb, le64_to_cpu(gr->rid), - le64_to_cpu(gr->flags)); + ret = insert_mounted_client(sb, le64_to_cpu(gr->rid), le64_to_cpu(gr->flags), + &conn->peername); ret = scoutfs_server_apply_commit(sb, ret); queue_work(server->wq, &server->farewell_work); @@ -1661,12 +1714,15 @@ struct farewell_request { u64 rid; }; -static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) -{ - return (iref->val_len != - sizeof(struct scoutfs_mounted_client_btree_val)); -} +/* + * Reclaim all the resources for a mount which has gone away. It's sent + * us a farewell promising to leave or we actively fenced it. + * + * It's safe to call this multiple times for a given rid. Each + * individual action knows to recognize that it's already been performed + * and return success. + */ static int reclaim_rid(struct super_block *sb, u64 rid) { int ret; @@ -1675,13 +1731,13 @@ static int reclaim_rid(struct super_block *sb, u64 rid) if (ret < 0) return ret; - /* delete mounted client last, client reconnect looks for it */ + /* delete mounted client last, recovery looks for it */ ret = scoutfs_lock_server_farewell(sb, rid) ?: remove_trans_seq(sb, rid) ?: reclaim_log_trees(sb, rid) ?: cancel_srch_compact(sb, rid) ?: - delete_mounted_client(sb, rid) ?: - scoutfs_omap_remove_rid(sb, rid); + scoutfs_omap_remove_rid(sb, rid) ?: + delete_mounted_client(sb, rid); return scoutfs_server_apply_commit(sb, ret); } @@ -1811,7 +1867,7 @@ static void farewell_worker(struct work_struct *work) } } - /* process and send farewell responses */ + /* clean up resources for mounts before sending responses */ list_for_each_entry_safe(fw, tmp, &send, entry) { ret = reclaim_rid(sb, fw->rid); if (ret) @@ -2012,29 +2068,37 @@ static void fence_pending_recov_worker(struct work_struct *work) struct server_info *server = container_of(work, struct server_info, fence_pending_recov_work); struct super_block *sb = server->sb; - u64 rid; - int ret; + union scoutfs_inet_addr addr; + u64 rid = 0; + int ret = 0; - while ((rid = scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_ALL)) > 0) { + while ((rid = scoutfs_recov_next_pending(sb, rid, SCOUTFS_RECOV_ALL)) > 0) { scoutfs_err(sb, "%lu ms recovery timeout expired for client rid %016llx, fencing", SERVER_RECOV_TIMEOUT_MS, rid); - ret = reclaim_rid(sb, rid); + ret = lookup_mounted_client_addr(sb, rid, &addr); if (ret < 0) { - scoutfs_err(sb, "error %d reclaiming rid %016llx, shutting down", ret, rid); - stop_server(server); + scoutfs_err(sb, "client rid addr lookup err %d, shutting down server", ret); break; } - scoutfs_server_recov_finish(sb, rid, SCOUTFS_RECOV_ALL); + ret = scoutfs_fence_start(sb, rid, le32_to_be32(addr.v4.addr), + SCOUTFS_FENCE_CLIENT_RECOVERY); + if (ret < 0) { + scoutfs_err(sb, "fence returned err %d, shutting down server", ret); + break; + } } + + if (ret < 0) + scoutfs_server_abort(sb); } static void recovery_timeout(struct super_block *sb) { DECLARE_SERVER_INFO(sb, server); - if (!server->shutting_down) + if (!test_shutting_down(server)) queue_work(server->wq, &server->fence_pending_recov_work); } @@ -2097,6 +2161,78 @@ out: return ret; } +static void queue_reclaim_work(struct server_info *server, unsigned long delay) +{ + if (!test_shutting_down(server)) + queue_delayed_work(server->wq, &server->reclaim_dwork, delay); +} + +#define RECLAIM_WORK_DELAY_MS MSEC_PER_SEC + +/* + * Fencing is performed by userspace and can happen as we're elected + * leader before the server is running. Once we're running we want to + * reclaim resources from any mounts that may have been fenced. + * + * The reclaim worker runs regularly in the background and reclaims the + * resources for mounts that have been fenced. Once the fenced rid has + * been reclaimed the fence request can be removed. + * + * This is queued by the server work as it starts up, requeues itself + * until shutdown, and is then canceled by the server work as it shuts + * down. + */ +static void reclaim_worker(struct work_struct *work) +{ + struct server_info *server = container_of(work, struct server_info, reclaim_dwork.work); + struct super_block *sb = server->sb; + bool error; + int reason; + u64 rid; + int ret; + + ret = scoutfs_fence_next(sb, &rid, &reason, &error); + if (ret < 0) + goto out; + + if (error == true) { + scoutfs_err(sb, "saw error indicator on fence request for rid %016llx, shutting down server", + rid); + scoutfs_server_abort(sb); + ret = -ESHUTDOWN; + goto out; + } + + ret = reclaim_rid(sb, rid); + if (ret < 0) { + scoutfs_err(sb, "failure to reclaim fenced rid %016llx: err %d, shutting down server", + rid, ret); + scoutfs_server_abort(sb); + goto out; + } + + scoutfs_info(sb, "successfully reclaimed resources for fenced rid %016llx", rid); + scoutfs_fence_free(sb, rid); + scoutfs_server_recov_finish(sb, rid, SCOUTFS_RECOV_ALL); + + /* tell quorum we've finished fencing all previous leaders */ + if (reason == SCOUTFS_FENCE_QUORUM_BLOCK_LEADER && + !scoutfs_fence_reason_pending(sb, reason)) { + ret = scoutfs_quorum_fence_complete(sb, server->term); + if (ret < 0) + goto out; + } + + ret = 0; + +out: + /* queue next reclaim immediately if we're making progress */ + if (ret == 0) + queue_reclaim_work(server, 0); + else + queue_reclaim_work(server, msecs_to_jiffies(RECLAIM_WORK_DELAY_MS)); +} + static void scoutfs_server_worker(struct work_struct *work) { struct server_info *server = container_of(work, struct server_info, @@ -2113,6 +2249,11 @@ static void scoutfs_server_worker(struct work_struct *work) trace_scoutfs_server_work_enter(sb, 0, 0); + /* first make sure no other servers are still running */ + ret = scoutfs_quorum_fence_leaders(sb, server->term); + if (ret < 0) + goto out; + scoutfs_quorum_slot_sin(super, opts->quorum_slot_nr, &sin); scoutfs_info(sb, "server setting up at "SIN_FMT, SIN_ARG(&sin)); @@ -2184,14 +2325,17 @@ static void scoutfs_server_worker(struct work_struct *work) scoutfs_info(sb, "server ready at "SIN_FMT, SIN_ARG(&sin)); complete(&server->start_comp); + queue_reclaim_work(server, 0); + /* wait_event/wake_up provide barriers */ - wait_event_interruptible(server->waitq, server->shutting_down); + wait_event_interruptible(server->waitq, test_shutting_down(server)); shutdown: scoutfs_info(sb, "server shutting down at "SIN_FMT, SIN_ARG(&sin)); /* wait for farewell to finish sending messages */ flush_work(&server->farewell_work); + cancel_delayed_work_sync(&server->reclaim_dwork); /* wait for requests to finish, no more requests */ scoutfs_net_shutdown(sb, conn); @@ -2204,6 +2348,7 @@ shutdown: /* wait for extra queues by requests, won't find waiters */ flush_work(&server->commit_work); + scoutfs_fence_stop(sb); scoutfs_lock_server_destroy(sb); scoutfs_omap_server_shutdown(sb); @@ -2211,7 +2356,7 @@ out: scoutfs_net_free_conn(sb, conn); /* let quorum know that we've shutdown */ - scoutfs_quorum_server_shutdown(sb); + scoutfs_quorum_server_shutdown(sb, server->term); scoutfs_info(sb, "server stopped at "SIN_FMT, SIN_ARG(&sin)); trace_scoutfs_server_work_exit(sb, 0, ret); @@ -2230,7 +2375,7 @@ int scoutfs_server_start(struct super_block *sb, u64 term) DECLARE_SERVER_INFO(sb, server); server->err = 0; - server->shutting_down = false; + set_shutting_down(server, false); server->term = term; init_completion(&server->start_comp); @@ -2294,6 +2439,7 @@ int scoutfs_server_setup(struct super_block *sb) seqcount_init(&server->volopt_seqcount); mutex_init(&server->volopt_mutex); INIT_WORK(&server->fence_pending_recov_work, fence_pending_recov_worker); + INIT_DELAYED_WORK(&server->reclaim_dwork, reclaim_worker); server->wq = alloc_workqueue("scoutfs_server", WQ_UNBOUND | WQ_NON_REENTRANT, 0); diff --git a/kmod/src/super.c b/kmod/src/super.c index d5d1063f..19e1503f 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -47,6 +47,7 @@ #include "recov.h" #include "omap.h" #include "volopt.h" +#include "fence.h" #include "scoutfs_trace.h" static struct dentry *scoutfs_debugfs_root; @@ -270,6 +271,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_block_destroy(sb); scoutfs_destroy_triggers(sb); + scoutfs_fence_destroy(sb); scoutfs_options_destroy(sb); scoutfs_sysfs_destroy_attrs(sb, &sbi->mopts_ssa); debugfs_remove(sbi->debug_root); @@ -283,6 +285,21 @@ static void scoutfs_put_super(struct super_block *sb) sb->s_fs_info = NULL; } +/* + * Record that we're performing a forced unmount. As put_super drives + * destruction of the filesystem we won't issue more network or storage + * operations because we assume that they'll hang. Pending operations + * can return errors when it's possible to do so. We may be racing with + * pending operations which can't be canceled. + */ +static void scoutfs_umount_begin(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + scoutfs_warn(sb, "forcing unmount, can return errors and lose unsynced data"); + sbi->forced_unmount = true; +} + static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, .drop_inode = scoutfs_drop_inode, @@ -292,6 +309,7 @@ static const struct super_operations scoutfs_super_ops = { .statfs = scoutfs_statfs, .show_options = scoutfs_show_options, .put_super = scoutfs_put_super, + .umount_begin = scoutfs_umount_begin, }; /* @@ -590,6 +608,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_sysfs_create_attrs(sb, &sbi->mopts_ssa, mount_options_attrs, "mount_options") ?: scoutfs_setup_triggers(sb) ?: + scoutfs_fence_setup(sb) ?: scoutfs_block_setup(sb) ?: scoutfs_forest_setup(sb) ?: scoutfs_item_setup(sb) ?: diff --git a/kmod/src/super.h b/kmod/src/super.h index 820ee6b8..e44d6575 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -29,6 +29,7 @@ struct srch_info; struct recov_info; struct omap_info; struct volopt_info; +struct fence_info; struct scoutfs_sb_info { struct super_block *sb; @@ -54,6 +55,7 @@ struct scoutfs_sb_info { struct omap_info *omap_info; struct volopt_info *volopt_info; struct item_cache_info *item_cache_info; + struct fence_info *fence_info; wait_queue_head_t trans_hold_wq; struct task_struct *trans_task; @@ -87,6 +89,8 @@ struct scoutfs_sb_info { struct dentry *debug_root; + bool forced_unmount; + unsigned long corruption_messages_once[SC_NR_LONGS]; }; @@ -107,6 +111,13 @@ static inline bool SCOUTFS_IS_META_BDEV(struct scoutfs_super_block *super_block) #define SCOUTFS_META_BDEV_MODE (FMODE_READ | FMODE_WRITE | FMODE_EXCL) +static inline bool scoutfs_forcing_unmount(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + return sbi->forced_unmount; +} + /* * A small string embedded in messages that's used to identify a * specific mount. It's the three most significant bytes of the fsid diff --git a/kmod/src/sysfs.c b/kmod/src/sysfs.c index bea667d0..5f2f024c 100644 --- a/kmod/src/sysfs.c +++ b/kmod/src/sysfs.c @@ -131,9 +131,10 @@ void scoutfs_sysfs_init_attrs(struct super_block *sb, * If this returns success then the file will be visible and show can * be called until unmount. */ -int scoutfs_sysfs_create_attrs(struct super_block *sb, - struct scoutfs_sysfs_attrs *ssa, - struct attribute **attrs, char *fmt, ...) +int scoutfs_sysfs_create_attrs_parent(struct super_block *sb, + struct kobject *parent, + struct scoutfs_sysfs_attrs *ssa, + struct attribute **attrs, char *fmt, ...) { va_list args; size_t name_len; @@ -174,8 +175,8 @@ int scoutfs_sysfs_create_attrs(struct super_block *sb, goto out; } - ret = kobject_init_and_add(&ssa->kobj, &ssa->ktype, - scoutfs_sysfs_sb_dir(sb), "%s", ssa->name); + ret = kobject_init_and_add(&ssa->kobj, &ssa->ktype, parent, + "%s", ssa->name); out: if (ret) { kfree(ssa->name); diff --git a/kmod/src/sysfs.h b/kmod/src/sysfs.h index 73788c00..2d218bba 100644 --- a/kmod/src/sysfs.h +++ b/kmod/src/sysfs.h @@ -10,6 +10,8 @@ #define SCOUTFS_ATTR_RO(_name) \ static struct kobj_attribute scoutfs_attr_##_name = __ATTR_RO(_name) +#define SCOUTFS_ATTR_RW(_name) \ + static struct kobj_attribute scoutfs_attr_##_name = __ATTR_RW(_name) #define SCOUTFS_ATTR_PTR(_name) \ &scoutfs_attr_##_name.attr @@ -34,9 +36,14 @@ struct scoutfs_sysfs_attrs { void scoutfs_sysfs_init_attrs(struct super_block *sb, struct scoutfs_sysfs_attrs *ssa); -int scoutfs_sysfs_create_attrs(struct super_block *sb, - struct scoutfs_sysfs_attrs *ssa, - struct attribute **attrs, char *fmt, ...); +int scoutfs_sysfs_create_attrs_parent(struct super_block *sb, + struct kobject *parent, + struct scoutfs_sysfs_attrs *ssa, + struct attribute **attrs, char *fmt, ...); +#define scoutfs_sysfs_create_attrs(sb, ssa, attrs, fmt, args...) \ + scoutfs_sysfs_create_attrs_parent(sb, scoutfs_sysfs_sb_dir(sb), \ + ssa, attrs, fmt, ##args) + void scoutfs_sysfs_destroy_attrs(struct super_block *sb, struct scoutfs_sysfs_attrs *ssa); diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 186239d4..07eea0fa 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -185,6 +185,11 @@ void scoutfs_trans_write_func(struct work_struct *work) wait_event(sbi->trans_hold_wq, drained_holders(tri)); + if (scoutfs_forcing_unmount(sb)) { + ret = -EIO; + goto out; + } + trace_scoutfs_trans_write_func(sb, scoutfs_block_writer_dirty_bytes(sb, &tri->wri)); @@ -202,7 +207,7 @@ void scoutfs_trans_write_func(struct work_struct *work) if (ret < 0) s = "clean advance seq"; } - goto out; + goto err; } if (sbi->trans_deadline_expired) @@ -222,11 +227,12 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_item_write_done(sb) ?: (s = "advance seq", scoutfs_client_advance_seq(sb, &trans_seq)) ?: (s = "get log trees", scoutfs_trans_get_log_trees(sb)); -out: +err: if (ret < 0) scoutfs_err(sb, "critical transaction commit failure: %s, %d", s, ret); +out: spin_lock(&sbi->trans_write_lock); sbi->trans_write_count++; sbi->trans_write_ret = ret; diff --git a/tests/.gitignore b/tests/.gitignore index b6f5b65c..d8268d17 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -4,3 +4,4 @@ src/dumb_setxattr src/handle_cat src/bulk_create_paths src/find_xattrs +src/stage_tmpfile diff --git a/tests/funcs/filter.sh b/tests/funcs/filter.sh index 5c793be4..8f146e34 100644 --- a/tests/funcs/filter.sh +++ b/tests/funcs/filter.sh @@ -62,5 +62,15 @@ t_filter_dmesg() # in debugging kernels we can slow things down a bit re="$re|hrtimer: interrupt took .*" + # fencing tests force unmounts and trigger timeouts + re="$re|scoutfs .* forcing unmount" + re="$re|scoutfs .* reconnect timed out" + re="$re|scoutfs .* recovery timeout expired" + re="$re|scoutfs .* fencing previous leader" + re="$re|scoutfs .* reclaimed resources" + re="$re|scoutfs .* quorum .* error" + re="$re|scoutfs .* error reading quorum block" + re="$re|scoutfs .* error .* writing quorum block" + egrep -v "($re)" } diff --git a/tests/funcs/fs.sh b/tests/funcs/fs.sh index 03fb9888..b68bcfe7 100644 --- a/tests/funcs/fs.sh +++ b/tests/funcs/fs.sh @@ -17,6 +17,17 @@ t_sync_seq_index() t_quiet sync } +t_mount_rid() +{ + local nr="${1:-0}" + local mnt="$(eval echo \$T_M$nr)" + local rid + + rid=$(scoutfs statfs -s rid -p "$mnt") + + echo "$rid" +} + # # Output the "f.$fsid.r.$rid" identifier string for the given mount # number, 0 is used by default if none is specified. @@ -132,6 +143,16 @@ t_umount() eval t_quiet umount \$T_M$nr } +t_force_umount() +{ + local nr="$1" + + test "$nr" -lt "$T_NR_MOUNTS" || \ + t_fail "fs nr $nr invalid" + + eval t_quiet umount -f \$T_M$nr +} + # # Attempt to mount all the configured mounts, assuming that they're # not already mounted. @@ -277,3 +298,67 @@ t_counter_diff_changed() { echo "counter $which didn't change" || echo "counter $which changed" } + +# +# See if we can find a local mount with the caller's rid. +# +t_rid_is_mounted() { + local rid="$1" + local fr="$1" + + for fr in /sys/fs/scoutfs/*; do + if [ "$(cat $fr/rid)" == "$rid" ]; then + return 0 + fi + done + + return 1 +} + +# +# A given mount is being fenced if any mount has a fence request pending +# for it which hasn't finished and been removed. +# +t_rid_is_fencing() { + local rid="$1" + local fr + + for fr in /sys/fs/scoutfs/*; do + if [ -d "$fr/fence/$rid" ]; then + return 0 + fi + done + + return 1 +} + +# +# Wait until the mount identified by the first rid arg is not in any +# states specified by the remaining state description word args. +# +t_wait_if_rid_is() { + local rid="$1" + + while ( [[ $* =~ mounted ]] && t_rid_is_mounted $rid ) || + ( [[ $* =~ fencing ]] && t_rid_is_fencing $rid ) ; do + sleep .5 + done +} + +# +# Wait until any mount identifies itself as the elected leader. We can +# be waiting while tests mount and unmount so mounts may not be mounted +# at the test's expected mount points. +# +t_wait_for_leader() { + local i + + while sleep .25; do + for i in $(t_fs_nrs); do + local ldr="$(t_sysfs_path $i 2>/dev/null)/quorum/is_leader" + if [ "$(cat $ldr 2>/dev/null)" == "1" ]; then + return + fi + done + done +} diff --git a/tests/golden/fence-and-reclaim b/tests/golden/fence-and-reclaim new file mode 100644 index 00000000..dadc5edd --- /dev/null +++ b/tests/golden/fence-and-reclaim @@ -0,0 +1,5 @@ +== make sure all mounts can see each other +== force unmount one client, connection timeout, fence nop, mount +== force unmount all non-server, connection timeout, fence nop, mount +== force unmount server, quorum elects new leader, fence nop, mount +== force unmount everything, new server fences all previous diff --git a/tests/run-tests.sh b/tests/run-tests.sh index 1caeeb44..d60865c1 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -18,10 +18,15 @@ die() { exit 1 } +timestamp() +{ + date '+%F %T.%N' +} + # output a message with a timestamp to the run.log log() { - echo "[$(date '+%F %T.%N')] $*" >> "$T_RESULTS/run.log" + echo "[$(timestamp)] $*" >> "$T_RESULTS/run.log" } # run a logged command, exiting if it fails @@ -367,6 +372,39 @@ cmd cat /sys/kernel/debug/tracing/set_event cmd grep . /sys/kernel/debug/tracing/options/trace_printk \ /proc/sys/kernel/ftrace_dump_on_oops +# +# Build a fenced config that runs scripts out of the repository rather +# than the default system directory +# +conf="$T_RESULTS/scoutfs-fencd.conf" +cat > $conf << EOF +SCOUTFS_FENCED_DELAY=1 +SCOUTFS_FENCED_RUN=$T_UTILS/fenced/local-force-unmount +SCOUTFS_FENCED_RUN_ARGS="" +EOF +export SCOUTFS_FENCED_CONFIG_FILE="$conf" + +# +# Run the agent in the background, log its output, an kill it if we +# exit +# +fenced_log() +{ + echo "[$(timestamp)] $*" >> "$T_RESULTS/fenced.stdout.log" +} +fenced_pid="" +kill_fenced() +{ + if test -n "$fenced_pid" -a -d "/proc/$fenced_pid" ; then + fenced_log "killing fenced pid $fenced_pid" + kill "$fenced_pid" + fi +} +trap kill_fenced EXIT +$T_UTILS/fenced/scoutfs-fenced > "$T_RESULTS/fenced.stdout.log" 2> "$T_RESULTS/fenced.stderr.log" & +fenced_pid=$! +fenced_log "started fenced pid $fenced_pid in the background" + # # mount concurrently so that a quorum is present to elect the leader and # start a server. diff --git a/tests/sequence b/tests/sequence index 972bebe2..0d709c1d 100644 --- a/tests/sequence +++ b/tests/sequence @@ -28,6 +28,7 @@ lock-conflicting-batch-commit.sh cross-mount-data-free.sh persistent-item-vers.sh setup-error-teardown.sh +fence-and-reclaim.sh mount-unmount-race.sh createmany-parallel-mounts.sh archive-light-cycle.sh diff --git a/tests/tests/fence-and-reclaim.sh b/tests/tests/fence-and-reclaim.sh new file mode 100644 index 00000000..1ce52048 --- /dev/null +++ b/tests/tests/fence-and-reclaim.sh @@ -0,0 +1,127 @@ +# +# Fence nodes and reclaim their resources. +# + +t_require_commands sleep touch grep sync scoutfs +t_require_mounts 2 + +# +# Make sure that all mounts can read the results of a write from each +# mount. And make sure that the greatest of all the written seqs is +# visible after the writes were commited by remote reads. +# +check_read_write() +{ + local expected + local greatest=0 + local seq + local path + local saw + local w + local r + + for w in $(t_fs_nrs); do + expected="$w wrote at $(date --rfc-3339=ns)" + eval path="\$T_D${w}/written" + echo "$expected" > "$path" + + seq=$(scoutfs stat -s meta_seq $path) + if [ "$seq" -gt "$greatest" ]; then + greatest=$seq + fi + + for r in $(t_fs_nrs); do + eval path="\$T_D${r}/written" + saw=$(cat "$path") + if [ "$saw" != "$expected" ]; then + echo "mount $r read '$saw' after mount $w wrote '$expected'" + fi + done + done + + seq=$(scoutfs statfs -s committed_seq -p $T_D0) + if [ "$seq" -lt "$greatest" ]; then + echo "committed_seq $seq less than greatest $greatest" + fi +} + +echo "== make sure all mounts can see each other" +check_read_write + +echo "== force unmount one client, connection timeout, fence nop, mount" +cl=$(t_first_client_nr) +sv=$(t_server_nr) +rid=$(t_mount_rid $cl) +echo "cl $cl sv $sv rid $rid" >> "$T_TMP.log" +sync +t_force_umount $cl +# wait for client reconnection to timeout +while grep -q $rid $(t_debugfs_path $sv)/connections; do + sleep .5 +done +while t_rid_is_fencing $rid; do + sleep .5 +done +t_mount $cl +check_read_write + +echo "== force unmount all non-server, connection timeout, fence nop, mount" +sv=$(t_server_nr) +pattern="nonsense" +sync +for cl in $(t_fs_nrs); do + if [ $cl == $sv ]; then + continue; + fi + + rid=$(t_mount_rid $cl) + pattern="$pattern|$rid" + echo "cl $cl sv $sv rid $rid" >> "$T_TMP.log" + + t_force_umount $cl +done + +# wait for all client reconnections to timeout +while egrep -q "($pattern)" $(t_debugfs_path $sv)/connections; do + sleep .5 +done +# wait for all fence requests to complete +while test -d $(echo /sys/fs/scoutfs/*/fence/* | cut -d " " -f 1); do + sleep .5 +done +# remount all the clients +for cl in $(t_fs_nrs); do + if [ $cl == $sv ]; then + continue; + fi + t_mount $cl +done +check_read_write + +echo "== force unmount server, quorum elects new leader, fence nop, mount" +sv=$(t_server_nr) +rid=$(t_mount_rid $sv) +echo "sv $sv rid $rid" >> "$T_TMP.log" +sync +t_force_umount $sv +t_wait_for_leader +# wait until new server is done fencing unmounted leader rid +while t_rid_is_fencing $rid; do + sleep .5 +done +t_mount $sv +check_read_write + +echo "== force unmount everything, new server fences all previous" +sync +for nr in $(t_fs_nrs); do + t_force_umount $nr +done +t_mount_all +# wait for all fence requests to complete +while test -d $(echo /sys/fs/scoutfs/*/fence/* | cut -d " " -f 1); do + sleep .5 +done +check_read_write + +t_pass diff --git a/utils/fenced/local-force-unmount b/utils/fenced/local-force-unmount new file mode 100755 index 00000000..9d97a79b --- /dev/null +++ b/utils/fenced/local-force-unmount @@ -0,0 +1,35 @@ +#!/usr/bin/bash + +echo_fail() { + echo "$@" > /dev/stderr + exit 1 +} + +rid="$SCOUTFS_FENCED_REQ_RID" + +# +# Look for a local mount with the rid to fence. Typically we'll at +# least find the mount with the server that requested the fence that +# we're processing. But it's possible that mounts are unmounted +# before, or while, we're running. +# +mnts=$(findmnt -l -n -t scoutfs -o TARGET) || \ + echo_fail "findmnt -t scoutfs failed" > /dev/stderr + +for mnt in $mnts; do + mnt_rid=$(scoutfs statfs -p "$mnt" -s rid) || \ + echo_fail "scoutfs statfs $mnt failed" + + if [ "$mnt_rid" == "$rid" ]; then + umount -f "$mnt" || \ + echo_fail "umout -f $mnt" + + exit 0 + fi +done + +# +# If the mount doesn't exist on this host then it can't access the +# devices by definition and can be considered fenced. +# +exit 0 diff --git a/utils/fenced/scoutfs-fenced b/utils/fenced/scoutfs-fenced new file mode 100755 index 00000000..2037de38 --- /dev/null +++ b/utils/fenced/scoutfs-fenced @@ -0,0 +1,94 @@ +#!/usr/bin/bash + +message_output() +{ + printf "[%s] %s\n" "$(date '+%F %T.%N')" "$@" +} + +error_message() +{ + message_output "$@" >> /dev/stderr +} + +error_exit() +{ + error_message "$@, exiting" + exit 1 +} + +log_message() +{ + message_output "$@" >> /dev/stdout +} + +# restart if we catch hup to re-read the config +hup_restart() +{ + log_message "caught SIGHUP, restarting" + exec "$@" +} +trap hup_restart SIGHUP + +# defaults +SCOUTFS_FENCED_CONFIG_FILE=${SCOUTFS_FENCED_CONFIG_FILE:-/etc/scoutfs/scoutfs-fenced.conf} +SCOUTFS_FENCED_DELAY=2 +#SCOUTFS_FENCED_RUN +#SCOUTFS_FENCED_RUN_ARGS + +test -n "$SCOUTFS_FENCED_CONFIG_FILE" || \ + error_exit "SCOUTFS_FENCED_CONFIG_FILE isn't set" +test -r "$SCOUTFS_FENCED_CONFIG_FILE" || \ + error_exit "SCOUTFS_FENCED_CONFIG_FILE isn't readable file" + +log_message "reading config file $SCOUTFS_FENCED_CONFIG_FILE" + +. "$SCOUTFS_FENCED_CONFIG_FILE" || \ + error_exit "error sourcing $SCOUTFS_FENCED_CONFIG_FILE as bash script" + +for conf in "${!SCOUTFS_FENCED_@}"; do + log_message " config var $conf=${!conf}" +done + +test -n "$SCOUTFS_FENCED_RUN" || \ + error_exit "SCOUTFS_FENCED_RUN must be set" +test -x "$SCOUTFS_FENCED_RUN" || \ + error_exit "SCOUTFS_FENCED_RUN '$SCOUTFS_FENCED_RUN' isn't executable" + +# +# main loop watching for fence request across all filesystems +# + +while sleep $SCOUTFS_FENCED_DELAY; do + for fence in /sys/fs/scoutfs/*/fence/*; do + # catches unmatched regex when no dirs + if [ ! -d "$fence" ]; then + continue + fi + + # skip requests that have been handled + if [ $(cat "$fence/fenced") == 1 -o $(cat "$fence/error") == 1 ]; then + continue + fi + + srv=$(basename $(dirname $(dirname $fence))) + rid="$(cat $fence/rid)" + ip="$(cat $fence/ipv4_addr)" + reason="$(cat $fence/reason)" + + log_message "server $srv fencing rid $rid at IP $ip for $reason" + + # export _REQ_ vars for run to use + export SCOUTFS_FENCED_REQ_RID="$rid" + export SCOUTFS_FENCED_REQ_IP="$ip" + + $run $SCOUTFS_FENCED_RUN_ARGS + rc=$? + if [ "$rc" != 0 ]; then + log_message "server $srv fencing rid $rid saw error status $rc from $run" + echo 1 > "$fence/error" + continue + fi + + echo 1 > "$fence/fenced" + done +done diff --git a/utils/man/scoutfs-fenced.8 b/utils/man/scoutfs-fenced.8 new file mode 100644 index 00000000..4be5348e --- /dev/null +++ b/utils/man/scoutfs-fenced.8 @@ -0,0 +1,66 @@ +.TH scoutfs-fenced 8 +.SH NAME +scoutfs-fenced \- scoutfs fence request monitoring and dispatch daemon +.SH DESCRIPTION +The +.B scoutfs-fenced +daemon runs on hosts with mounts that are configured as quorum members +and could create fence requests. It watches sysfs directories of +mounted scoutfs volumes for the directories store requests +to fence a mount. + +.SH ENVIRONMENT +scoutfs-fenced reads the +.I SCOUTFS_FENCED_CONFIG_FILE +environment variable for the path to the config file that contains its +configuration. The file must be readable and is sourced as a bash +script and is expected to set the following configuration variables. + +.SH CONFIGURATION + +.TP +.B SCOUTFS_FENCED_DELAY +The number of seconds to wait beteween checking for fence request +directories in the sysfs directories of all mounts on the host. + +.TP +.B SCOUTFS_FENCED_RUN +The path to the command to execute for each fence request. The file at +the path must be executable. + +.TP +.B SCOUTFS_FENCED_RUN_ARGS +The arguments that are unconditionally passed through to the run +command. + +.SH DAEMONIZING AND LOGGING + +scoutfs-fenced runs in the foreground and writes to stderr and stdout. +Disconnecting it from parents and redirecting its output are the +responsibility of the host environment. + +.SH RUN COMMAND INTERFACE + +scoutfs-fenced sets enviroment variables for the run command with +information about the mount that must be fenced: + +.TP +.B SCOUTFS_FENCED_REQ_RID +The RID of the mount to be fenced. +.TP +.B SCOUTFS_FENCED_REQ_IP +The dotted quad IPv4 address of the last connection from the mount. + +.RE +The return status of the run command indicates if the mount was +fenced, or not. If the mount was successfully fenced then the command +should return a 0 success status. If the run command returns a non-zero +failure status then the request will be set as errored and the server +will shut down. The next server that starts will create another fence +request for the mount. + +.SH SEE ALSO +.BR scoutfs (5), + +.SH AUTHORS +Zach Brown diff --git a/utils/man/scoutfs.5 b/utils/man/scoutfs.5 index c489a5d5..150eb1fb 100644 --- a/utils/man/scoutfs.5 +++ b/utils/man/scoutfs.5 @@ -1,6 +1,6 @@ .TH scoutfs 5 .SH NAME -scoutfs \- overview and mount options for the scoutfs filesystem +scoutfs \- high level overview of the scoutfs filesystem .SH DESCRIPTION A scoutfs filesystem is stored on two block devices. Multiple mounts of the filesystem are supported between hosts that share access to the @@ -68,7 +68,82 @@ in that zone. When the mount is not in a zone, or its zone has no more free extents, the server will try and find free extents in a zone that no other mount currently occupies. The result is to try and produce write streams where only one mount is writing into each zone. -.SH FURTHER READING +.SH FENCING +.B scoutfs +mounts coordinate exclusive access to shared resources through +comminication with the mount that was elected leader. +A mount can malfunction and stop participating at which point it needs +to be safely isolated ("fenced off") from shared resources before other mounts can +have their turn at exclusive access. +.sp +Only the elected leader can fence mounts. As the leader decides that a +mount must be fenced, typically by timeouts expiring without +comminication from the mount, it creates a fence request. Fence +requests are visible as directories in the leader mount's sysfs +directory. The fence request directory is named for the RID of the +mount being fenced. The directory contains the following files: + +.RS +.TP +.B elapsec_secs +Reading this file gives the number of seconds that have passed since +this fence request was created. +.TP +.B error +This file contains 0 when the fence request is created. Userspace +fencing agents write 1 into this file if they are unable to fence the +mount. The volume can not make progress until the mount is fenced so +this will cause the server to stop and another mount will be elected +leader. +.TP +.B fenced +This file contains 0 when the fence request is created. Userspace +fencing agents write 1 into this file once the mount has been fenced. +.TP +.B ipv4_addr +This file contains the dotted quad IPv4 peer address of the last +connected socket from the mount. Userspace fencing agents can use this +to find the host that contains the mount. +.TP +.B reason +This file contains a text string that indicates the reason that the +mount is being fenced: + +.B client_recovery +- During startup the server found persistent items recording the presence +of a mount that didn't reconnect to the server in time. +.sp +.B client_reconnect +- A mount disconnected from the server and didn't reconnect in time. +.sp +.B quorum_block_leader +- As a leader was elected it read persistent blocks that indicated that +a previous leader had not shut down and cleared their quorum block. +.TP +.B rid +This file contains the hex string of the RID of the mount to be fenced. +.RE + +The request directories enable userspace processes to gather the +information to find the host with the mount to fence, isolate the mount +by whatever means are appropriate (f.e. cut off network and storage +communication, force unmount the mount, isolate storage fabric ports, +reboot the host) and write to the +.I fenced +file. +.sp +Once the +.I fenced +file is written to the server reclaims the resources +associated with the fenced mount and resumes normal operations. +.sp +If the +.I error +file is written to then the server cannot make forward progress and +shuts down. The request can similarly enter an errored state if enough +time passes before userspace completes the request. + +.SH CORRUPTION DETECTION A .B scoutfs filesystem can detect corruption at runtime. A catalog of kernel log diff --git a/utils/scoutfs-utils.spec.in b/utils/scoutfs-utils.spec.in index 42bb86c5..859419da 100644 --- a/utils/scoutfs-utils.spec.in +++ b/utils/scoutfs-utils.spec.in @@ -54,12 +54,15 @@ cp man/*.8.gz $RPM_BUILD_ROOT%{_mandir}/man8/. install -m 755 -D src/scoutfs $RPM_BUILD_ROOT%{_sbindir}/scoutfs install -m 644 -D src/ioctl.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/ioctl.h install -m 644 -D src/format.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/format.h +install -m 755 -D fenced/scoutfs-fenced $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/scoutfs-fenced +install -m 755 -D fenced/local-force-unmount $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/local-force-unmount %files %defattr(644,root,root,755) %{_mandir}/man*/scoutfs*.gz %defattr(755,root,root,755) %{_sbindir}/scoutfs +%{_libexecdir}/scoutfs-fenced %files -n scoutfs-devel %defattr(644,root,root,755) diff --git a/utils/src/print.c b/utils/src/print.c index 9ddc3e4b..5fa57bdb 100644 --- a/utils/src/print.c +++ b/utils/src/print.c @@ -372,9 +372,13 @@ static int print_mounted_client_entry(struct scoutfs_key *key, void *val, unsigned val_len, void *arg) { struct scoutfs_mounted_client_btree_val *mcv = val; + struct in_addr in; - printf(" rid %016llx flags 0x%x\n", - le64_to_cpu(key->skmc_rid), mcv->flags); + memset(&in, 0, sizeof(in)); + in.s_addr = htonl(le32_to_cpu(mcv->addr.v4.addr)); + + printf(" rid %016llx ipv4_addr %s flags 0x%x\n", + le64_to_cpu(key->skmc_rid), inet_ntoa(in), mcv->flags); return 0; } @@ -812,16 +816,16 @@ static char *alloc_addr_str(union scoutfs_inet_addr *ia) static int print_quorum_blocks(int fd, struct scoutfs_super_block *super) { - struct print_events { - size_t offset; - char *name; - } events[] = { - OFF_NAME(write), OFF_NAME(update_term), OFF_NAME(set_leader), - OFF_NAME(clear_leader), OFF_NAME(fenced), + const static char *event_names[] = { + [SCOUTFS_QUORUM_EVENT_BEGIN] = "begin", + [SCOUTFS_QUORUM_EVENT_TERM] = "term", + [SCOUTFS_QUORUM_EVENT_ELECT] = "elect", + [SCOUTFS_QUORUM_EVENT_FENCE] = "fence", + [SCOUTFS_QUORUM_EVENT_STOP] = "stop", + [SCOUTFS_QUORUM_EVENT_END] = "end", }; struct scoutfs_quorum_block *blk = NULL; struct scoutfs_quorum_block_event *ev; - char *log_addr = NULL; u64 blkno; int ret; int i; @@ -830,6 +834,7 @@ static int print_quorum_blocks(int fd, struct scoutfs_super_block *super) for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { blkno = SCOUTFS_QUORUM_BLKNO + i; free(blk); + blk = NULL; ret = read_block(fd, blkno, SCOUTFS_BLOCK_SM_SHIFT, (void **)&blk); if (ret) goto out; @@ -837,24 +842,19 @@ static int print_quorum_blocks(int fd, struct scoutfs_super_block *super) printf("quorum blkno %llu (slot %llu)\n", blkno, blkno - SCOUTFS_QUORUM_BLKNO); print_block_header(&blk->hdr, SCOUTFS_BLOCK_SM_SIZE); - printf(" term %llu random_write_mark 0x%llx flags 0x%llx\n", - le64_to_cpu(blk->term), - le64_to_cpu(blk->random_write_mark), - le64_to_cpu(blk->flags)); - for (e = 0; e < array_size(events); e++) { - ev = (void *)blk + events[e].offset; + for (e = 0; e < array_size(event_names); e++) { + ev = &blk->events[e]; - printf(" %12s: rid %016llx ts %llu.%08u\n", - events[e].name, le64_to_cpu(ev->rid), - le64_to_cpu(ev->ts.sec), - le32_to_cpu(ev->ts.nsec)); + printf(" %12s: rid %016llx term %llu ts %llu.%08u\n", + event_names[e], le64_to_cpu(ev->rid), le64_to_cpu(ev->term), + le64_to_cpu(ev->ts.sec), le32_to_cpu(ev->ts.nsec)); } } ret = 0; out: - free(log_addr); + free(blk); return ret; }