From accd680a7eb00bf527eae1765b9d716ef18052ac Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 18 Mar 2021 16:39:03 -0700 Subject: [PATCH 01/19] Fix block setup always returning 0 Another case of returning 0 instead of ret. Signed-off-by: Zach Brown --- kmod/src/block.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index bb581173..43dc2a66 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -1245,7 +1245,7 @@ out: if (ret) scoutfs_block_destroy(sb); - return 0; + return ret; } void scoutfs_block_destroy(struct super_block *sb) From da1af9b8414649047e0653b2c12dea82963e33b9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 24 Mar 2021 11:28:16 -0700 Subject: [PATCH 02/19] Add scoutfs inode ino lock coverage Add lock coverage which tracks if the inode has been refreshed and is covered by the inode group cluster lock. This will be used by drop_inode and evict_inode to discover that the inode is current and doesn't need to be refreshed. Signed-off-by: Zach Brown --- kmod/src/inode.c | 6 ++++++ kmod/src/inode.h | 2 ++ kmod/src/lock.c | 23 ++++++++++++++--------- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 42761a3f..e0e11e59 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -82,6 +82,7 @@ static void scoutfs_inode_ctor(void *obj) init_waitqueue_head(&si->data_waitq.waitq); init_rwsem(&si->xattr_rwsem); RB_CLEAR_NODE(&si->writeback_node); + scoutfs_lock_init_coverage(&si->ino_lock_cov); inode_init_once(&si->inode); } @@ -141,12 +142,15 @@ static void remove_writeback_inode(struct inode_sb_info *inf, void scoutfs_destroy_inode(struct inode *inode) { + struct scoutfs_inode_info *si = SCOUTFS_I(inode); DECLARE_INODE_SB_INFO(inode->i_sb, inf); spin_lock(&inf->writeback_lock); remove_writeback_inode(inf, SCOUTFS_I(inode)); spin_unlock(&inf->writeback_lock); + scoutfs_lock_del_coverage(inode->i_sb, &si->ino_lock_cov); + call_rcu(&inode->i_rcu, scoutfs_i_callback); } @@ -307,6 +311,7 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, if (ret == 0) { load_inode(inode, &sinode); atomic64_set(&si->last_refreshed, refresh_gen); + scoutfs_lock_add_coverage(sb, lock, &si->ino_lock_cov); } } else { ret = 0; @@ -1384,6 +1389,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, si->next_xattr_id = 0; si->have_item = false; atomic64_set(&si->last_refreshed, lock->refresh_gen); + scoutfs_lock_add_coverage(sb, lock, &si->ino_lock_cov); si->flags = 0; scoutfs_inode_set_meta_seq(inode); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index ad517752..60213d73 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -51,6 +51,8 @@ struct scoutfs_inode_info { struct rw_semaphore xattr_rwsem; struct rb_node writeback_node; + struct scoutfs_lock_coverage ino_lock_cov; + struct inode inode; }; diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 10d95507..7bddb762 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -127,15 +127,19 @@ static bool lock_modes_match(int granted, int requested) */ static void invalidate_inode(struct super_block *sb, u64 ino) { + struct scoutfs_inode_info *si; struct inode *inode; inode = scoutfs_ilookup(sb, ino); if (inode) { + si = SCOUTFS_I(inode); + scoutfs_inc_counter(sb, lock_invalidate_inode); if (S_ISREG(inode->i_mode)) { truncate_inode_pages(inode->i_mapping, 0); scoutfs_data_wait_changed(inode); } + iput(inode); } } @@ -172,6 +176,16 @@ static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, /* have to invalidate if we're not in the only usable case */ if (!(prev == SCOUTFS_LOCK_WRITE && mode == SCOUTFS_LOCK_READ)) { retry: + /* invalidate inodes before removing coverage */ + if (lock->start.sk_zone == SCOUTFS_FS_ZONE) { + ino = le64_to_cpu(lock->start.ski_ino); + last = le64_to_cpu(lock->end.ski_ino); + while (ino <= last) { + invalidate_inode(sb, ino); + ino++; + } + } + /* remove cov items to tell users that their cache is stale */ spin_lock(&lock->cov_list_lock); list_for_each_entry_safe(cov, tmp, &lock->cov_list, head) { @@ -187,15 +201,6 @@ retry: } spin_unlock(&lock->cov_list_lock); - if (lock->start.sk_zone == SCOUTFS_FS_ZONE) { - ino = le64_to_cpu(lock->start.ski_ino); - last = le64_to_cpu(lock->end.ski_ino); - while (ino <= last) { - invalidate_inode(sb, ino); - ino++; - } - } - scoutfs_item_invalidate(sb, &lock->start, &lock->end); } From a65775588f687596aca3f12b48143c77fbff5f44 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 25 Mar 2021 14:29:15 -0700 Subject: [PATCH 03/19] Add server recovery helpers Add a little set of functions to help the server track which clients are waiting to recover which state. The open map messages need to wait for recovery so we're moving recovery out of being only in the lock server. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/recov.c | 280 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/recov.h | 23 ++++ kmod/src/super.h | 2 + 4 files changed, 306 insertions(+) create mode 100644 kmod/src/recov.c create mode 100644 kmod/src/recov.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index db5f5576..ba8dc505 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -30,6 +30,7 @@ scoutfs-y += \ options.o \ per_task.o \ quorum.o \ + recov.o \ scoutfs_trace.o \ server.o \ sort_priv.o \ diff --git a/kmod/src/recov.c b/kmod/src/recov.c new file mode 100644 index 00000000..b0d894c2 --- /dev/null +++ b/kmod/src/recov.c @@ -0,0 +1,280 @@ +/* + * Copyright (C) 2021 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "recov.h" + +/* + * There are a few server messages which can't be processed until they + * know that they have state for all possibly active clients. These + * little helpers track which clients have recovered what state and give + * those message handlers a call to check if recovery has completed. We + * track the timeout here, but all we do is call back into the server to + * take steps to evict timed out clients and then let us know that their + * recovery has finished. + */ + +struct recov_info { + struct super_block *sb; + spinlock_t lock; + struct list_head pending; + struct timer_list timer; + void (*timeout_fn)(struct super_block *); +}; + +#define DECLARE_RECOV_INFO(sb, name) \ + struct recov_info *name = SCOUTFS_SB(sb)->recov_info + +struct recov_pending { + struct list_head head; + u64 rid; + int which; +}; + +static struct recov_pending *find_pending(struct recov_info *recinf, u64 rid, int which) +{ + struct recov_pending *pend; + + list_for_each_entry(pend, &recinf->pending, head) { + if ((rid == 0 || pend->rid == rid) && (pend->which & which)) + return pend; + } + + return NULL; +} + +/* + * Record that we'll be waiting for a client to recover something. + * _finished will eventually be called for every _prepare, either + * because recovery naturally finished or because it timed out and the + * server evicted the client. + */ +int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct recov_pending *alloc; + struct recov_pending *pend; + + if (WARN_ON_ONCE(which & SCOUTFS_RECOV_INVALID)) + return -EINVAL; + + alloc = kmalloc(sizeof(*pend), GFP_NOFS); + if (!alloc) + return -ENOMEM; + + spin_lock(&recinf->lock); + + pend = find_pending(recinf, rid, SCOUTFS_RECOV_ALL); + if (pend) { + pend->which |= which; + } else { + swap(pend, alloc); + pend->rid = rid; + pend->which = which; + list_add(&pend->head, &recinf->pending); + } + + spin_unlock(&recinf->lock); + + kfree(alloc); + return 0; +} + +/* + * Recovery is only finished once we've begun (which sets the timer) and + * all clients have finished. If we didn't test the timer we could + * claim it finished prematurely as clients are being prepared. + */ +static int recov_finished(struct recov_info *recinf) +{ + return !!(recinf->timeout_fn != NULL && list_empty(&recinf->pending)); +} + +static void timer_callback(struct timer_list *timer) +{ + struct recov_info *recinf = from_timer(recinf, timer, timer); + + recinf->timeout_fn(recinf->sb); +} + +/* + * Begin waiting for recovery once we've prepared all the clients. If + * the timeout period elapses before _finish is called on all prepared + * clients then the timer will call the callback. + * + * Returns > 0 if all the prepared clients finish recovery before begin + * is called. + */ +int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_block *), + unsigned int timeout_ms) +{ + DECLARE_RECOV_INFO(sb, recinf); + int ret; + + spin_lock(&recinf->lock); + + recinf->timeout_fn = timeout_fn; + recinf->timer.expires = jiffies + msecs_to_jiffies(timeout_ms); + add_timer(&recinf->timer); + + ret = recov_finished(recinf); + + spin_unlock(&recinf->lock); + + if (ret > 0) + del_timer_sync(&recinf->timer); + + return ret; +} + +/* + * A given client has recovered the given state. If it's finished all + * recovery then we free it, and if all clients have finished recovery + * then we cancel the timeout timer. + * + * Returns > 0 if _begin has been called and all clients have finished. + * The caller will only see > 0 returned once. + */ +int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct recov_pending *pend; + int ret = 0; + + spin_lock(&recinf->lock); + + pend = find_pending(recinf, rid, which); + if (pend) { + pend->which &= ~which; + if (pend->which) { + pend = NULL; + } else { + list_del(&pend->head); + ret = recov_finished(recinf); + } + } + + spin_unlock(&recinf->lock); + + if (ret > 0) + del_timer_sync(&recinf->timer); + + kfree(pend); + + return ret; +} + +/* + * Returns true if the given client is still trying to recover + * the given state. + */ +bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which) +{ + DECLARE_RECOV_INFO(sb, recinf); + bool is_pending; + + spin_lock(&recinf->lock); + is_pending = find_pending(recinf, rid, which) != NULL; + spin_unlock(&recinf->lock); + + return is_pending; +} + +/* + * Returns 0 if there are no rids waiting for the given state to be + * recovered. Returns the rid of a client still waiting if there are + * any, in no specified order. + * + * This is inherently racey. Callers are responsible for resolving any + * actions taken based on pending with the recovery finishing, perhaps + * before we return. + */ +u64 scoutfs_recov_next_pending(struct super_block *sb, int which) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct recov_pending *pend; + u64 rid; + + spin_lock(&recinf->lock); + pend = find_pending(recinf, 0, which); + rid = pend ? pend->rid : 0; + spin_unlock(&recinf->lock); + + return rid; +} + +/* + * The server is shutting down and doesn't need to worry about recovery + * anymore. It'll be built up again by the next server, if needed. + */ +void scoutfs_recov_shutdown(struct super_block *sb) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct recov_pending *pend; + struct recov_pending *tmp; + LIST_HEAD(list); + + del_timer_sync(&recinf->timer); + + spin_lock(&recinf->lock); + list_splice_init(&recinf->pending, &list); + recinf->timeout_fn = NULL; + spin_unlock(&recinf->lock); + + list_for_each_entry_safe(pend, tmp, &recinf->pending, head) { + list_del(&pend->head); + kfree(pend); + } +} + +int scoutfs_recov_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct recov_info *recinf; + int ret; + + recinf = kzalloc(sizeof(struct recov_info), GFP_KERNEL); + if (!recinf) { + ret = -ENOMEM; + goto out; + } + + recinf->sb = sb; + spin_lock_init(&recinf->lock); + INIT_LIST_HEAD(&recinf->pending); + timer_setup(&recinf->timer, timer_callback, 0); + + sbi->recov_info = recinf; + ret = 0; +out: + return ret; +} + +void scoutfs_recov_destroy(struct super_block *sb) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (recinf) { + scoutfs_recov_shutdown(sb); + + kfree(recinf); + sbi->recov_info = NULL; + } +} diff --git a/kmod/src/recov.h b/kmod/src/recov.h new file mode 100644 index 00000000..cfdca30e --- /dev/null +++ b/kmod/src/recov.h @@ -0,0 +1,23 @@ +#ifndef _SCOUTFS_RECOV_H_ +#define _SCOUTFS_RECOV_H_ + +enum { + SCOUTFS_RECOV_GREETING = ( 1 << 0), + SCOUTFS_RECOV_LOCKS = ( 1 << 1), + + SCOUTFS_RECOV_INVALID = (~0 << 2), + SCOUTFS_RECOV_ALL = (~SCOUTFS_RECOV_INVALID), +}; + +int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which); +int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_block *), + unsigned int timeout_ms); +int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which); +bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which); +u64 scoutfs_recov_next_pending(struct super_block *sb, int which); +void scoutfs_recov_shutdown(struct super_block *sb); + +int scoutfs_recov_setup(struct super_block *sb); +void scoutfs_recov_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/super.h b/kmod/src/super.h index 13912bdc..8d004db3 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -26,6 +26,7 @@ struct net_info; struct block_info; struct forest_info; struct srch_info; +struct recov_info; struct scoutfs_sb_info { struct super_block *sb; @@ -70,6 +71,7 @@ struct scoutfs_sb_info { struct lock_server_info *lock_server_info; struct client_info *client_info; struct server_info *server_info; + struct recov_info *recov_info; struct sysfs_info *sfsinfo; struct scoutfs_counters *counters; From 592f472a1c771606b9d291e73018fbdf94d6426f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 26 Mar 2021 10:26:33 -0700 Subject: [PATCH 04/19] Use recov in server to recover client greetings The server starts recovery when it finds mounted client items as it starts up. The clients are done recovering once they send their greeting. If they don't recover in time then they'll be fenced. Signed-off-by: Zach Brown --- kmod/src/server.c | 184 ++++++++++++++++++++++++++++++++++++++++++---- kmod/src/server.h | 1 + kmod/src/super.c | 3 + 3 files changed, 174 insertions(+), 14 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index 6ecd8e85..b25b2019 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -38,6 +38,7 @@ #include "srch.h" #include "alloc.h" #include "forest.h" +#include "recov.h" /* * Every active mount can act as the server that listens on a net @@ -96,6 +97,9 @@ struct server_info { /* stable versions stored from commits, given in locks and rpcs */ seqcount_t roots_seqcount; struct scoutfs_net_roots roots; + + /* recovery timeout fences from work */ + struct work_struct fence_pending_recov_work; }; #define DECLARE_SERVER_INFO(sb, name) \ @@ -1198,8 +1202,13 @@ static int server_greeting(struct super_block *sb, ret = scoutfs_server_apply_commit(sb, ret); queue_work(server->wq, &server->farewell_work); + if (ret < 0) + goto send_err; } + scoutfs_server_recov_finish(sb, le64_to_cpu(gr->rid), SCOUTFS_RECOV_GREETING); + ret = 0; + send_err: err = ret; @@ -1259,6 +1268,24 @@ static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) sizeof(struct scoutfs_mounted_client_btree_val)); } +static int reclaim_rid(struct super_block *sb, u64 rid) +{ + int ret; + + ret = scoutfs_server_hold_commit(sb); + if (ret < 0) + return ret; + + /* delete mounted client last, client reconnect looks for it */ + ret = scoutfs_lock_server_farewell(sb, rid) ?: + remove_trans_seq(sb, rid) ?: + reclaim_log_trees(sb, rid) ?: + cancel_srch_compact(sb, rid) ?: + delete_mounted_client(sb, rid); + + return scoutfs_server_apply_commit(sb, ret); +} + /* * This work processes farewell requests asynchronously. Requests from * quorum members can be held until only the final majority remains and @@ -1386,18 +1413,7 @@ static void farewell_worker(struct work_struct *work) /* process and send farewell responses */ list_for_each_entry_safe(fw, tmp, &send, entry) { - ret = scoutfs_server_hold_commit(sb); - if (ret) - goto out; - - /* delete mounted client last, client reconnect looks for it */ - ret = scoutfs_lock_server_farewell(sb, fw->rid) ?: - remove_trans_seq(sb, fw->rid) ?: - reclaim_log_trees(sb, fw->rid) ?: - cancel_srch_compact(sb, fw->rid) ?: - delete_mounted_client(sb, fw->rid); - - ret = scoutfs_server_apply_commit(sb, ret); + ret = reclaim_rid(sb, fw->rid); if (ret) goto out; } @@ -1540,6 +1556,141 @@ static void server_notify_down(struct super_block *sb, } } +/* + * All clients have recovered all state. Now we can kick all the work + * that was waiting on recovery. + * + * It's a bit of a false dependency to have all work wait for completion + * before any work can make progress, but recovery is naturally + * concerned about in-memory state. It should all be quick to recover + * once a client arrives. + */ +static void finished_recovery(struct super_block *sb) +{ + DECLARE_SERVER_INFO(sb, server); + int ret = 0; + + scoutfs_info(sb, "all clients recovered"); + + if (ret < 0) { + scoutfs_err(sb, "error %d resuming after recovery finished, shutting down", ret); + stop_server(server); + } +} + +void scoutfs_server_recov_finish(struct super_block *sb, u64 rid, int which) +{ + if (scoutfs_recov_finish(sb, rid, which) > 0) + finished_recovery(sb); +} + +/* + * If the recovery timeout is too short we'll prematurely evict mounts + * that would have recovered. They need time to have their sockets + * timeout, reconnect to the current server, and fully recover their + * state. + * + * If it's too long we'll needlessly delay resuming operations after + * clients crash and will never recover. + */ +#define SERVER_RECOV_TIMEOUT_MS (30 * MSEC_PER_SEC) + +/* + * Not all clients recovered in time. We fence them and reclaim + * whatever resources they were using. If we see a rid here then we're + * going to fence it, regardless of if it manages to finish recovery + * while we're fencing it. + */ +static void fence_pending_recov_worker(struct work_struct *work) +{ + struct server_info *server = container_of(work, struct server_info, + fence_pending_recov_work); + struct super_block *sb = server->sb; + u64 rid; + int ret; + + while ((rid = scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_ALL)) > 0) { + scoutfs_err(sb, "%lu ms recovery timeout expired for client rid %016llx, fencing", + SERVER_RECOV_TIMEOUT_MS, rid); + + ret = reclaim_rid(sb, rid); + if (ret < 0) { + scoutfs_err(sb, "error %d reclaiming rid %016llx, shutting down", ret, rid); + stop_server(server); + break; + } + + scoutfs_server_recov_finish(sb, rid, SCOUTFS_RECOV_ALL); + } +} + +static void recovery_timeout(struct super_block *sb) +{ + DECLARE_SERVER_INFO(sb, server); + + if (!server->shutting_down) + queue_work(server->wq, &server->fence_pending_recov_work); +} + +/* + * As the server starts up it needs to start waiting for recovery from + * any clients which were previously still mounted in the last running + * server. This is done before networking is started so we won't + * receive any messages from clients until we've prepared them all. If + * the clients don't recover in time then they'll be fenced. + */ +static int start_recovery(struct super_block *sb) +{ + DECLARE_SERVER_INFO(sb, server); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + unsigned int nr = 0; + u64 rid; + int ret; + + for (rid = 0; ; rid++) { + init_mounted_client_key(&key, rid); + ret = scoutfs_btree_next(sb, &super->mounted_clients, &key, &iref); + if (ret == -ENOENT) { + ret = 0; + break; + } + if (ret == 0) { + rid = le64_to_cpu(iref.key->skmc_rid); + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) + goto out; + + ret = scoutfs_recov_prepare(sb, rid, SCOUTFS_RECOV_ALL); + if (ret < 0) { + scoutfs_err(sb, "error %d preparing recovery for client rid %016llx, shutting down", + ret, rid); + goto out; + } + + nr++; + } + + if (nr > 0) { + scoutfs_info(sb, "waiting for %u clients to recover", nr); + + ret = scoutfs_recov_begin(sb, recovery_timeout, SERVER_RECOV_TIMEOUT_MS); + if (ret > 0) { + finished_recovery(sb); + ret = 0; + } + } + +out: + if (ret < 0) { + scoutfs_err(sb, "error %d starting recovery, shutting down", ret); + stop_server(server); + } + return ret; +} + static void scoutfs_server_worker(struct work_struct *work) { struct server_info *server = container_of(work, struct server_info, @@ -1610,8 +1761,8 @@ static void scoutfs_server_worker(struct work_struct *work) goto shutdown; } - ret = scoutfs_lock_server_setup(sb, &server->alloc, &server->wri, - max_vers); + ret = scoutfs_lock_server_setup(sb, &server->alloc, &server->wri, max_vers) ?: + start_recovery(sb); if (ret) goto shutdown; @@ -1635,6 +1786,10 @@ shutdown: scoutfs_net_shutdown(sb, conn); server->conn = NULL; + /* stop tracking recovery, cancel timer, flush any fencing */ + scoutfs_recov_shutdown(sb); + flush_work(&server->fence_pending_recov_work); + /* wait for extra queues by requests, won't find waiters */ flush_work(&server->commit_work); @@ -1724,6 +1879,7 @@ int scoutfs_server_setup(struct super_block *sb) mutex_init(&server->srch_mutex); mutex_init(&server->mounted_clients_mutex); seqcount_init(&server->roots_seqcount); + INIT_WORK(&server->fence_pending_recov_work, fence_pending_recov_worker); server->wq = alloc_workqueue("scoutfs_server", WQ_UNBOUND | WQ_NON_REENTRANT, 0); diff --git a/kmod/src/server.h b/kmod/src/server.h index 84e25ddb..e06c0818 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -64,6 +64,7 @@ int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, struct scoutfs_key *key); int scoutfs_server_hold_commit(struct super_block *sb); int scoutfs_server_apply_commit(struct super_block *sb, int err); +void scoutfs_server_recov_finish(struct super_block *sb, u64 rid, int which); struct sockaddr_in; struct scoutfs_quorum_elected_info; diff --git a/kmod/src/super.c b/kmod/src/super.c index 1a795a69..e66160ed 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -44,6 +44,7 @@ #include "srch.h" #include "item.h" #include "alloc.h" +#include "recov.h" #include "scoutfs_trace.h" static struct dentry *scoutfs_debugfs_root; @@ -260,6 +261,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_quorum_destroy(sb); scoutfs_lock_shutdown(sb); scoutfs_server_destroy(sb); + scoutfs_recov_destroy(sb); scoutfs_net_destroy(sb); scoutfs_lock_destroy(sb); @@ -593,6 +595,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_setup_trans(sb) ?: scoutfs_lock_setup(sb) ?: scoutfs_net_setup(sb) ?: + scoutfs_recov_setup(sb) ?: scoutfs_server_setup(sb) ?: scoutfs_quorum_setup(sb) ?: scoutfs_client_setup(sb) ?: From c6fd80763861b3de4063c2e19041dac1fe6b2bc1 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 29 Mar 2021 12:46:41 -0700 Subject: [PATCH 05/19] Use recov to manage lock recovery Now that we have the recov layer we can have the lock server use it to track lock recovery. The lock server no longer needs its own recovery tracking structures and can instead call recov. We add a call for the server to call to kick lock processing once lock recovery finishes. We can get rid of the persistent lock_client items now that the server is driving recovery from the mounted_client items. Signed-off-by: Zach Brown --- kmod/src/format.h | 13 +-- kmod/src/lock_server.c | 256 ++++------------------------------------- kmod/src/lock_server.h | 4 +- kmod/src/server.c | 11 +- utils/src/print.c | 17 --- 5 files changed, 33 insertions(+), 268 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index b67d066d..3a1a4cca 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -195,9 +195,6 @@ struct scoutfs_key { #define sklt_rid _sk_first #define sklt_nr _sk_second -/* lock clients */ -#define sklc_rid _sk_first - /* seqs */ #define skts_trans_seq _sk_first #define skts_rid _sk_second @@ -493,11 +490,10 @@ struct scoutfs_bloom_block { #define SCOUTFS_LOCK_ZONE 4 /* Items only stored in server btrees */ #define SCOUTFS_LOG_TREES_ZONE 6 -#define SCOUTFS_LOCK_CLIENTS_ZONE 7 -#define SCOUTFS_TRANS_SEQ_ZONE 8 -#define SCOUTFS_MOUNTED_CLIENT_ZONE 9 -#define SCOUTFS_SRCH_ZONE 10 -#define SCOUTFS_FREE_EXTENT_ZONE 11 +#define SCOUTFS_TRANS_SEQ_ZONE 7 +#define SCOUTFS_MOUNTED_CLIENT_ZONE 8 +#define SCOUTFS_SRCH_ZONE 9 +#define SCOUTFS_FREE_EXTENT_ZONE 10 /* inode index zone */ #define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 1 @@ -653,7 +649,6 @@ struct scoutfs_super_block { struct scoutfs_alloc_list_head server_meta_freed[2]; struct scoutfs_btree_root fs_root; struct scoutfs_btree_root logs_root; - struct scoutfs_btree_root lock_clients; struct scoutfs_btree_root trans_seqs; struct scoutfs_btree_root mounted_clients; struct scoutfs_btree_root srch_root; diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index a2d96dbf..fb33fd80 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -20,10 +20,10 @@ #include "tseq.h" #include "spbm.h" #include "block.h" -#include "btree.h" #include "msg.h" #include "scoutfs_trace.h" #include "lock_server.h" +#include "recov.h" /* * The scoutfs server implements a simple lock service. Client mounts @@ -56,14 +56,11 @@ * Message requests and responses are reliably delivered in order across * reconnection. * - * The server maintains a persistent record of connected clients. A new - * server instance discovers these and waits for previously connected - * clients to reconnect and recover their state before proceeding. If - * clients don't reconnect they are forcefully prevented from unsafely - * accessing the shared persistent storage. (fenced, according to the - * rules of the platform.. could range from being powered off to having - * their switch port disabled to having their local block device set - * read-only.) + * As a new server comes up it recovers lock state from existing clients + * which were connected to a previous lock server. Recover requests are + * sent to clients as they connect and they respond with all there + * locks. Once all clients and locks are accounted for normal + * processing can resume. * * The lock server doesn't respond to memory pressure. The only way * locks are freed is if they are invalidated to null on behalf of a @@ -77,12 +74,8 @@ struct lock_server_info { struct super_block *sb; spinlock_t lock; - struct mutex mutex; struct rb_root locks_root; - struct scoutfs_spbm recovery_pending; - struct delayed_work recovery_dwork; - struct scoutfs_tseq_tree tseq_tree; struct dentry *tseq_dentry; @@ -430,7 +423,7 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 rid, goto out; } - /* XXX should always have a server lock here? recovery? */ + /* XXX should always have a server lock here? */ snode = get_server_lock(inf, &nl->key, NULL, false); if (!snode) { ret = -EINVAL; @@ -473,12 +466,9 @@ out: * so we unlock the snode mutex. * * All progress must wait for all clients to finish with recovery - * because we don't know which locks they'll hold. The unlocked - * recovery_pending test here is OK. It's filled by setup before - * anything runs. It's emptied by recovery completion. We can get a - * false nonempty result if we race with recovery completion, but that's - * OK because recovery completion processes all the locks that have - * requests after emptying, including the unlikely loser of that race. + * because we don't know which locks they'll hold. Once recover + * finishes the server calls us to kick all the locks that were waiting + * during recovery. */ static int process_waiting_requests(struct super_block *sb, struct server_lock_node *snode) @@ -496,7 +486,7 @@ static int process_waiting_requests(struct super_block *sb, /* processing waits for all invalidation responses or recovery */ if (!list_empty(&snode->invalidated) || - !scoutfs_spbm_empty(&inf->recovery_pending)) { + scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_LOCKS) != 0) { ret = 0; goto out; } @@ -569,89 +559,39 @@ out: return ret; } -static void init_lock_clients_key(struct scoutfs_key *key, u64 rid) -{ - *key = (struct scoutfs_key) { - .sk_zone = SCOUTFS_LOCK_CLIENTS_ZONE, - .sklc_rid = cpu_to_le64(rid), - }; -} - /* * The server received a greeting from a client for the first time. If - * the client had already talked to the server then we must find an - * existing record for it and should begin recovery. If it doesn't have - * a record then its timed out and we can't allow it to reconnect. If - * we're creating a new record for a client we can see EEXIST if the - * greeting is resent to a new server after the record was committed but - * before the response was received by the client. + * the client is in lock recovery then we send the initial lock request. * * This is running in concurrent client greeting processing contexts. */ -int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid, - bool should_exist) +int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid) { - DECLARE_LOCK_SERVER_INFO(sb, inf); - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_key key; int ret; - init_lock_clients_key(&key, rid); - - mutex_lock(&inf->mutex); - if (should_exist) { - ret = scoutfs_btree_lookup(sb, &super->lock_clients, &key, - &iref); - if (ret == 0) - scoutfs_btree_put_iref(&iref); - } else { - ret = scoutfs_btree_insert(sb, inf->alloc, inf->wri, - &super->lock_clients, - &key, NULL, 0); - if (ret == -EEXIST) - ret = 0; - } - mutex_unlock(&inf->mutex); - - if (should_exist && ret == 0) { + if (scoutfs_recov_is_pending(sb, rid, SCOUTFS_RECOV_LOCKS)) { scoutfs_key_set_zeros(&key); ret = scoutfs_server_lock_recover_request(sb, rid, &key); - if (ret) - goto out; + } else { + ret = 0; } -out: return ret; } /* - * A client sent their last recovery response and can exit recovery. If - * they were the last client in recovery then we can process all the - * server locks that had requests. + * All clients have finished lock recovery, we can make forward process + * on all the queued requests that were waiting on recovery. */ -static int finished_recovery(struct super_block *sb, u64 rid, bool cancel) +int scoutfs_lock_server_finished_recovery(struct super_block *sb) { DECLARE_LOCK_SERVER_INFO(sb, inf); struct server_lock_node *snode; struct scoutfs_key key; - bool still_pending; int ret = 0; - spin_lock(&inf->lock); - scoutfs_spbm_clear(&inf->recovery_pending, rid); - still_pending = !scoutfs_spbm_empty(&inf->recovery_pending); - spin_unlock(&inf->lock); - if (still_pending) - return 0; - - if (cancel) - cancel_delayed_work_sync(&inf->recovery_dwork); - scoutfs_key_set_zeros(&key); - - scoutfs_info(sb, "all lock clients recovered"); - while ((snode = get_server_lock(inf, &key, NULL, true))) { key = snode->key; @@ -695,16 +635,15 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid, int i; /* client must be in recovery */ - spin_lock(&inf->lock); - if (!scoutfs_spbm_test(&inf->recovery_pending, rid)) + if (!scoutfs_recov_is_pending(sb, rid, SCOUTFS_RECOV_LOCKS)) { ret = -EINVAL; - spin_unlock(&inf->lock); - if (ret) goto out; + } /* client has sent us all their locks */ if (nlr->nr == 0) { - ret = finished_recovery(sb, rid, true); + scoutfs_server_recov_finish(sb, rid, SCOUTFS_RECOV_LOCKS); + ret = 0; goto out; } @@ -755,101 +694,15 @@ out: return ret; } -static int get_rid_and_put_ref(struct scoutfs_btree_item_ref *iref, u64 *rid) -{ - int ret; - - if (iref->val_len == 0) { - *rid = le64_to_cpu(iref->key->sklc_rid); - ret = 0; - } else { - ret = -EIO; - } - scoutfs_btree_put_iref(iref); - return ret; -} - -/* - * This work executes if enough time passes without all of the clients - * finishing with recovery and canceling the work. We walk through the - * client records and find any that still have their recovery pending. - */ -static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) -{ - struct lock_server_info *inf = container_of(work, - struct lock_server_info, - recovery_dwork.work); - struct super_block *sb = inf->sb; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_key key; - bool timed_out; - u64 rid; - int ret; - - ret = scoutfs_server_hold_commit(sb); - if (ret) - goto out; - - /* we enter recovery if there are any client records */ - for (rid = 0; ; rid++) { - init_lock_clients_key(&key, rid); - ret = scoutfs_btree_next(sb, &super->lock_clients, &key, &iref); - if (ret == -ENOENT) { - ret = 0; - break; - } - if (ret == 0) - ret = get_rid_and_put_ref(&iref, &rid); - if (ret < 0) - break; - - spin_lock(&inf->lock); - if (scoutfs_spbm_test(&inf->recovery_pending, rid)) { - scoutfs_spbm_clear(&inf->recovery_pending, rid); - timed_out = true; - } else { - timed_out = false; - } - spin_unlock(&inf->lock); - - if (!timed_out) - continue; - - scoutfs_err(sb, "client rid %016llx lock recovery timed out", - rid); - - init_lock_clients_key(&key, rid); - ret = scoutfs_btree_delete(sb, inf->alloc, inf->wri, - &super->lock_clients, &key); - if (ret) - break; - } - - ret = scoutfs_server_apply_commit(sb, ret); -out: - /* force processing all pending lock requests */ - if (ret == 0) - ret = finished_recovery(sb, 0, false); - - if (ret < 0) { - scoutfs_err(sb, "lock server saw err %d while timing out clients, shutting down", ret); - scoutfs_server_abort(sb); - } -} - /* * A client is leaving the lock service. They aren't using locks and * won't send any more requests. We tear down all the state we had for * them. This can be called multiple times for a given client as their * farewell is resent to new servers. It's OK to not find any state. - * If we fail to delete a persistent entry then we have to shut down and - * hope that the next server has more luck. */ int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid) { DECLARE_LOCK_SERVER_INFO(sb, inf); - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct client_lock_entry *clent; struct client_lock_entry *tmp; struct server_lock_node *snode; @@ -858,20 +711,7 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid) bool freed; int ret = 0; - mutex_lock(&inf->mutex); - init_lock_clients_key(&key, rid); - ret = scoutfs_btree_delete(sb, inf->alloc, inf->wri, - &super->lock_clients, &key); - mutex_unlock(&inf->mutex); - if (ret == -ENOENT) { - ret = 0; - goto out; - } - if (ret < 0) - goto out; - scoutfs_key_set_zeros(&key); - while ((snode = get_server_lock(inf, &key, NULL, true))) { freed = false; @@ -956,23 +796,14 @@ static void lock_server_tseq_show(struct seq_file *m, /* * Setup the lock server. This is called before networking can deliver - * requests. If we find existing client records then we enter recovery. - * Lock request processing is deferred until recovery is resolved for - * all the existing clients, either they reconnect and replay locks or - * we time them out. + * requests. */ int scoutfs_lock_server_setup(struct super_block *sb, struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, u64 max_vers) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct lock_server_info *inf; - SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_key key; - unsigned int nr; - u64 rid; - int ret; inf = kzalloc(sizeof(struct lock_server_info), GFP_KERNEL); if (!inf) @@ -980,11 +811,7 @@ int scoutfs_lock_server_setup(struct super_block *sb, inf->sb = sb; spin_lock_init(&inf->lock); - mutex_init(&inf->mutex); inf->locks_root = RB_ROOT; - scoutfs_spbm_init(&inf->recovery_pending); - INIT_DELAYED_WORK(&inf->recovery_dwork, - scoutfs_lock_server_recovery_timeout); scoutfs_tseq_tree_init(&inf->tseq_tree, lock_server_tseq_show); inf->alloc = alloc; inf->wri = wri; @@ -999,36 +826,7 @@ int scoutfs_lock_server_setup(struct super_block *sb, sbi->lock_server_info = inf; - /* we enter recovery if there are any client records */ - nr = 0; - for (rid = 0; ; rid++) { - init_lock_clients_key(&key, rid); - ret = scoutfs_btree_next(sb, &super->lock_clients, &key, &iref); - if (ret == -ENOENT) - break; - if (ret == 0) - ret = get_rid_and_put_ref(&iref, &rid); - if (ret < 0) - goto out; - - ret = scoutfs_spbm_set(&inf->recovery_pending, rid); - if (ret) - goto out; - nr++; - - if (rid == U64_MAX) - break; - } - ret = 0; - - if (nr) { - schedule_delayed_work(&inf->recovery_dwork, - msecs_to_jiffies(LOCK_SERVER_RECOVERY_MS)); - scoutfs_info(sb, "waiting for %u lock clients to recover", nr); - } - -out: - return ret; + return 0; } /* @@ -1046,8 +844,6 @@ void scoutfs_lock_server_destroy(struct super_block *sb) LIST_HEAD(list); if (inf) { - cancel_delayed_work_sync(&inf->recovery_dwork); - debugfs_remove(inf->tseq_dentry); rbtree_postorder_for_each_entry_safe(snode, stmp, @@ -1066,8 +862,6 @@ void scoutfs_lock_server_destroy(struct super_block *sb) kfree(snode); } - scoutfs_spbm_destroy(&inf->recovery_pending); - kfree(inf); sbi->lock_server_info = NULL; } diff --git a/kmod/src/lock_server.h b/kmod/src/lock_server.h index 357fd5af..e77f116f 100644 --- a/kmod/src/lock_server.h +++ b/kmod/src/lock_server.h @@ -3,10 +3,10 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid, struct scoutfs_net_lock_recover *nlr); +int scoutfs_lock_server_finished_recovery(struct super_block *sb); int scoutfs_lock_server_request(struct super_block *sb, u64 rid, u64 net_id, struct scoutfs_net_lock *nl); -int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid, - bool should_exist); +int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid); int scoutfs_lock_server_response(struct super_block *sb, u64 rid, struct scoutfs_net_lock *nl); int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid); diff --git a/kmod/src/server.c b/kmod/src/server.c index b25b2019..0210dbe9 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -1239,15 +1239,7 @@ send_err: /* lock server might send recovery request */ if (le64_to_cpu(gr->server_term) != server->term) { - - /* we're now doing two commits per greeting, not great */ - ret = scoutfs_server_hold_commit(sb); - if (ret) - goto out; - - ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid), - gr->server_term != 0); - ret = scoutfs_server_apply_commit(sb, ret); + ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid)); if (ret) goto out; } @@ -1572,6 +1564,7 @@ static void finished_recovery(struct super_block *sb) scoutfs_info(sb, "all clients recovered"); + ret = scoutfs_lock_server_finished_recovery(sb); if (ret < 0) { scoutfs_err(sb, "error %d resuming after recovery finished, shutting down", ret); stop_server(server); diff --git a/utils/src/print.c b/utils/src/print.c index 10140be6..c60c11d9 100644 --- a/utils/src/print.c +++ b/utils/src/print.c @@ -339,14 +339,6 @@ static int print_srch_root_item(struct scoutfs_key *key, void *val, return 0; } -static int print_lock_clients_entry(struct scoutfs_key *key, void *val, - unsigned val_len, void *arg) -{ - printf(" rid %016llx\n", le64_to_cpu(key->sklc_rid)); - - return 0; -} - static int print_trans_seqs_entry(struct scoutfs_key *key, void *val, unsigned val_len, void *arg) { @@ -876,7 +868,6 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno) " server_meta_avail[1]: "AL_HEAD_F"\n" " server_meta_freed[0]: "AL_HEAD_F"\n" " server_meta_freed[1]: "AL_HEAD_F"\n" - " lock_clients root: height %u blkno %llu seq %llu\n" " mounted_clients root: height %u blkno %llu seq %llu\n" " srch_root root: height %u blkno %llu seq %llu\n" " trans_seqs root: height %u blkno %llu seq %llu\n" @@ -896,9 +887,6 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno) AL_HEAD_A(&super->server_meta_avail[1]), AL_HEAD_A(&super->server_meta_freed[0]), AL_HEAD_A(&super->server_meta_freed[1]), - super->lock_clients.height, - le64_to_cpu(super->lock_clients.ref.blkno), - le64_to_cpu(super->lock_clients.ref.seq), super->mounted_clients.height, le64_to_cpu(super->mounted_clients.ref.blkno), le64_to_cpu(super->mounted_clients.ref.seq), @@ -947,11 +935,6 @@ static int print_volume(int fd) ret = print_quorum_blocks(fd, super); - err = print_btree(fd, super, "lock_clients", &super->lock_clients, - print_lock_clients_entry, NULL); - if (err && !ret) - ret = err; - err = print_btree(fd, super, "mounted_clients", &super->mounted_clients, print_mounted_client_entry, NULL); if (err && !ret) From 22371fe5bdc33d4fd3df6c1a8ba36276ada54e69 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 18 Mar 2021 17:27:46 -0700 Subject: [PATCH 06/19] Fully destroy inodes after all mounts evict Today an inode's items are deleted once its nlink reaches zero and the final iput is called in a local mount. This can delete inodes from under other mounts which have opened the inode before it was unlinked on another mount. We fix this by adding cached inode tracking. Each mount maintains groups of cached inode bitmaps at the same granularity as inode locking. As a mount performs its final iput it gets a bitmap from the server which indicates if any other mount has inodes in the group open. This makes the two fast paths of opening and closing linked files and of deleting a file that was unlinked locally only pay a moderate cost of either maintaining the bitmap locally and only getting the open map once per lock group. Removing many files in a group will only lock and get the open map once per group. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/client.c | 35 ++ kmod/src/client.h | 4 + kmod/src/dir.c | 1 + kmod/src/file.c | 1 + kmod/src/format.h | 39 ++ kmod/src/inode.c | 65 ++- kmod/src/lock.c | 3 + kmod/src/lock.h | 6 + kmod/src/omap.c | 1046 ++++++++++++++++++++++++++++++++++++++ kmod/src/omap.h | 24 + kmod/src/scoutfs_trace.h | 83 +++ kmod/src/server.c | 68 ++- kmod/src/server.h | 5 + kmod/src/super.c | 3 + kmod/src/super.h | 2 + 16 files changed, 1364 insertions(+), 22 deletions(-) create mode 100644 kmod/src/omap.c create mode 100644 kmod/src/omap.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index ba8dc505..35f9fd07 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -27,6 +27,7 @@ scoutfs-y += \ lock_server.o \ msg.o \ net.o \ + omap.o \ options.o \ per_task.o \ quorum.o \ diff --git a/kmod/src/client.c b/kmod/src/client.c index 642a0c5d..99c7ba06 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -31,6 +31,7 @@ #include "net.h" #include "endian_swap.h" #include "quorum.h" +#include "omap.h" /* * The client is responsible for maintaining a connection to the server. @@ -215,6 +216,39 @@ int scoutfs_client_srch_commit_compact(struct super_block *sb, res, sizeof(*res), NULL, 0); } +int scoutfs_client_send_omap_response(struct super_block *sb, u64 id, + struct scoutfs_open_ino_map *map) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_response(sb, client->conn, SCOUTFS_NET_CMD_OPEN_INO_MAP, + id, 0, map, sizeof(*map)); +} + +/* The client is receiving an omap request from the server */ +static int client_open_ino_map(struct super_block *sb, struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + if (arg_len != sizeof(struct scoutfs_open_ino_map_args)) + return -EINVAL; + + return scoutfs_omap_client_handle_request(sb, id, arg); +} + +/* The client is sending an omap request to the server */ +int scoutfs_client_open_ino_map(struct super_block *sb, u64 group_nr, + struct scoutfs_open_ino_map *map) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + struct scoutfs_open_ino_map_args args = { + .group_nr = cpu_to_le64(group_nr), + .req_id = 0, + }; + + return scoutfs_net_sync_request(sb, client->conn, SCOUTFS_NET_CMD_OPEN_INO_MAP, + &args, sizeof(args), map, sizeof(*map)); +} + /* The client is receiving a invalidation request from the server */ static int client_lock(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, @@ -413,6 +447,7 @@ out: static scoutfs_net_request_t client_req_funcs[] = { [SCOUTFS_NET_CMD_LOCK] = client_lock, [SCOUTFS_NET_CMD_LOCK_RECOVER] = client_lock_recover, + [SCOUTFS_NET_CMD_OPEN_INO_MAP] = client_open_ino_map, }; /* diff --git a/kmod/src/client.h b/kmod/src/client.h index ae830ef8..c569f038 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -22,6 +22,10 @@ int scoutfs_client_srch_get_compact(struct super_block *sb, struct scoutfs_srch_compact *sc); int scoutfs_client_srch_commit_compact(struct super_block *sb, struct scoutfs_srch_compact *res); +int scoutfs_client_send_omap_response(struct super_block *sb, u64 id, + struct scoutfs_open_ino_map *map); +int scoutfs_client_open_ino_map(struct super_block *sb, u64 group_nr, + struct scoutfs_open_ino_map *map); int scoutfs_client_setup(struct super_block *sb); void scoutfs_client_destroy(struct super_block *sb); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 42c70a4a..2223a4ab 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -30,6 +30,7 @@ #include "item.h" #include "lock.h" #include "hash.h" +#include "omap.h" #include "counters.h" #include "scoutfs_trace.h" diff --git a/kmod/src/file.c b/kmod/src/file.c index f35ef039..b1555b08 100644 --- a/kmod/src/file.c +++ b/kmod/src/file.c @@ -27,6 +27,7 @@ #include "file.h" #include "inode.h" #include "per_task.h" +#include "omap.h" /* TODO: Direct I/O, AIO */ ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov, diff --git a/kmod/src/format.h b/kmod/src/format.h index 3a1a4cca..86d10d3d 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -840,6 +840,7 @@ enum scoutfs_net_cmd { SCOUTFS_NET_CMD_LOCK_RECOVER, SCOUTFS_NET_CMD_SRCH_GET_COMPACT, SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT, + SCOUTFS_NET_CMD_OPEN_INO_MAP, SCOUTFS_NET_CMD_FAREWELL, SCOUTFS_NET_CMD_UNKNOWN, }; @@ -960,4 +961,42 @@ enum scoutfs_corruption_sources { #define SC_NR_LONGS DIV_ROUND_UP(SC_NR_SOURCES, BITS_PER_LONG) +#define SCOUTFS_OPEN_INO_MAP_SHIFT 10 +#define SCOUTFS_OPEN_INO_MAP_BITS (1 << SCOUTFS_OPEN_INO_MAP_SHIFT) +#define SCOUTFS_OPEN_INO_MAP_MASK (SCOUTFS_OPEN_INO_MAP_BITS - 1) +#define SCOUTFS_OPEN_INO_MAP_LE64S (SCOUTFS_OPEN_INO_MAP_BITS / 64) + +/* + * The request and response conversation is as follows: + * + * client[init] -> server: + * group_nr = G + * req_id = 0 (I) + * server -> client[*] + * group_nr = G + * req_id = R + * client[*] -> server + * group_nr = G (I) + * req_id = R + * bits + * server -> client[init] + * group_nr = G (I) + * req_id = R (I) + * bits + * + * Many of the fields in individual messages are ignored ("I") because + * the net id or the omap req_id can be used to identify the + * conversation. We always include them on the wire to make inspected + * messages easier to follow. + */ +struct scoutfs_open_ino_map_args { + __le64 group_nr; + __le64 req_id; +}; + +struct scoutfs_open_ino_map { + struct scoutfs_open_ino_map_args args; + __le64 bits[SCOUTFS_OPEN_INO_MAP_LE64S]; +}; + #endif diff --git a/kmod/src/inode.c b/kmod/src/inode.c index e0e11e59..97a68aeb 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -33,6 +33,7 @@ #include "item.h" #include "client.h" #include "cmp.h" +#include "omap.h" /* * XXX @@ -693,6 +694,8 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino) atomic64_set(&si->last_refreshed, 0); ret = scoutfs_inode_refresh(inode, lock, 0); + if (ret == 0) + ret = scoutfs_omap_inc(sb, ino); if (ret) { iget_failed(inode); inode = ERR_PTR(ret); @@ -1405,10 +1408,17 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, store_inode(&sinode, inode); init_inode_key(&key, scoutfs_ino(inode)); + ret = scoutfs_omap_inc(sb, ino); + if (ret < 0) + goto out; + ret = scoutfs_item_create(sb, &key, &sinode, sizeof(sinode), lock); + if (ret < 0) + scoutfs_omap_dec(sb, ino); +out: if (ret) { iput(inode); - return ERR_PTR(ret); + inode = ERR_PTR(ret); } return inode; @@ -1453,15 +1463,15 @@ int scoutfs_orphan_delete(struct super_block *sb, u64 ino) /* * Remove all the items associated with a given inode. This is only - * called once nlink has dropped to zero so we don't have to worry about - * dirents referencing the inode or link backrefs. Dropping nlink to 0 - * also created an orphan item. That orphan item will continue - * triggering attempts to finish previous partial deletion until all - * deletion is complete and the orphan item is removed. + * called once nlink has dropped to zero and nothing has the inode open + * so we don't have to worry about dirents referencing the inode or link + * backrefs. Dropping nlink to 0 also created an orphan item. That + * orphan item will continue triggering attempts to finish previous + * partial deletion until all deletion is complete and the orphan item + * is removed. */ -static int delete_inode_items(struct super_block *sb, u64 ino) +static int delete_inode_items(struct super_block *sb, u64 ino, struct scoutfs_lock *lock) { - struct scoutfs_lock *lock = NULL; struct scoutfs_inode sinode; struct scoutfs_key key; LIST_HEAD(ind_locks); @@ -1471,10 +1481,6 @@ static int delete_inode_items(struct super_block *sb, u64 ino) u64 size; int ret; - ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_WRITE, 0, ino, &lock); - if (ret) - return ret; - init_inode_key(&key, ino); ret = scoutfs_item_lookup_exact(sb, &key, &sinode, sizeof(sinode), @@ -1539,18 +1545,24 @@ out: if (release) scoutfs_release_trans(sb); scoutfs_inode_index_unlock(sb, &ind_locks); - scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + return ret; } /* * iput_final has already written out the dirty pages to the inode * before we get here. We're left with a clean inode that we have to - * tear down. If there are no more links to the inode then we also - * remove all its persistent structures. + * tear down. We use locking and open inode number bitmaps to decide if + * we should finally destroy an inode that is no longer open nor + * reachable through directory entries. */ void scoutfs_evict_inode(struct inode *inode) { + struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); + struct scoutfs_lock *lock; + int ret; + trace_scoutfs_evict_inode(inode->i_sb, scoutfs_ino(inode), inode->i_nlink, is_bad_inode(inode)); @@ -1559,8 +1571,17 @@ void scoutfs_evict_inode(struct inode *inode) truncate_inode_pages_final(&inode->i_data); - if (inode->i_nlink == 0) - delete_inode_items(inode->i_sb, scoutfs_ino(inode)); + ret = scoutfs_omap_should_delete(sb, inode, &lock); + if (ret > 0) { + ret = delete_inode_items(inode->i_sb, scoutfs_ino(inode), lock); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + } + if (ret < 0) + scoutfs_err(sb, "error %d while checking to delete inode nr %llu, it might linger.", + ret, ino); + + scoutfs_omap_dec(sb, ino); + clear: clear_inode(inode); } @@ -1588,8 +1609,10 @@ int scoutfs_scan_orphans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock = sbi->rid_lock; + struct scoutfs_lock *inode_lock = NULL; struct scoutfs_key key; struct scoutfs_key last; + u64 ino; int err = 0; int ret; @@ -1605,7 +1628,13 @@ int scoutfs_scan_orphans(struct super_block *sb) if (ret < 0) goto out; - ret = delete_inode_items(sb, le64_to_cpu(key.sko_ino)); + ino = le64_to_cpu(key.sko_ino); + + ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_WRITE, 0, ino, &inode_lock); + if (ret == 0) { + ret = delete_inode_items(sb, le64_to_cpu(key.sko_ino), inode_lock); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE); + } if (ret && ret != -ENOENT && !err) err = ret; diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 7bddb762..f761ba49 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -34,6 +34,7 @@ #include "data.h" #include "xattr.h" #include "item.h" +#include "omap.h" /* * scoutfs uses a lock service to manage item cache consistency between @@ -234,6 +235,7 @@ static void lock_free(struct lock_info *linfo, struct scoutfs_lock *lock) BUG_ON(!list_empty(&lock->shrink_head)); BUG_ON(!list_empty(&lock->cov_list)); + scoutfs_omap_free_lock_data(lock->omap_data); kfree(lock); } @@ -269,6 +271,7 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb, lock->mode = SCOUTFS_LOCK_NULL; atomic64_set(&lock->forest_bloom_nr, 0); + spin_lock_init(&lock->omap_spinlock); trace_scoutfs_lock_alloc(sb, lock); diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 46dcdf96..5485bb35 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -10,6 +10,8 @@ #define SCOUTFS_LOCK_NR_MODES SCOUTFS_LOCK_INVALID +struct scoutfs_omap_lock; + /* * A few fields (start, end, refresh_gen, write_version, granted_mode) * are referenced by code outside lock.c. @@ -47,6 +49,10 @@ struct scoutfs_lock { /* the forest tracks which log tree last saw bloom bit updates */ atomic64_t forest_bloom_nr; + + /* open ino mapping has a valid map for a held write lock */ + spinlock_t omap_spinlock; + struct scoutfs_omap_lock_data *omap_data; }; struct scoutfs_lock_coverage { diff --git a/kmod/src/omap.c b/kmod/src/omap.c new file mode 100644 index 00000000..bb3ac8c4 --- /dev/null +++ b/kmod/src/omap.c @@ -0,0 +1,1046 @@ +/* + * Copyright (C) 2021 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "counters.h" +#include "cmp.h" +#include "inode.h" +#include "client.h" +#include "server.h" +#include "omap.h" +#include "recov.h" +#include "scoutfs_trace.h" + +/* + * As a client removes an inode from its cache with an nlink of 0 it + * needs to decide if it is the last client using the inode and should + * fully delete all its items. It needs to know if other mounts still + * have the inode in use. + * + * We need a way to communicate between mounts that an inode is open. + * We don't want to pay the synchronous per-file locking round trip + * costs associated with per-inode open locks that you'd typically see + * in systems to solve this problem. + * + * Instead clients maintain open bitmaps that cover groups of inodes. + * As inodes enter the cache their bit is set, and as the inode is + * evicted the bit is cleared. As an inode is evicted messages are sent + * around the cluster to get the current bitmaps for that inode's group + * from all active mounts. If the inode's bit is clear then it can be + * deleted. + * + * We associate the open bitmaps with our cluster locking of inode + * groups to cache these open bitmaps. As long as we have the lock then + * nlink can't be changed on any remote mounts. Specifically, it can't + * increase from 0 so any clear bits can gain references on remote + * mounts. As long as we have the lock, all clear bits in the group for + * inodes with 0 nlink can be deleted. + * + * This layer maintains a list of client rids to send messages to. The + * server calls us as clients enter and leave the cluster. We can't + * process requests until all clients are present as a server starts up + * so we hook into recovery and delay processing until all previously + * existing clients are recovered or fenced. + */ + +struct omap_rid_list { + int nr_rids; + struct list_head head; +}; + +struct omap_rid_entry { + struct list_head head; + u64 rid; +}; + +struct omap_info { + /* client */ + struct rhashtable group_ht; + + /* server */ + struct rhashtable req_ht; + struct llist_head requests; + spinlock_t lock; + struct omap_rid_list rids; + atomic64_t next_req_id; +}; + +#define DECLARE_OMAP_INFO(sb, name) \ + struct omap_info *name = SCOUTFS_SB(sb)->omap_info + +/* + * The presence of an inode in the inode cache increases the count of + * its inode number's position within its lock group. These structs + * track the counts for all the inodes in a lock group and maintain a + * bitmap whose bits are set for each non-zero count. + * + * We don't want to add additional global synchronization of inode cache + * maintenance so these are tracked in an rcu hash table. Once their + * total count reaches zero they're removed from the hash and queued for + * freeing and readers should ignore them. + */ +struct omap_group { + struct super_block *sb; + struct rhash_head ht_head; + struct rcu_head rcu; + u64 nr; + spinlock_t lock; + unsigned int total; + unsigned int *counts; + __le64 bits[SCOUTFS_OPEN_INO_MAP_LE64S]; +}; + +#define trace_group(sb, which, group, bit_nr) \ +do { \ + __typeof__(group) _grp = (group); \ + __typeof__(bit_nr) _nr = (bit_nr); \ + \ + trace_scoutfs_omap_group_##which(sb, _grp, _grp->nr, _grp->total, _nr, \ + _nr < 0 ? -1 : _grp->counts[_nr]); \ +} while (0) + +/* + * Each request is initialized with the rids of currently mounted + * clients. As each responds we remove their rid and send the response + * once everyone has contributed. + * + * The request frequency will typically be low, but in a mass rm -rf + * load we will see O(groups * clients) messages flying around. + */ +struct omap_request { + struct llist_node llnode; + struct rhash_head ht_head; + struct rcu_head rcu; + spinlock_t lock; + u64 client_rid; + u64 client_id; + struct omap_rid_list rids; + struct scoutfs_open_ino_map map; +}; + +/* + * In each inode group cluster lock we store data to track the open ino + * map which tracks all the inodes that the cluster lock covers. When + * the version shows that the map is stale we send a request to update + * it. + */ +struct scoutfs_omap_lock_data { + u64 version; + bool req_in_flight; + wait_queue_head_t waitq; + struct scoutfs_open_ino_map map; +}; + +static inline void init_rid_list(struct omap_rid_list *list) +{ + INIT_LIST_HEAD(&list->head); + list->nr_rids = 0; +} + +/* + * Negative searches almost never happen. + */ +static struct omap_rid_entry *find_rid(struct omap_rid_list *list, u64 rid) +{ + struct omap_rid_entry *entry; + + list_for_each_entry(entry, &list->head, head) { + if (rid == entry->rid) + return entry; + } + + return NULL; +} + +static int free_rid(struct omap_rid_list *list, struct omap_rid_entry *entry) +{ + int nr; + + list_del(&entry->head); + nr = --list->nr_rids; + + kfree(entry); + return nr; +} + +static int copy_rids(struct omap_rid_list *to, struct omap_rid_list *from, spinlock_t *from_lock) +{ + struct omap_rid_entry *entry; + struct omap_rid_entry *src; + struct omap_rid_entry *dst; + int nr; + + spin_lock(from_lock); + + while (to->nr_rids != from->nr_rids) { + nr = from->nr_rids; + spin_unlock(from_lock); + + while (to->nr_rids < nr) { + entry = kmalloc(sizeof(struct omap_rid_entry), GFP_NOFS); + if (!entry) + return -ENOMEM; + + list_add_tail(&entry->head, &to->head); + to->nr_rids++; + } + + while (to->nr_rids > nr) { + entry = list_first_entry(&to->head, struct omap_rid_entry, head); + list_del(&entry->head); + kfree(entry); + to->nr_rids--; + } + + spin_lock(from_lock); + } + + dst = list_first_entry(&to->head, struct omap_rid_entry, head); + list_for_each_entry(src, &from->head, head) { + dst->rid = src->rid; + dst = list_next_entry(dst, head); + } + + spin_unlock(from_lock); + + return 0; +} + +static void free_rids(struct omap_rid_list *list) +{ + struct omap_rid_entry *entry; + struct omap_rid_entry *tmp; + + list_for_each_entry_safe(entry, tmp, &list->head, head) { + list_del(&entry->head); + kfree(entry); + } +} + +static void calc_group_nrs(u64 ino, u64 *group_nr, int *bit_nr) +{ + *group_nr = ino >> SCOUTFS_OPEN_INO_MAP_SHIFT; + *bit_nr = ino & SCOUTFS_OPEN_INO_MAP_MASK; +} + +static struct omap_group *alloc_group(struct super_block *sb, u64 group_nr) +{ + struct omap_group *group; + + BUILD_BUG_ON((sizeof(group->counts[0]) * SCOUTFS_OPEN_INO_MAP_BITS) > PAGE_SIZE); + + group = kzalloc(sizeof(struct omap_group), GFP_NOFS); + if (group) { + group->sb = sb; + group->nr = group_nr; + spin_lock_init(&group->lock); + + group->counts = (void *)get_zeroed_page(GFP_NOFS); + if (!group->counts) { + kfree(group); + group = NULL; + } else { + trace_group(sb, alloc, group, -1); + } + } + + return group; +} + +static void free_group(struct super_block *sb, struct omap_group *group) +{ + trace_group(sb, free, group, -1); + free_page((unsigned long)group->counts); + kfree(group); +} + +static void free_group_rcu(struct rcu_head *rcu) +{ + struct omap_group *group = container_of(rcu, struct omap_group, rcu); + + free_group(group->sb, group); +} + +static const struct rhashtable_params group_ht_params = { + .key_len = member_sizeof(struct omap_group, nr), + .key_offset = offsetof(struct omap_group, nr), + .head_offset = offsetof(struct omap_group, ht_head), +}; + +/* + * Track an cached inode in its group. Our increment can be racing with + * a final decrement that removes the group from the hash, sets total to + * UINT_MAX, and calls rcu free. We can retry until the dead group is + * no longer visible in the hash table and we can insert a new allocated + * group. + */ +int scoutfs_omap_inc(struct super_block *sb, u64 ino) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct omap_group *group; + u64 group_nr; + int bit_nr; + bool found; + int ret = 0; + + calc_group_nrs(ino, &group_nr, &bit_nr); + +retry: + found = false; + rcu_read_lock(); + group = rhashtable_lookup(&ominf->group_ht, &group_nr, group_ht_params); + if (group) { + spin_lock(&group->lock); + if (group->total < UINT_MAX) { + found = true; + if (group->counts[bit_nr]++ == 0) { + set_bit_le(bit_nr, group->bits); + group->total++; + } + } + trace_group(sb, inc, group, bit_nr); + spin_unlock(&group->lock); + } + rcu_read_unlock(); + + if (!found) { + group = alloc_group(sb, group_nr); + if (group) { + ret = rhashtable_lookup_insert_fast(&ominf->group_ht, &group->ht_head, + group_ht_params); + if (ret < 0) + free_group(sb, group); + if (ret == -EEXIST) + ret = 0; + if (ret == -EBUSY) { + /* wait for rehash to finish */ + synchronize_rcu(); + ret = 0; + } + if (ret == 0) + goto retry; + } else { + ret = -ENOMEM; + } + } + + return ret; +} + +/* + * Decrement a previously incremented ino count. Not finding a count + * implies imbalanced inc/dec or bugs freeing groups. We only free + * groups here as the last dec drops the group's total count to 0. + */ +void scoutfs_omap_dec(struct super_block *sb, u64 ino) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct omap_group *group; + u64 group_nr; + int bit_nr; + + calc_group_nrs(ino, &group_nr, &bit_nr); + + rcu_read_lock(); + group = rhashtable_lookup(&ominf->group_ht, &group_nr, group_ht_params); + if (group) { + spin_lock(&group->lock); + WARN_ON_ONCE(group->counts[bit_nr] == 0); + WARN_ON_ONCE(group->total == 0); + WARN_ON_ONCE(group->total == UINT_MAX); + if (--group->counts[bit_nr] == 0) { + clear_bit_le(bit_nr, group->bits); + if (--group->total == 0) { + group->total = UINT_MAX; + rhashtable_remove_fast(&ominf->group_ht, &group->ht_head, + group_ht_params); + call_rcu(&group->rcu, free_group_rcu); + } + } + trace_group(sb, dec, group, bit_nr); + spin_unlock(&group->lock); + } + rcu_read_unlock(); + + WARN_ON_ONCE(!group); +} + +/* + * The server adds rids as it discovers clients. We add them to the + * list of rids to send map requests to. + */ +int scoutfs_omap_add_rid(struct super_block *sb, u64 rid) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct omap_rid_entry *entry; + struct omap_rid_entry *found; + + entry = kmalloc(sizeof(struct omap_rid_entry), GFP_NOFS); + if (!entry) + return -ENOMEM; + + spin_lock(&ominf->lock); + found = find_rid(&ominf->rids, rid); + if (!found) { + entry->rid = rid; + list_add_tail(&entry->head, &ominf->rids.head); + ominf->rids.nr_rids++; + } + spin_unlock(&ominf->lock); + + if (found) + kfree(entry); + + return 0; +} + +static void free_req(struct omap_request *req) +{ + free_rids(&req->rids); + kfree(req); +} + +static void free_req_rcu(struct rcu_head *rcu) +{ + struct omap_request *req = container_of(rcu, struct omap_request, rcu); + + free_req(req); +} + +static const struct rhashtable_params req_ht_params = { + .key_len = member_sizeof(struct omap_request, map.args.req_id), + .key_offset = offsetof(struct omap_request, map.args.req_id), + .head_offset = offsetof(struct omap_request, ht_head), +}; + +/* + * Remove a rid from all the pending requests. If it's the last rid we + * give the caller the details to send a response, they'll call back to + * keep removing. If their send fails they're going to shutdown the + * server so we can queue freeing the request as we give it to them. + */ +static int remove_rid_from_reqs(struct omap_info *ominf, u64 rid, u64 *resp_rid, u64 *resp_id, + struct scoutfs_open_ino_map *map) +{ + struct omap_rid_entry *entry; + struct rhashtable_iter iter; + struct omap_request *req; + int ret = 0; + + rhashtable_walk_enter(&ominf->req_ht, &iter); + rhashtable_walk_start(&iter); + + for (;;) { + req = rhashtable_walk_next(&iter); + if (req == NULL) + break; + if (req == ERR_PTR(-EAGAIN)) + continue; + + spin_lock(&req->lock); + entry = find_rid(&req->rids, rid); + if (entry && free_rid(&req->rids, entry) == 0) { + *resp_rid = req->client_rid; + *resp_id = req->client_id; + memcpy(map, &req->map, sizeof(struct scoutfs_open_ino_map)); + rhashtable_remove_fast(&ominf->req_ht, &req->ht_head, req_ht_params); + call_rcu(&req->rcu, free_req_rcu); + ret = 1; + } + spin_unlock(&req->lock); + if (ret > 0) + break; + } + + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); + + if (ret <= 0) { + *resp_rid = 0; + *resp_id = 0; + } + + return ret; +} + +/* + * A client has been evicted. Remove its rid from the list and walk + * through all the pending requests and remove its rids, sending the + * response if it was the last rid waiting for a response. + * + * If this returns an error then the server will shut down. + */ +int scoutfs_omap_remove_rid(struct super_block *sb, u64 rid) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct scoutfs_open_ino_map *map = NULL; + struct omap_rid_entry *entry; + u64 resp_rid = 0; + u64 resp_id = 0; + int ret; + + map = kmalloc(sizeof(struct scoutfs_open_ino_map), GFP_NOFS); + if (!map) { + ret = -ENOMEM; + goto out; + } + + spin_lock(&ominf->lock); + entry = find_rid(&ominf->rids, rid); + if (entry) + free_rid(&ominf->rids, entry); + spin_unlock(&ominf->lock); + + /* the server really shouldn't be removing a rid it never added */ + if (WARN_ON_ONCE(!entry)) { + ret = -ENOENT; + goto out; + } + + /* remove the rid from all pending requests, sending responses if it was final */ + for (;;) { + ret = remove_rid_from_reqs(ominf, rid, &resp_rid, &resp_id, map); + if (ret <= 0) + break; + ret = scoutfs_server_send_omap_response(sb, resp_rid, resp_id, map, 0); + if (ret < 0) + break; + } + +out: + kfree(map); + return ret; +} + +/* + * Handle a single incoming request in the server. This could have been + * delayed by recovery. This only returns an error if we couldn't send + * a processing error response to the client. + */ +static int handle_request(struct super_block *sb, struct omap_request *req) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct omap_rid_list priv_rids; + struct omap_rid_entry *entry; + int ret; + + init_rid_list(&priv_rids); + + ret = copy_rids(&priv_rids, &ominf->rids, &ominf->lock); + if (ret < 0) + goto out; + + /* don't send a request to the client who originated this request */ + entry = find_rid(&priv_rids, req->client_rid); + if (entry && free_rid(&priv_rids, entry) == 0) { + ret = scoutfs_server_send_omap_response(sb, req->client_rid, req->client_id, + &req->map, 0); + kfree(req); + req = NULL; + goto out; + } + + /* this lock isn't needed but sparse gave warnings with conditional locking */ + ret = copy_rids(&req->rids, &priv_rids, &ominf->lock); + if (ret < 0) + goto out; + + do { + ret = rhashtable_insert_fast(&ominf->req_ht, &req->ht_head, req_ht_params); + if (ret == -EBUSY) + synchronize_rcu(); /* wait for rehash to finish */ + } while (ret == -EBUSY); + + if (ret < 0) + goto out; + + /* + * We can start getting responses the moment we send the first response. After + * we send the last request the req can be freed. + */ + while ((entry = list_first_entry_or_null(&priv_rids.head, struct omap_rid_entry, head))) { + ret = scoutfs_server_send_omap_request(sb, entry->rid, &req->map.args); + if (ret < 0) { + rhashtable_remove_fast(&ominf->req_ht, &req->ht_head, req_ht_params); + goto out; + } + + free_rid(&priv_rids, entry); + } + + ret = 0; +out: + free_rids(&priv_rids); + if (ret < 0) { + ret = scoutfs_server_send_omap_response(sb, req->client_rid, req->client_id, + NULL, ret); + free_req(req); + } + + /* it's fine if we couldn't send to a client that left */ + if (ret == -ENOTCONN) + ret = 0; + + return ret; +} + +/* + * Handle all previously received omap requests from clients. Once + * we've finished recovery and can send requests to all clients we can + * handle all pending requests. The handling function frees the request + * and only returns an error if it couldn't send a response to the + * client. + */ +static int handle_requests(struct super_block *sb) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct llist_node *requests; + struct omap_request *req; + struct omap_request *tmp; + int ret; + int err; + + if (scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_GREETING)) + return 0; + + ret = 0; + requests = llist_del_all(&ominf->requests); + + llist_for_each_entry_safe(req, tmp, requests, llnode) { + err = handle_request(sb, req); + if (err < 0 && ret == 0) + ret = err; + } + + return ret; +} + +int scoutfs_omap_finished_recovery(struct super_block *sb) +{ + return handle_requests(sb); +} + +/* + * The server is receiving a request from a client for the bitmap of all + * open inodes around their ino. Queue it for processing which is + * typically immediate and inline but which can be deferred by recovery + * as the server first starts up. + */ +int scoutfs_omap_server_handle_request(struct super_block *sb, u64 rid, u64 id, + struct scoutfs_open_ino_map_args *args) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct omap_request *req; + + req = kzalloc(sizeof(struct omap_request), GFP_NOFS); + if (req == NULL) + return -ENOMEM; + + spin_lock_init(&req->lock); + req->client_rid = rid; + req->client_id = id; + init_rid_list(&req->rids); + req->map.args.group_nr = args->group_nr; + req->map.args.req_id = cpu_to_le64(atomic64_inc_return(&ominf->next_req_id)); + + llist_add(&req->llnode, &ominf->requests); + + return handle_requests(sb); +} + +/* + * The client is receiving a request from the server for its map for the + * given group. Look up the group and copy the bits to the map for + * non-zero open counts. + * + * The mount originating the request for this bitmap has the inode group + * write locked. We can't be adding links to any inodes in the group + * because that requires the lock. Inodes bits can be set and cleared + * while we're sampling the bitmap. These races are fine, they can't be + * adding cached inodes if nlink is 0 and we don't have the lock. If + * the caller is removing a set bit then they're about to try and delete + * the inode themselves and will first have to acquire the cluster lock + * themselves. + */ +int scoutfs_omap_client_handle_request(struct super_block *sb, u64 id, + struct scoutfs_open_ino_map_args *args) +{ + DECLARE_OMAP_INFO(sb, ominf); + u64 group_nr = le64_to_cpu(args->group_nr); + struct scoutfs_open_ino_map *map; + struct omap_group *group; + bool copied = false; + int ret; + + map = kmalloc(sizeof(struct scoutfs_open_ino_map), GFP_NOFS); + if (!map) + return -ENOMEM; + + map->args = *args; + + rcu_read_lock(); + group = rhashtable_lookup(&ominf->group_ht, &group_nr, group_ht_params); + if (group) { + spin_lock(&group->lock); + trace_group(sb, request, group, -1); + if (group->total > 0 && group->total < UINT_MAX) { + memcpy(map->bits, group->bits, sizeof(map->bits)); + copied = true; + } + spin_unlock(&group->lock); + } + rcu_read_unlock(); + + if (!copied) + memset(map->bits, 0, sizeof(map->bits)); + + ret = scoutfs_client_send_omap_response(sb, id, map); + kfree(map); + return ret; +} + +/* + * The server has received an open ino map response from a client. Find + * the original request that it's serving, or in the response's map, and + * send a reply if this was the last response from a client we were + * waiting for. + * + * We can get responses for requests we're no longer tracking if, for + * example, sending to a client gets an error. We'll have already sent + * the response to the requesting client so we drop these responses on + * the floor. + */ +int scoutfs_omap_server_handle_response(struct super_block *sb, u64 rid, + struct scoutfs_open_ino_map *resp_map) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct scoutfs_open_ino_map *map; + struct omap_rid_entry *entry; + bool send_response = false; + struct omap_request *req; + u64 resp_rid; + u64 resp_id; + int ret; + + map = kmalloc(sizeof(struct scoutfs_open_ino_map), GFP_NOFS); + if (!map) { + ret = -ENOMEM; + goto out; + } + + rcu_read_lock(); + req = rhashtable_lookup(&ominf->req_ht, &resp_map->args.req_id, req_ht_params); + if (req) { + spin_lock(&req->lock); + entry = find_rid(&req->rids, rid); + if (entry) { + bitmap_or((unsigned long *)req->map.bits, (unsigned long *)req->map.bits, + (unsigned long *)resp_map->bits, SCOUTFS_OPEN_INO_MAP_BITS); + if (free_rid(&req->rids, entry) == 0) + send_response = true; + } + spin_unlock(&req->lock); + + if (send_response) { + resp_rid = req->client_rid; + resp_id = req->client_id; + memcpy(map, &req->map, sizeof(struct scoutfs_open_ino_map)); + rhashtable_remove_fast(&ominf->req_ht, &req->ht_head, req_ht_params); + call_rcu(&req->rcu, free_req_rcu); + } + } + rcu_read_unlock(); + + if (send_response) + ret = scoutfs_server_send_omap_response(sb, resp_rid, resp_id, map, 0); + else + ret = 0; + kfree(map); +out: + return ret; +} + +/* + * The server is shutting down. Free all the server state associated + * with ongoing request processing. Clients who still have requests + * pending will resend them to the next server. + */ +void scoutfs_omap_server_shutdown(struct super_block *sb) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct rhashtable_iter iter; + struct llist_node *requests; + struct omap_request *req; + struct omap_request *tmp; + + rhashtable_walk_enter(&ominf->req_ht, &iter); + rhashtable_walk_start(&iter); + + for (;;) { + req = rhashtable_walk_next(&iter); + if (req == NULL) + break; + if (req == ERR_PTR(-EAGAIN)) + continue; + + if (req->rids.nr_rids != 0) { + free_rids(&req->rids); + rhashtable_remove_fast(&ominf->req_ht, &req->ht_head, req_ht_params); + call_rcu(&req->rcu, free_req_rcu); + } + } + + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); + + requests = llist_del_all(&ominf->requests); + llist_for_each_entry_safe(req, tmp, requests, llnode) + kfree(req); + + synchronize_rcu(); +} + +static bool omap_req_in_flight(struct scoutfs_lock *lock, struct scoutfs_omap_lock_data *ldata) +{ + bool in_flight; + + spin_lock(&lock->omap_spinlock); + in_flight = ldata->req_in_flight; + spin_unlock(&lock->omap_spinlock); + + return in_flight; +} + +/* + * Make sure the map covered by the cluster lock is current. The caller + * holds the cluster lock so once we store lock_data on the cluster lock + * it won't be freed and the write_version in the cluster lock won't + * change. + * + * The omap_spinlock protects the omap_data in the cluster lock. We + * have to drop it if we have to block to allocate lock_data, send a + * request for a new map, or wait for a request in flight to finish. + */ +static int get_current_lock_data(struct super_block *sb, struct scoutfs_lock *lock, + struct scoutfs_omap_lock_data **ldata_ret, u64 group_nr) +{ + struct scoutfs_omap_lock_data *ldata; + bool send_req; + int ret = 0; + + spin_lock(&lock->omap_spinlock); + + ldata = lock->omap_data; + if (ldata == NULL) { + spin_unlock(&lock->omap_spinlock); + ldata = kzalloc(sizeof(struct scoutfs_omap_lock_data), GFP_NOFS); + spin_lock(&lock->omap_spinlock); + + if (!ldata) { + ret = -ENOMEM; + goto out; + } + + if (lock->omap_data == NULL) { + ldata->version = lock->write_version - 1; /* ensure refresh */ + init_waitqueue_head(&ldata->waitq); + + lock->omap_data = ldata; + } else { + kfree(ldata); + ldata = lock->omap_data; + } + } + + while (ldata->version != lock->write_version) { + /* only one waiter sends a request at a time */ + if (!ldata->req_in_flight) { + ldata->req_in_flight = true; + send_req = true; + } else { + send_req = false; + } + + spin_unlock(&lock->omap_spinlock); + if (send_req) + ret = scoutfs_client_open_ino_map(sb, group_nr, &ldata->map); + else + wait_event(ldata->waitq, !omap_req_in_flight(lock, ldata)); + spin_lock(&lock->omap_spinlock); + + /* only sender can return error, other waiters retry */ + if (send_req) { + ldata->req_in_flight = false; + if (ret == 0) + ldata->version = lock->write_version; + wake_up(&ldata->waitq); + if (ret < 0) + goto out; + } + } + +out: + spin_unlock(&lock->omap_spinlock); + + if (ret == 0) + *ldata_ret = ldata; + else + *ldata_ret = NULL; + + return ret; +} + +/* + * Return 1 and give the caller a write inode lock if it is safe to be + * deleted. It's safe to be deleted when it is no longer reachable and + * nothing is referencing it. + * + * The inode is unreachable when nlink hits zero. Cluster locks protect + * modification and testing of nlink. We use the ino_lock_cov covrage + * to short circuit the common case of having a locked inode that hasn't + * been deleted. If it isn't locked, we have to acquire the lock to + * refresh the inode to see its current nlink. + * + * Then we use an open inode bitmap that covers all the inodes in the + * lock group to determine if the inode is present in any other mount's + * caches. We refresh it by asking the server for all clients' maps and + * then store it in the lock. As long as we hold the lock nothing can + * increase nlink from zero and let people get a reference to the inode. + */ +int scoutfs_omap_should_delete(struct super_block *sb, struct inode *inode, + struct scoutfs_lock **lock_ret) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct scoutfs_lock *lock = NULL; + const u64 ino = scoutfs_ino(inode); + struct scoutfs_omap_lock_data *ldata; + u64 group_nr; + int bit_nr; + int ret; + + /* lock group and omap constants are defined independently */ + BUILD_BUG_ON(SCOUTFS_OPEN_INO_MAP_BITS != SCOUTFS_LOCK_INODE_GROUP_NR); + + if (scoutfs_lock_is_covered(sb, &si->ino_lock_cov) && inode->i_nlink > 0) { + ret = 0; + goto out; + } + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, SCOUTFS_LKF_REFRESH_INODE, inode, &lock); + if (ret < 0) + goto out; + + if (inode->i_nlink > 0) { + ret = 0; + goto out; + } + + calc_group_nrs(ino, &group_nr, &bit_nr); + + /* only one request to refresh the map at a time */ + ret = get_current_lock_data(sb, lock, &ldata, group_nr); + if (ret < 0) + goto out; + + /* can delete caller's zero nlink inode if it's not cached in other mounts */ + ret = !test_bit_le(bit_nr, ldata->map.bits); +out: + trace_scoutfs_omap_should_delete(sb, ino, inode->i_nlink, ret); + + if (ret <= 0) { + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + lock = NULL; + } + + *lock_ret = lock; + return ret; +} + +void scoutfs_omap_free_lock_data(struct scoutfs_omap_lock_data *ldata) +{ + if (ldata) { + WARN_ON_ONCE(ldata->req_in_flight); + WARN_ON_ONCE(waitqueue_active(&ldata->waitq)); + kfree(ldata); + } +} + +int scoutfs_omap_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct omap_info *ominf; + int ret; + + ominf = kzalloc(sizeof(struct omap_info), GFP_KERNEL); + if (!ominf) { + ret = -ENOMEM; + goto out; + } + + ret = rhashtable_init(&ominf->group_ht, &group_ht_params); + if (ret < 0) { + kfree(ominf); + goto out; + } + + ret = rhashtable_init(&ominf->req_ht, &req_ht_params); + if (ret < 0) { + rhashtable_destroy(&ominf->group_ht); + kfree(ominf); + goto out; + } + + init_llist_head(&ominf->requests); + spin_lock_init(&ominf->lock); + init_rid_list(&ominf->rids); + atomic64_set(&ominf->next_req_id, 0); + + sbi->omap_info = ominf; + ret = 0; +out: + return ret; +} + +/* + * To get here the server must have shut down, freeing requests, and + * evict must have been called on all cached inodes so we can just + * synchronize all the pending group frees. + */ +void scoutfs_omap_destroy(struct super_block *sb) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct rhashtable_iter iter; + + if (ominf) { + synchronize_rcu(); + + /* double check that all the groups deced to 0 and were freed */ + rhashtable_walk_enter(&ominf->group_ht, &iter); + rhashtable_walk_start(&iter); + WARN_ON_ONCE(rhashtable_walk_peek(&iter) != NULL); + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); + + rhashtable_destroy(&ominf->group_ht); + rhashtable_destroy(&ominf->req_ht); + kfree(ominf); + sbi->omap_info = NULL; + } +} diff --git a/kmod/src/omap.h b/kmod/src/omap.h new file mode 100644 index 00000000..0e5fa29d --- /dev/null +++ b/kmod/src/omap.h @@ -0,0 +1,24 @@ +#ifndef _SCOUTFS_OMAP_H_ +#define _SCOUTFS_OMAP_H_ + +int scoutfs_omap_inc(struct super_block *sb, u64 ino); +void scoutfs_omap_dec(struct super_block *sb, u64 ino); +int scoutfs_omap_should_delete(struct super_block *sb, struct inode *inode, + struct scoutfs_lock **lock_ret); +void scoutfs_omap_free_lock_data(struct scoutfs_omap_lock_data *ldata); +int scoutfs_omap_client_handle_request(struct super_block *sb, u64 id, + struct scoutfs_open_ino_map_args *args); + +int scoutfs_omap_add_rid(struct super_block *sb, u64 rid); +int scoutfs_omap_remove_rid(struct super_block *sb, u64 rid); +int scoutfs_omap_finished_recovery(struct super_block *sb); +int scoutfs_omap_server_handle_request(struct super_block *sb, u64 rid, u64 id, + struct scoutfs_open_ino_map_args *args); +int scoutfs_omap_server_handle_response(struct super_block *sb, u64 rid, + struct scoutfs_open_ino_map *resp_map); +void scoutfs_omap_server_shutdown(struct super_block *sb); + +int scoutfs_omap_setup(struct super_block *sb); +void scoutfs_omap_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 91504e9c..8d1209e7 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2402,6 +2402,89 @@ TRACE_EVENT(scoutfs_item_invalidate_page, sk_trace_args(pg_start), sk_trace_args(pg_end), __entry->pgi) ); +DECLARE_EVENT_CLASS(scoutfs_omap_group_class, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(void *, grp) + __field(__u64, group_nr) + __field(unsigned int, group_total) + __field(int, bit_nr) + __field(int, bit_count) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->grp = grp; + __entry->group_nr = group_nr; + __entry->group_total = group_total; + __entry->bit_nr = bit_nr; + __entry->bit_count = bit_count; + ), + + TP_printk(SCSBF" grp %p group_nr %llu group_total %u bit_nr %d bit_count %d", + SCSB_TRACE_ARGS, __entry->grp, __entry->group_nr, __entry->group_total, + __entry->bit_nr, __entry->bit_count) +); + +DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_alloc, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count) +); +DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_free, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count) +); +DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_inc, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count) +); +DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_dec, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count) +); +DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_request, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count) +); +DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_destroy, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count) +); + +TRACE_EVENT(scoutfs_omap_should_delete, + TP_PROTO(struct super_block *sb, u64 ino, unsigned int nlink, int ret), + + TP_ARGS(sb, ino, nlink, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(unsigned int, nlink) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->nlink = nlink; + __entry->ret = ret; + ), + + TP_printk(SCSBF" ino %llu nlink %u ret %d", + SCSB_TRACE_ARGS, __entry->ino, __entry->nlink, __entry->ret) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/server.c b/kmod/src/server.c index 0210dbe9..369b2797 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -39,6 +39,7 @@ #include "alloc.h" #include "forest.h" #include "recov.h" +#include "omap.h" /* * Every active mount can act as the server that listens on a net @@ -1020,6 +1021,60 @@ out: return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); } +/* The server is receiving an omap response from the client */ +static int open_ino_map_response(struct super_block *sb, struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, int error, void *data) +{ + u64 rid = scoutfs_net_client_rid(conn); + + if (resp_len != sizeof(struct scoutfs_open_ino_map)) + return -EINVAL; + + return scoutfs_omap_server_handle_response(sb, rid, resp); +} + +/* The server is sending an omap request to the client */ +int scoutfs_server_send_omap_request(struct super_block *sb, u64 rid, + struct scoutfs_open_ino_map_args *args) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + + return scoutfs_net_submit_request_node(sb, server->conn, rid, SCOUTFS_NET_CMD_OPEN_INO_MAP, + args, sizeof(*args), + open_ino_map_response, NULL, NULL); +} + +/* The server is sending an omap response to the client */ +int scoutfs_server_send_omap_response(struct super_block *sb, u64 rid, u64 id, + struct scoutfs_open_ino_map *map, int err) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + + return scoutfs_net_response_node(sb, server->conn, rid, + SCOUTFS_NET_CMD_OPEN_INO_MAP, id, err, + map, sizeof(*map)); +} + +/* The server is receiving an omap request from the client */ +static int server_open_ino_map(struct super_block *sb, struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + u64 rid = scoutfs_net_client_rid(conn); + int ret; + + if (arg_len != sizeof(struct scoutfs_open_ino_map_args)) { + ret = -EINVAL; + goto out; + } + + ret = scoutfs_omap_server_handle_request(sb, rid, id, arg); +out: + if (ret < 0) + return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); + + return 0; +} + static void init_mounted_client_key(struct scoutfs_key *key, u64 rid) { *key = (struct scoutfs_key) { @@ -1237,9 +1292,10 @@ send_err: scoutfs_net_server_greeting(sb, conn, le64_to_cpu(gr->rid), id, reconnecting, first_contact, farewell); - /* lock server might send recovery request */ + /* let layers know we have a client connecting for the first time */ if (le64_to_cpu(gr->server_term) != server->term) { - ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid)); + ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid)) ?: + scoutfs_omap_add_rid(sb, le64_to_cpu(gr->rid)); if (ret) goto out; } @@ -1273,7 +1329,8 @@ static int reclaim_rid(struct super_block *sb, u64 rid) remove_trans_seq(sb, rid) ?: reclaim_log_trees(sb, rid) ?: cancel_srch_compact(sb, rid) ?: - delete_mounted_client(sb, rid); + delete_mounted_client(sb, rid) ?: + scoutfs_omap_remove_rid(sb, rid); return scoutfs_server_apply_commit(sb, ret); } @@ -1507,6 +1564,7 @@ static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_LOCK] = server_lock, [SCOUTFS_NET_CMD_SRCH_GET_COMPACT] = server_srch_get_compact, [SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT] = server_srch_commit_compact, + [SCOUTFS_NET_CMD_OPEN_INO_MAP] = server_open_ino_map, [SCOUTFS_NET_CMD_FAREWELL] = server_farewell, }; @@ -1564,7 +1622,8 @@ static void finished_recovery(struct super_block *sb) scoutfs_info(sb, "all clients recovered"); - ret = scoutfs_lock_server_finished_recovery(sb); + ret = scoutfs_omap_finished_recovery(sb) ?: + scoutfs_lock_server_finished_recovery(sb); if (ret < 0) { scoutfs_err(sb, "error %d resuming after recovery finished, shutting down", ret); stop_server(server); @@ -1787,6 +1846,7 @@ shutdown: flush_work(&server->commit_work); scoutfs_lock_server_destroy(sb); + scoutfs_omap_server_shutdown(sb); out: scoutfs_net_free_conn(sb, conn); diff --git a/kmod/src/server.h b/kmod/src/server.h index e06c0818..8d31a271 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -66,6 +66,11 @@ int scoutfs_server_hold_commit(struct super_block *sb); int scoutfs_server_apply_commit(struct super_block *sb, int err); void scoutfs_server_recov_finish(struct super_block *sb, u64 rid, int which); +int scoutfs_server_send_omap_request(struct super_block *sb, u64 rid, + struct scoutfs_open_ino_map_args *args); +int scoutfs_server_send_omap_response(struct super_block *sb, u64 rid, u64 id, + struct scoutfs_open_ino_map *map, int err); + struct sockaddr_in; struct scoutfs_quorum_elected_info; int scoutfs_server_start(struct super_block *sb, u64 term); diff --git a/kmod/src/super.c b/kmod/src/super.c index e66160ed..27b2f832 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -45,6 +45,7 @@ #include "item.h" #include "alloc.h" #include "recov.h" +#include "omap.h" #include "scoutfs_trace.h" static struct dentry *scoutfs_debugfs_root; @@ -264,6 +265,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_recov_destroy(sb); scoutfs_net_destroy(sb); scoutfs_lock_destroy(sb); + scoutfs_omap_destroy(sb); scoutfs_block_destroy(sb); scoutfs_destroy_triggers(sb); @@ -593,6 +595,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_inode_setup(sb) ?: scoutfs_data_setup(sb) ?: scoutfs_setup_trans(sb) ?: + scoutfs_omap_setup(sb) ?: scoutfs_lock_setup(sb) ?: scoutfs_net_setup(sb) ?: scoutfs_recov_setup(sb) ?: diff --git a/kmod/src/super.h b/kmod/src/super.h index 8d004db3..a74166bf 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -27,6 +27,7 @@ struct block_info; struct forest_info; struct srch_info; struct recov_info; +struct omap_info; struct scoutfs_sb_info { struct super_block *sb; @@ -49,6 +50,7 @@ struct scoutfs_sb_info { struct block_info *block_info; struct forest_info *forest_info; struct srch_info *srch_info; + struct omap_info *omap_info; struct item_cache_info *item_cache_info; wait_queue_head_t trans_hold_wq; From b244b2d59cea6204855e0762d4d704314b1c2d05 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 19 Mar 2021 09:22:42 -0700 Subject: [PATCH 07/19] Add inode-deletion test Signed-off-by: Zach Brown --- tests/golden/inode-deletion | 27 ++++++++++ tests/sequence | 1 + tests/tests/inode-deletion.sh | 98 +++++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 tests/golden/inode-deletion create mode 100644 tests/tests/inode-deletion.sh diff --git a/tests/golden/inode-deletion b/tests/golden/inode-deletion new file mode 100644 index 00000000..1e36ff37 --- /dev/null +++ b/tests/golden/inode-deletion @@ -0,0 +1,27 @@ +== basic unlink deletes +ino found in dseq index +ino not found in dseq index +== local open-unlink waits for close to delete +contents after rm: contents +ino found in dseq index +ino not found in dseq index +== multiple local opens are protected +contents after rm 1: contents +contents after rm 2: contents +ino found in dseq index +ino not found in dseq index +== remote unopened unlink deletes +ino not found in dseq index +ino not found in dseq index +== unlink wait for open on other mount +mount 0 contents after mount 1 rm: contents +ino found in dseq index +ino found in dseq index +stat: cannot stat ‘/mnt/test/test/inode-deletion/file’: No such file or directory +ino not found in dseq index +ino not found in dseq index +== lots of deletions use one open map +== open files survive remote scanning orphans +mount 0 contents after mount 1 remounted: contents +ino not found in dseq index +ino not found in dseq index diff --git a/tests/sequence b/tests/sequence index fe988b3a..e98afbad 100644 --- a/tests/sequence +++ b/tests/sequence @@ -30,4 +30,5 @@ mount-unmount-race.sh createmany-parallel-mounts.sh archive-light-cycle.sh block-stale-reads.sh +inode-deletion.sh xfstests.sh diff --git a/tests/tests/inode-deletion.sh b/tests/tests/inode-deletion.sh new file mode 100644 index 00000000..5af97354 --- /dev/null +++ b/tests/tests/inode-deletion.sh @@ -0,0 +1,98 @@ +# +# test deleting an inode once all its links and references are gone. +# + +t_require_commands cat scoutfs +t_require_mounts 2 + +FILE="$T_D0/file" + +check_ino_index() { + local ino="$1" + local dseq="$2" + local mnt="$3" + + t_sync_seq_index + + scoutfs walk-inodes -p "$mnt" -- data_seq $dseq $(($dseq + 1)) | + awk 'BEGIN { not = "not " } + ($4 == '$ino') { not = ""; exit; } + END { print "ino " not "found in dseq index" }' +} + +echo "== basic unlink deletes" +echo "contents" > "$FILE" +ino=$(stat -c "%i" "$FILE") +dseq=$(scoutfs stat -s data_seq "$FILE") +check_ino_index "$ino" "$dseq" "$T_M0" +rm -f "$FILE" +check_ino_index "$ino" "$dseq" "$T_M0" + +echo "== local open-unlink waits for close to delete" +echo "contents" > "$FILE" +ino=$(stat -c "%i" "$FILE") +dseq=$(scoutfs stat -s data_seq "$FILE") +exec {FD}<"$FILE" # open unused fd, assign to FD +rm -f "$FILE" +echo "contents after rm: $(cat <&$FD)" +check_ino_index "$ino" "$dseq" "$T_M0" +exec {FD}>&- # close +check_ino_index "$ino" "$dseq" "$T_M0" + +echo "== multiple local opens are protected" +echo "contents" > "$FILE" +ino=$(stat -c "%i" "$FILE") +dseq=$(scoutfs stat -s data_seq "$FILE") +exec {FD1}<"$FILE" +exec {FD2}<"$FILE" +rm -f "$FILE" +echo "contents after rm 1: $(cat <&$FD1)" +echo "contents after rm 2: $(cat <&$FD2)" +check_ino_index "$ino" "$dseq" "$T_M0" +exec {FD1}>&- # close +exec {FD2}>&- # close +check_ino_index "$ino" "$dseq" "$T_M0" + +echo "== remote unopened unlink deletes" +echo "contents" > "$T_D0/file" +ino=$(stat -c "%i" "$T_D0/file") +dseq=$(scoutfs stat -s data_seq "$T_D0/file") +rm -f "$T_D1/file" +check_ino_index "$ino" "$dseq" "$T_M0" +check_ino_index "$ino" "$dseq" "$T_M1" + +echo "== unlink wait for open on other mount" +echo "contents" > "$T_D0/file" +ino=$(stat -c "%i" "$T_D0/file") +dseq=$(scoutfs stat -s data_seq "$T_D0/file") +exec {FD}<"$T_D0/file" +rm -f "$T_D1/file" +echo "mount 0 contents after mount 1 rm: $(cat <&$FD)" +check_ino_index "$ino" "$dseq" "$T_M0" +check_ino_index "$ino" "$dseq" "$T_M1" +exec {FD}>&- # close +# we know that revalidating will unhash the remote dentry +stat "$T_D0/file" 2>&1 | t_filter_fs +check_ino_index "$ino" "$dseq" "$T_M0" +check_ino_index "$ino" "$dseq" "$T_M1" + +echo "== lots of deletions use one open map" +mkdir "$T_D0/dir" +touch "$T_D0/dir"/files-{1..5} +rm -f "$T_D0/dir"/files-* +rmdir "$T_D0/dir" + +echo "== open files survive remote scanning orphans" +echo "contents" > "$T_D0/file" +ino=$(stat -c "%i" "$T_D0/file") +dseq=$(scoutfs stat -s data_seq "$T_D0/file") +exec {FD}<"$T_D0/file" +rm -f "$T_D0/file" +t_umount 1 +t_mount 1 +echo "mount 0 contents after mount 1 remounted: $(cat <&$FD)" +exec {FD}>&- # close +check_ino_index "$ino" "$dseq" "$T_M0" +check_ino_index "$ino" "$dseq" "$T_M1" + +t_pass From 715c29aad3d340506554ef367209e6a066bfacc2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 22 Mar 2021 09:33:00 -0700 Subject: [PATCH 08/19] Proactively drop dentry/inode caches outside locks Previously we wouldn't try and remove cached dentries and inodes as lock revocation removed cluster lock coverage. The next time we tried to use the cached dentries or inodes we'd acquire a lock and refresh them. But now cached inodes prevent final inode deletion. If they linger outside cluster locking then any final deletion will need to be deferred until all its cached inodes are naturally dropped at some point in the future across the cluster. It might take refreshing the dentries or for memory pressure to push out the old cached inodes. This tries to proctively drop cached dentries and inodes as we lose cluster lock coverage if they're not actively referenced. We need to be careful not to perform final inode deletion during lock invalidation because it will deadlock, so we defer an iput which could delete during evict out to async work. Now deletion can be done synchronously in the task that is performing the unlink because previous use of the inode on remote mounts hasn't left unused cached inodes sitting around. Signed-off-by: Zach Brown --- kmod/src/inode.c | 28 ++++++++++++-- kmod/src/inode.h | 5 +++ kmod/src/lock.c | 82 +++++++++++++++++++++++++++++++++++++--- kmod/src/lock.h | 1 + kmod/src/scoutfs_trace.h | 11 ++++-- kmod/src/super.c | 3 ++ 6 files changed, 116 insertions(+), 14 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 97a68aeb..87f270c3 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -84,6 +84,7 @@ static void scoutfs_inode_ctor(void *obj) init_rwsem(&si->xattr_rwsem); RB_CLEAR_NODE(&si->writeback_node); scoutfs_lock_init_coverage(&si->ino_lock_cov); + atomic_set(&si->inv_iput_count, 0); inode_init_once(&si->inode); } @@ -313,6 +314,7 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, load_inode(inode, &sinode); atomic64_set(&si->last_refreshed, refresh_gen); scoutfs_lock_add_coverage(sb, lock, &si->ino_lock_cov); + si->drop_invalidated = false; } } else { ret = 0; @@ -1393,6 +1395,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, si->have_item = false; atomic64_set(&si->last_refreshed, lock->refresh_gen); scoutfs_lock_add_coverage(sb, lock, &si->ino_lock_cov); + si->drop_invalidated = false; si->flags = 0; scoutfs_inode_set_meta_seq(inode); @@ -1586,13 +1589,30 @@ clear: clear_inode(inode); } +/* + * We want to remove inodes from the cache as their count goes to 0 if + * they're no longer covered by a cluster lock or if while locked they + * were unlinked. + * + * We don't want unused cached inodes to linger outside of cluster + * locking so that they don't prevent final inode deletion on other + * nodes. We don't have specific per-inode or per-dentry locks which + * would otherwise remove the stale caches as they're invalidated. + * Stale cached inodes provide little value because they're going to be + * refreshed the next time they're locked. Populating the item cache + * and loading the inode item is a lot more expensive than initializing + * and inserting a newly allocated vfs inode. + */ int scoutfs_drop_inode(struct inode *inode) { - int ret = generic_drop_inode(inode); + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; - trace_scoutfs_drop_inode(inode->i_sb, scoutfs_ino(inode), - inode->i_nlink, inode_unhashed(inode)); - return ret; + trace_scoutfs_drop_inode(sb, scoutfs_ino(inode), inode->i_nlink, inode_unhashed(inode), + si->drop_invalidated); + + return si->drop_invalidated || !scoutfs_lock_is_covered(sb, &si->ino_lock_cov) || + generic_drop_inode(inode); } /* diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 60213d73..070d6492 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -53,6 +53,11 @@ struct scoutfs_inode_info { struct scoutfs_lock_coverage ino_lock_cov; + /* drop if i_count hits 0, allows drop while invalidate holds coverage */ + bool drop_invalidated; + struct llist_node inv_iput_llnode; + atomic_t inv_iput_count; + struct inode inode; }; diff --git a/kmod/src/lock.c b/kmod/src/lock.c index f761ba49..ba893bfe 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -75,6 +75,7 @@ struct lock_info { struct super_block *sb; spinlock_t lock; bool shutdown; + bool unmounting; struct rb_root lock_tree; struct rb_root lock_range_tree; struct shrinker shrinker; @@ -88,6 +89,9 @@ struct lock_info { struct work_struct shrink_work; struct list_head shrink_list; atomic64_t next_refresh_gen; + struct work_struct inv_iput_work; + struct llist_head inv_iput_llist; + struct dentry *tseq_dentry; struct scoutfs_tseq_tree tseq_tree; }; @@ -122,12 +126,53 @@ static bool lock_modes_match(int granted, int requested) requested == SCOUTFS_LOCK_READ); } +/* + * Final iput can get into evict and perform final inode deletion which + * can delete a lot of items under locks and transactions. We really + * don't want to be doing all that in an iput during invalidation. When + * invalidation sees that iput might perform final deletion it puts them + * on a list and queues this work. + * + * Nothing stops multiple puts for multiple invalidations of an inode + * before the work runs so we can track multiple puts in flight. + */ +static void lock_inv_iput_worker(struct work_struct *work) +{ + struct lock_info *linfo = container_of(work, struct lock_info, inv_iput_work); + struct scoutfs_inode_info *si; + struct scoutfs_inode_info *tmp; + struct llist_node *inodes; + bool more; + + inodes = llist_del_all(&linfo->inv_iput_llist); + + llist_for_each_entry_safe(si, tmp, inodes, inv_iput_llnode) { + do { + more = atomic_dec_return(&si->inv_iput_count) > 0; + iput(&si->inode); + } while (more); + } +} + /* * invalidate cached data associated with an inode whose lock is going * away. + * + * We try to drop cached dentries and inodes covered by the lock if they + * aren't referenced. This removes them from the mount's open map and + * allows deletions to be performed by unlink without having to wait for + * remote cached inodes to be dropped. + * + * If the cached inode was already deferring final inode deletion then + * we can't perform that inline in invalidation. The locking alone + * deadlock, and it might also take multiple transactions to fully + * delete an inode with significant metadata. We only perform the iput + * inline if we know that possible eviction can't perform the final + * deletion, otherwise we kick it off to async work. */ static void invalidate_inode(struct super_block *sb, u64 ino) { + DECLARE_LOCK_INFO(sb, linfo); struct scoutfs_inode_info *si; struct inode *inode; @@ -141,7 +186,20 @@ static void invalidate_inode(struct super_block *sb, u64 ino) scoutfs_data_wait_changed(inode); } - iput(inode); + /* can't touch during unmount, dcache destroys w/o locks */ + if (!linfo->unmounting) + d_prune_aliases(inode); + + si->drop_invalidated = true; + if (scoutfs_lock_is_covered(sb, &si->ino_lock_cov) && inode->i_nlink > 0) { + iput(inode); + } else { + /* defer iput to work context so we don't evict inodes from invalidation */ + if (atomic_inc_return(&si->inv_iput_count) == 1) + llist_add(&si->inv_iput_llnode, &linfo->inv_iput_llist); + smp_wmb(); /* count and list visible before work executes */ + queue_work(linfo->workq, &linfo->inv_iput_work); + } } } @@ -1536,11 +1594,21 @@ static void lock_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent) } /* - * The caller is going to be calling _destroy soon and, critically, is - * about to shutdown networking before calling us so that we don't get - * any callbacks while we're destroying. We have to ensure that we - * won't call networking after this returns. - * + * shrink_dcache_for_umount() tears down dentries with no locking. We + * need to make sure that our invalidation won't touch dentries before + * we return and the caller calls the generic vfs unmount path. + */ +void scoutfs_lock_unmount_begin(struct super_block *sb) +{ + DECLARE_LOCK_INFO(sb, linfo); + + if (linfo) { + linfo->unmounting = true; + flush_delayed_work(&linfo->inv_dwork); + } +} + +/* * Internal fs threads can be using locking, and locking can have async * work pending. We use ->shutdown to force callers to return * -ESHUTDOWN and to prevent the future queueing of work that could call @@ -1682,6 +1750,8 @@ int scoutfs_lock_setup(struct super_block *sb) INIT_WORK(&linfo->shrink_work, lock_shrink_worker); INIT_LIST_HEAD(&linfo->shrink_list); atomic64_set(&linfo->next_refresh_gen, 0); + INIT_WORK(&linfo->inv_iput_work, lock_inv_iput_worker); + init_llist_head(&linfo->inv_iput_llist); scoutfs_tseq_tree_init(&linfo->tseq_tree, lock_tseq_show); sbi->lock_info = linfo; diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 5485bb35..bf9a9b8a 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -104,6 +104,7 @@ bool scoutfs_lock_protected(struct scoutfs_lock *lock, struct scoutfs_key *key, void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr); int scoutfs_lock_setup(struct super_block *sb); +void scoutfs_lock_unmount_begin(struct super_block *sb); void scoutfs_lock_shutdown(struct super_block *sb); void scoutfs_lock_destroy(struct super_block *sb); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 8d1209e7..7dce85f0 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -690,15 +690,16 @@ TRACE_EVENT(scoutfs_evict_inode, TRACE_EVENT(scoutfs_drop_inode, TP_PROTO(struct super_block *sb, __u64 ino, unsigned int nlink, - unsigned int unhashed), + unsigned int unhashed, bool drop_invalidated), - TP_ARGS(sb, ino, nlink, unhashed), + TP_ARGS(sb, ino, nlink, unhashed, drop_invalidated), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, ino) __field(unsigned int, nlink) __field(unsigned int, unhashed) + __field(unsigned int, drop_invalidated) ), TP_fast_assign( @@ -706,10 +707,12 @@ TRACE_EVENT(scoutfs_drop_inode, __entry->ino = ino; __entry->nlink = nlink; __entry->unhashed = unhashed; + __entry->drop_invalidated = !!drop_invalidated; ), - TP_printk(SCSBF" ino %llu nlink %u unhashed %d", SCSB_TRACE_ARGS, - __entry->ino, __entry->nlink, __entry->unhashed) + TP_printk(SCSBF" ino %llu nlink %u unhashed %d drop_invalidated %u", SCSB_TRACE_ARGS, + __entry->ino, __entry->nlink, __entry->unhashed, + __entry->drop_invalidated) ); TRACE_EVENT(scoutfs_inode_walk_writeback, diff --git a/kmod/src/super.c b/kmod/src/super.c index 27b2f832..5338ec65 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -649,6 +649,9 @@ static void scoutfs_kill_sb(struct super_block *sb) { trace_scoutfs_kill_sb(sb); + if (SCOUTFS_HAS_SBI(sb)) + scoutfs_lock_unmount_begin(sb); + kill_block_super(sb); } From dba88705f7a1b0c92374782ad661fb3765b50a37 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 26 Feb 2021 10:10:17 -0800 Subject: [PATCH 09/19] Fix t_umount mount point number t_umount had a typo that had it try to unmount a mount based on a caller's variable, which accidentally happened to work for its only caller. Future callers would not have been so lucky. Signed-off-by: Zach Brown --- tests/funcs/fs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/funcs/fs.sh b/tests/funcs/fs.sh index b4b17cf3..03fb9888 100644 --- a/tests/funcs/fs.sh +++ b/tests/funcs/fs.sh @@ -129,7 +129,7 @@ t_umount() test "$nr" -lt "$T_NR_MOUNTS" || \ t_fail "fs nr $nr invalid" - eval t_quiet umount \$T_M$i + eval t_quiet umount \$T_M$nr } # From 4389c73c14e48fa058a46f434f92059661dfacc0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 Mar 2021 10:20:34 -0700 Subject: [PATCH 10/19] Fix deadlock between lock invalidate and evict We've had a long-standing deadlock between lock invalidation and eviction. Invalidating a lock wants to lookup inodes and drop their resources while blocking locks. Eviction wants to get a lock to perform final deletion while the inodes has I_FREEING set which blocks lookups. We only saw this deadlock a handful of times in all of the time we've run the code, but it's now much more common now that we're acquiring locks in iput to test that nlink is zero instead of only when nlink is zero. I see unmount hang regularly when testing final inode deletion. This adds a lookup variant for invalidation which will refuse to return freeing inodes so they won't be waited on. Once they're freeing they can't be seen by future lock users so they don't need to be invalidated. This keeps the lock invalication promise and avoids sleeping on freeing inodes which creates the deadlock. Signed-off-by: Zach Brown --- kmod/src/inode.c | 22 ++++++++++++++++++++++ kmod/src/inode.h | 1 + kmod/src/lock.c | 8 +++++--- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 87f270c3..cdade67d 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -672,6 +672,28 @@ struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino) return ilookup5(sb, ino, scoutfs_iget_test, &ino); } +static int iget_test_nofreeing(struct inode *inode, void *arg) +{ + return !(inode->i_state & I_FREEING) && scoutfs_iget_test(inode, arg); +} + +/* + * There's a natural risk of a deadlock between lock invalidation and + * eviction. Invalidation blocks locks while looking up inodes and + * invalidating local caches. Inode eviction gets a lock to check final + * inode deletion while the inode is marked FREEING which blocks + * lookups. + * + * We have a lookup variant which doesn't return I_FREEING inodes + * instead of waiting on them. If an inode has made it to I_FREEING + * then it doesn't have any local caches that are reachable and the lock + * invalidation promise is kept. + */ +struct inode *scoutfs_ilookup_nofreeing(struct super_block *sb, u64 ino) +{ + return ilookup5(sb, ino, iget_test_nofreeing, &ino); +} + struct inode *scoutfs_iget(struct super_block *sb, u64 ino) { struct scoutfs_lock *lock = NULL; diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 070d6492..f1884ad6 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -79,6 +79,7 @@ int scoutfs_orphan_inode(struct inode *inode); struct inode *scoutfs_iget(struct super_block *sb, u64 ino); struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino); +struct inode *scoutfs_ilookup_nofreeing(struct super_block *sb, u64 ino); void scoutfs_inode_init_index_key(struct scoutfs_key *key, u8 type, u64 major, u32 minor, u64 ino); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index ba893bfe..a1ec92f7 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -155,8 +155,10 @@ static void lock_inv_iput_worker(struct work_struct *work) } /* - * invalidate cached data associated with an inode whose lock is going - * away. + * Invalidate cached data associated with an inode whose lock is going + * away. We ignore indoes with I_FREEING instead of waiting on them to + * avoid a deadlock, if they're freeing then they won't be visible to + * future lock users and we don't need to invalidate them. * * We try to drop cached dentries and inodes covered by the lock if they * aren't referenced. This removes them from the mount's open map and @@ -176,7 +178,7 @@ static void invalidate_inode(struct super_block *sb, u64 ino) struct scoutfs_inode_info *si; struct inode *inode; - inode = scoutfs_ilookup(sb, ino); + inode = scoutfs_ilookup_nofreeing(sb, ino); if (inode) { si = SCOUTFS_I(inode); From fead263af334d8a56632149c614c86f47802d48d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 Mar 2021 12:53:25 -0700 Subject: [PATCH 11/19] Remove unused sb_info shutdown We're no longer using the shutdown field in our sb info struct. Signed-off-by: Zach Brown --- kmod/src/super.c | 2 -- kmod/src/super.h | 2 -- 2 files changed, 4 deletions(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index 5338ec65..b72cc18b 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -245,8 +245,6 @@ static void scoutfs_put_super(struct super_block *sb) trace_scoutfs_put_super(sb); - sbi->shutdown = true; - scoutfs_data_destroy(sb); scoutfs_srch_destroy(sb); diff --git a/kmod/src/super.h b/kmod/src/super.h index a74166bf..1790d40f 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -85,8 +85,6 @@ struct scoutfs_sb_info { struct dentry *debug_root; - bool shutdown; - unsigned long corruption_messages_once[SC_NR_LONGS]; }; From 04f4b8bcb35b77fd5c16b71bab2fc5441ba38063 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 Mar 2021 13:31:52 -0700 Subject: [PATCH 12/19] Perform final transaction write before shutdown Shutting down the transaction during unmount relied on the vfs unmount path to perform a sync of any remaining dirty transaction. There are ways that we can dirty a transaction during unmount after it calls the fs sync, so we try to write any remaining dirty transaction before shutting down. Signed-off-by: Zach Brown --- kmod/src/trans.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 742919f6..186239d4 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -564,8 +564,15 @@ int scoutfs_setup_trans(struct super_block *sb) } /* - * kill_sb calls sync before getting here so we know that dirty data - * should be in flight. We just have to wait for it to quiesce. + * While the vfs will have done an fs level sync before calling + * put_super, we may have done work down in our level after all the fs + * ops were done. An example is final inode deletion in iput, that's + * done in generic_shutdown_super after the sync and before calling our + * put_super. + * + * So we always try to write any remaining dirty transactions before + * shutting down. Typically there won't be any dirty data and the + * worker will just return. */ void scoutfs_shutdown_trans(struct super_block *sb) { @@ -573,13 +580,18 @@ void scoutfs_shutdown_trans(struct super_block *sb) DECLARE_TRANS_INFO(sb, tri); if (tri) { - scoutfs_block_writer_forget_all(sb, &tri->wri); if (sbi->trans_write_workq) { + /* immediately queues pending timer */ + flush_delayed_work(&sbi->trans_write_work); + /* prevents re-arming if it has to wait */ cancel_delayed_work_sync(&sbi->trans_write_work); destroy_workqueue(sbi->trans_write_workq); /* trans work schedules after shutdown see null */ sbi->trans_write_workq = NULL; } + + scoutfs_block_writer_forget_all(sb, &tri->wri); + kfree(tri); sbi->trans_info = NULL; } From d9aea98220df67d09cb79d290eeb47bfcf8b8820 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 Mar 2021 16:08:27 -0700 Subject: [PATCH 13/19] Shutdown locking before transactions Shutting down the lock client waits for invalidation work and prevents future work from being queued. We're currently shutting down the subsystems that lock calls before lock itself, leading to crashes if we happen to have invalidations executing as we unmount. Shutting down locking before its dependencies fixes this. This was hit in testing during the inode deletion fixes because it created the perfect race by acquiring locks during unmount so that the server was very unlikely to send invalidations on behalf to one mount on behalf of another as they both unmounted. Signed-off-by: Zach Brown --- kmod/src/super.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index b72cc18b..96bc6aca 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -251,6 +251,8 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_unlock(sb, sbi->rid_lock, SCOUTFS_LOCK_WRITE); sbi->rid_lock = NULL; + scoutfs_lock_shutdown(sb); + scoutfs_shutdown_trans(sb); scoutfs_client_destroy(sb); scoutfs_inode_destroy(sb); @@ -258,7 +260,6 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_forest_destroy(sb); scoutfs_quorum_destroy(sb); - scoutfs_lock_shutdown(sb); scoutfs_server_destroy(sb); scoutfs_recov_destroy(sb); scoutfs_net_destroy(sb); From 1e88aa6c0ffaf0795be33a9df2d928a633cbdec6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 Mar 2021 16:17:52 -0700 Subject: [PATCH 14/19] Shutdown data after trans The data_info struct holds the data allocator that is filled by transactions as they commit. We have to free it after we've shutdown transactions. It's more like the forest in this regard so we move its desctruction down by the forest to group similar behaviour. Signed-off-by: Zach Brown --- kmod/src/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index 96bc6aca..3d0d2050 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -245,7 +245,6 @@ static void scoutfs_put_super(struct super_block *sb) trace_scoutfs_put_super(sb); - scoutfs_data_destroy(sb); scoutfs_srch_destroy(sb); scoutfs_unlock(sb, sbi->rid_lock, SCOUTFS_LOCK_WRITE); @@ -258,6 +257,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_inode_destroy(sb); scoutfs_item_destroy(sb); scoutfs_forest_destroy(sb); + scoutfs_data_destroy(sb); scoutfs_quorum_destroy(sb); scoutfs_server_destroy(sb); From 2949b6063f55006b3d56b85843436b353e69e78e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 5 Apr 2021 09:39:11 -0700 Subject: [PATCH 15/19] Clear lock invalidate_pending during destroy Locks have a bunch of state that reflects concurrent processing. Testing that state determines when it's safe to free a lock because nothing is going on. During unmount we abruptly stop processing locks. Unmount will send a farewell to the server which will remove all the state associated with the client that's unmounting for all its locks, regardless of the state the locks were in. The client unmount path has to clean up the interupted lock state and free it, carefully avoiding assertions that would otherwise indicate that we're freeing used locks. The move to async lock invalidation forgot to clean up the invalidation state. Previously a synchronous work function would set and clear invalidate_pending while it was running. Once we finished waiting for it invalidate_pending would be clear. The move to async invalidation work meant that we can still have invalidate_pending with no work executing. Lock destruction removed locks from the invalidation list but forgot to clear the invalidate_pending flag. This triggered assertions during unmount that were otherwise harmless. There was other use of the lock, we just forgot to clean up the lock state. Signed-off-by: Zach Brown --- kmod/src/lock.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index a1ec92f7..8f9ebbf9 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -1714,8 +1714,10 @@ void scoutfs_lock_destroy(struct super_block *sb) __lock_del_lru(linfo, lock); if (!list_empty(&lock->grant_head)) list_del_init(&lock->grant_head); - if (!list_empty(&lock->inv_head)) + if (!list_empty(&lock->inv_head)) { list_del_init(&lock->inv_head); + lock->invalidate_pending = 0; + } if (!list_empty(&lock->shrink_head)) list_del_init(&lock->shrink_head); lock_remove(linfo, lock); From ba8bf13ae1c5ebc32ca0cc08678dc041a9b9778a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 5 Apr 2021 09:48:47 -0700 Subject: [PATCH 16/19] Update dmesg whitelist for recovery The shared recovery layer outputs different messages than when it ran only for lock_recovery in the lock server. Signed-off-by: Zach Brown --- tests/funcs/filter.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/funcs/filter.sh b/tests/funcs/filter.sh index 8226adad..5c793be4 100644 --- a/tests/funcs/filter.sh +++ b/tests/funcs/filter.sh @@ -52,8 +52,8 @@ t_filter_dmesg() # tests that drop unmount io triggers fencing re="$re|scoutfs .* error: fencing " - re="$re|scoutfs .*: waiting for .* lock clients" - re="$re|scoutfs .*: all lock clients recovered" + re="$re|scoutfs .*: waiting for .* clients" + re="$re|scoutfs .*: all clients recovered" re="$re|scoutfs .* error: client rid.*lock recovery timed out" # some tests mount w/o options From 841d22e26e9532473a954e29520f3eb6497ad710 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 7 Apr 2021 14:27:16 -0700 Subject: [PATCH 17/19] Disable task reclaim flags for block cache vmalloc Even though we can pass in gfp flags to vmalloc it eventually calls pte alloc functions which ignore the caller's flags and use user gfp flags. This risks reclaim re-entering fs paths during allocations in the block cache. These allocs that allowed reclaim deep in the fs was causing lockdep to add RECLAIM dependencies between locks and holler about deadlocks. We apply the same pattern that xfs does for disabling reclaim while allocating vmalloced block payloads. Setting PF_MEMALLOC_NOIO causes reclaim in that task to clear __GFP_IO and __GFP_FS, regardless of the individual allocation flags in the task, preventing recursion. Signed-off-by: Zach Brown --- kmod/src/block.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 43dc2a66..53578725 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -128,6 +128,7 @@ static __le32 block_calc_crc(struct scoutfs_block_header *hdr, u32 size) static struct block_private *block_alloc(struct super_block *sb, u64 blkno) { struct block_private *bp; + unsigned int noio_flags; /* * If we had multiple blocks per page we'd need to be a little @@ -147,8 +148,19 @@ static struct block_private *block_alloc(struct super_block *sb, u64 blkno) set_bit(BLOCK_BIT_PAGE_ALLOC, &bp->bits); bp->bl.data = page_address(bp->page); } else { - bp->virt = __vmalloc(SCOUTFS_BLOCK_LG_SIZE, - GFP_NOFS | __GFP_HIGHMEM, PAGE_KERNEL); + /* + * __vmalloc doesn't pass the gfp flags down to pte + * allocs, they're done with user alloc flags. + * Unfortunately, some lockdep doesn't know that + * PF_NOMEMALLOC prevents __GFP_FS reclaim and generates + * spurious reclaim-on dependencies and warnings. + */ + lockdep_off(); + noio_flags = memalloc_noio_save(); + bp->virt = __vmalloc(SCOUTFS_BLOCK_LG_SIZE, GFP_NOFS | __GFP_HIGHMEM, PAGE_KERNEL); + memalloc_noio_restore(noio_flags); + lockdep_on(); + if (!bp->virt) { kfree(bp); bp = NULL; From 94dd86f7623809693a820ed90744f692b953fc64 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 12 Apr 2021 14:17:36 -0700 Subject: [PATCH 18/19] Process lock invalidation after shutdown Lock teardown during unmount involves first calling shutdown and then destroy. The shutdown call is meant to ensure that it's safe to tear down the client network connections. Once shutdown returns locking is promising that it won't call into the client to send new lock requests. The current shutdown implementation is very heavy handed and shuts down everything. This creates a deadlock. After calling lock shutdown, the client will send its farewell and wait for a response. The server might not send the farewell response until other mounts have unmounted if our client is in the server's mount. In this case we stil have to be processing lock invalidation requests to allow other unmounting clients to make forward progress. This is reasonably easy and safe to do. We only use the shutdown flag to stop lock calls that would change lock state and send requests. We don't have it stop incoming requests processing in the work queueing functions. It's safe to keep processing incoming requests between _shutdown and _destroy because the requests already come in through the client. As the client shuts down it will stop calling us. Signed-off-by: Zach Brown --- kmod/src/lock.c | 51 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 8f9ebbf9..d136e95f 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -621,7 +621,7 @@ static void queue_grant_work(struct lock_info *linfo) { assert_spin_locked(&linfo->lock); - if (!list_empty(&linfo->grant_list) && !linfo->shutdown) + if (!list_empty(&linfo->grant_list)) queue_work(linfo->workq, &linfo->grant_work); } @@ -637,7 +637,7 @@ static void queue_inv_work(struct lock_info *linfo) { assert_spin_locked(&linfo->lock); - if (!list_empty(&linfo->inv_list) && !linfo->shutdown) + if (!list_empty(&linfo->inv_list)) mod_delayed_work(linfo->workq, &linfo->inv_dwork, 0); } @@ -868,8 +868,11 @@ static void lock_invalidate_worker(struct work_struct *work) nl = &lock->inv_nl; net_id = lock->inv_net_id; - ret = lock_invalidate(sb, lock, nl->old_mode, nl->new_mode); - BUG_ON(ret); + /* only lock protocol, inv can't call subsystems after shutdown */ + if (!linfo->shutdown) { + ret = lock_invalidate(sb, lock, nl->old_mode, nl->new_mode); + BUG_ON(ret); + } /* respond with the key and modes from the request */ ret = scoutfs_client_lock_response(sb, net_id, nl); @@ -1059,7 +1062,7 @@ static int lock_key_range(struct super_block *sb, enum scoutfs_lock_mode mode, i lock_inc_count(lock->waiters, mode); for (;;) { - if (linfo->shutdown) { + if (WARN_ON_ONCE(linfo->shutdown)) { ret = -ESHUTDOWN; break; } @@ -1541,7 +1544,7 @@ restart: BUG_ON(lock->mode == SCOUTFS_LOCK_NULL); BUG_ON(!list_empty(&lock->shrink_head)); - if (linfo->shutdown || nr-- == 0) + if (nr-- == 0) break; __lock_del_lru(linfo, lock); @@ -1611,10 +1614,25 @@ void scoutfs_lock_unmount_begin(struct super_block *sb) } /* - * Internal fs threads can be using locking, and locking can have async - * work pending. We use ->shutdown to force callers to return - * -ESHUTDOWN and to prevent the future queueing of work that could call - * networking. Locks whose work is stopped will be torn down by _destroy. + * The caller is going to be shutting down transactions and the client. + * We need to make sure that locking won't call either after we return. + * + * At this point all fs callers and internal services that use locks + * should have stopped. We won't have any callers initiating lock + * transitions and sending requests. We set the shutdown flag to catch + * anyone who breaks this rule. + * + * We unregister the shrinker so that we won't try and send null + * requests in response to memory pressure. The locks will all be + * unceremoniously dropped once we get a farewell response from the + * server which indicates that they destroyed our locking state. + * + * We will still respond to invalidation requests that have to be + * processed to let unmount in other mounts acquire locks and make + * progress. However, we don't fully process the invalidation because + * we're shutting down. We only update the lock state and send the + * response. We shouldn't have any users of locking that require + * invalidation correctness at this point. */ void scoutfs_lock_shutdown(struct super_block *sb) { @@ -1627,19 +1645,18 @@ void scoutfs_lock_shutdown(struct super_block *sb) trace_scoutfs_lock_shutdown(sb, linfo); - spin_lock(&linfo->lock); + /* stop the shrinker from queueing work */ + unregister_shrinker(&linfo->shrinker); + flush_work(&linfo->shrink_work); + /* cause current and future lock calls to return errors */ + spin_lock(&linfo->lock); linfo->shutdown = true; for (node = rb_first(&linfo->lock_tree); node; node = rb_next(node)) { lock = rb_entry(node, struct scoutfs_lock, node); wake_up(&lock->waitq); } - spin_unlock(&linfo->lock); - - flush_work(&linfo->grant_work); - flush_delayed_work(&linfo->inv_dwork); - flush_work(&linfo->shrink_work); } /* @@ -1667,8 +1684,6 @@ void scoutfs_lock_destroy(struct super_block *sb) trace_scoutfs_lock_destroy(sb, linfo); - /* stop the shrinker from queueing work */ - unregister_shrinker(&linfo->shrinker); /* make sure that no one's actively using locks */ spin_lock(&linfo->lock); From dfc2f7a4e844ae97a93d7de78b73c68c27fdf97e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 14 Apr 2021 10:19:13 -0700 Subject: [PATCH 19/19] Remove unused scoutfs_free_unused_locks nr arg The nr argument wasn't used. It always tries to free as many as the shrinker call will let it. Signed-off-by: Zach Brown --- kmod/src/lock.c | 2 +- kmod/src/lock.h | 2 +- kmod/src/super.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index d136e95f..120bcdf7 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -1571,7 +1571,7 @@ out: return ret; } -void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr) +void scoutfs_free_unused_locks(struct super_block *sb) { struct lock_info *linfo = SCOUTFS_SB(sb)->lock_info; struct shrink_control sc = { diff --git a/kmod/src/lock.h b/kmod/src/lock.h index bf9a9b8a..40f8f5b9 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -101,7 +101,7 @@ void scoutfs_lock_del_coverage(struct super_block *sb, bool scoutfs_lock_protected(struct scoutfs_lock *lock, struct scoutfs_key *key, enum scoutfs_lock_mode mode); -void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr); +void scoutfs_free_unused_locks(struct super_block *sb); int scoutfs_lock_setup(struct super_block *sb); void scoutfs_lock_unmount_begin(struct super_block *sb); diff --git a/kmod/src/super.c b/kmod/src/super.c index 3d0d2050..b691b038 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -168,7 +168,7 @@ out: * try to free as many locks as possible. */ if (scoutfs_trigger(sb, STATFS_LOCK_PURGE)) - scoutfs_free_unused_locks(sb, -1UL); + scoutfs_free_unused_locks(sb); return ret; }