diff --git a/kmod/src/Makefile b/kmod/src/Makefile index db5f5576..35f9fd07 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -27,9 +27,11 @@ scoutfs-y += \ lock_server.o \ msg.o \ net.o \ + omap.o \ options.o \ per_task.o \ quorum.o \ + recov.o \ scoutfs_trace.o \ server.o \ sort_priv.o \ diff --git a/kmod/src/block.c b/kmod/src/block.c index bb581173..53578725 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -128,6 +128,7 @@ static __le32 block_calc_crc(struct scoutfs_block_header *hdr, u32 size) static struct block_private *block_alloc(struct super_block *sb, u64 blkno) { struct block_private *bp; + unsigned int noio_flags; /* * If we had multiple blocks per page we'd need to be a little @@ -147,8 +148,19 @@ static struct block_private *block_alloc(struct super_block *sb, u64 blkno) set_bit(BLOCK_BIT_PAGE_ALLOC, &bp->bits); bp->bl.data = page_address(bp->page); } else { - bp->virt = __vmalloc(SCOUTFS_BLOCK_LG_SIZE, - GFP_NOFS | __GFP_HIGHMEM, PAGE_KERNEL); + /* + * __vmalloc doesn't pass the gfp flags down to pte + * allocs, they're done with user alloc flags. + * Unfortunately, some lockdep doesn't know that + * PF_NOMEMALLOC prevents __GFP_FS reclaim and generates + * spurious reclaim-on dependencies and warnings. + */ + lockdep_off(); + noio_flags = memalloc_noio_save(); + bp->virt = __vmalloc(SCOUTFS_BLOCK_LG_SIZE, GFP_NOFS | __GFP_HIGHMEM, PAGE_KERNEL); + memalloc_noio_restore(noio_flags); + lockdep_on(); + if (!bp->virt) { kfree(bp); bp = NULL; @@ -1245,7 +1257,7 @@ out: if (ret) scoutfs_block_destroy(sb); - return 0; + return ret; } void scoutfs_block_destroy(struct super_block *sb) diff --git a/kmod/src/client.c b/kmod/src/client.c index 642a0c5d..99c7ba06 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -31,6 +31,7 @@ #include "net.h" #include "endian_swap.h" #include "quorum.h" +#include "omap.h" /* * The client is responsible for maintaining a connection to the server. @@ -215,6 +216,39 @@ int scoutfs_client_srch_commit_compact(struct super_block *sb, res, sizeof(*res), NULL, 0); } +int scoutfs_client_send_omap_response(struct super_block *sb, u64 id, + struct scoutfs_open_ino_map *map) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_response(sb, client->conn, SCOUTFS_NET_CMD_OPEN_INO_MAP, + id, 0, map, sizeof(*map)); +} + +/* The client is receiving an omap request from the server */ +static int client_open_ino_map(struct super_block *sb, struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + if (arg_len != sizeof(struct scoutfs_open_ino_map_args)) + return -EINVAL; + + return scoutfs_omap_client_handle_request(sb, id, arg); +} + +/* The client is sending an omap request to the server */ +int scoutfs_client_open_ino_map(struct super_block *sb, u64 group_nr, + struct scoutfs_open_ino_map *map) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + struct scoutfs_open_ino_map_args args = { + .group_nr = cpu_to_le64(group_nr), + .req_id = 0, + }; + + return scoutfs_net_sync_request(sb, client->conn, SCOUTFS_NET_CMD_OPEN_INO_MAP, + &args, sizeof(args), map, sizeof(*map)); +} + /* The client is receiving a invalidation request from the server */ static int client_lock(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, @@ -413,6 +447,7 @@ out: static scoutfs_net_request_t client_req_funcs[] = { [SCOUTFS_NET_CMD_LOCK] = client_lock, [SCOUTFS_NET_CMD_LOCK_RECOVER] = client_lock_recover, + [SCOUTFS_NET_CMD_OPEN_INO_MAP] = client_open_ino_map, }; /* diff --git a/kmod/src/client.h b/kmod/src/client.h index ae830ef8..c569f038 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -22,6 +22,10 @@ int scoutfs_client_srch_get_compact(struct super_block *sb, struct scoutfs_srch_compact *sc); int scoutfs_client_srch_commit_compact(struct super_block *sb, struct scoutfs_srch_compact *res); +int scoutfs_client_send_omap_response(struct super_block *sb, u64 id, + struct scoutfs_open_ino_map *map); +int scoutfs_client_open_ino_map(struct super_block *sb, u64 group_nr, + struct scoutfs_open_ino_map *map); int scoutfs_client_setup(struct super_block *sb); void scoutfs_client_destroy(struct super_block *sb); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 42c70a4a..2223a4ab 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -30,6 +30,7 @@ #include "item.h" #include "lock.h" #include "hash.h" +#include "omap.h" #include "counters.h" #include "scoutfs_trace.h" diff --git a/kmod/src/file.c b/kmod/src/file.c index f35ef039..b1555b08 100644 --- a/kmod/src/file.c +++ b/kmod/src/file.c @@ -27,6 +27,7 @@ #include "file.h" #include "inode.h" #include "per_task.h" +#include "omap.h" /* TODO: Direct I/O, AIO */ ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov, diff --git a/kmod/src/format.h b/kmod/src/format.h index b67d066d..86d10d3d 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -195,9 +195,6 @@ struct scoutfs_key { #define sklt_rid _sk_first #define sklt_nr _sk_second -/* lock clients */ -#define sklc_rid _sk_first - /* seqs */ #define skts_trans_seq _sk_first #define skts_rid _sk_second @@ -493,11 +490,10 @@ struct scoutfs_bloom_block { #define SCOUTFS_LOCK_ZONE 4 /* Items only stored in server btrees */ #define SCOUTFS_LOG_TREES_ZONE 6 -#define SCOUTFS_LOCK_CLIENTS_ZONE 7 -#define SCOUTFS_TRANS_SEQ_ZONE 8 -#define SCOUTFS_MOUNTED_CLIENT_ZONE 9 -#define SCOUTFS_SRCH_ZONE 10 -#define SCOUTFS_FREE_EXTENT_ZONE 11 +#define SCOUTFS_TRANS_SEQ_ZONE 7 +#define SCOUTFS_MOUNTED_CLIENT_ZONE 8 +#define SCOUTFS_SRCH_ZONE 9 +#define SCOUTFS_FREE_EXTENT_ZONE 10 /* inode index zone */ #define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 1 @@ -653,7 +649,6 @@ struct scoutfs_super_block { struct scoutfs_alloc_list_head server_meta_freed[2]; struct scoutfs_btree_root fs_root; struct scoutfs_btree_root logs_root; - struct scoutfs_btree_root lock_clients; struct scoutfs_btree_root trans_seqs; struct scoutfs_btree_root mounted_clients; struct scoutfs_btree_root srch_root; @@ -845,6 +840,7 @@ enum scoutfs_net_cmd { SCOUTFS_NET_CMD_LOCK_RECOVER, SCOUTFS_NET_CMD_SRCH_GET_COMPACT, SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT, + SCOUTFS_NET_CMD_OPEN_INO_MAP, SCOUTFS_NET_CMD_FAREWELL, SCOUTFS_NET_CMD_UNKNOWN, }; @@ -965,4 +961,42 @@ enum scoutfs_corruption_sources { #define SC_NR_LONGS DIV_ROUND_UP(SC_NR_SOURCES, BITS_PER_LONG) +#define SCOUTFS_OPEN_INO_MAP_SHIFT 10 +#define SCOUTFS_OPEN_INO_MAP_BITS (1 << SCOUTFS_OPEN_INO_MAP_SHIFT) +#define SCOUTFS_OPEN_INO_MAP_MASK (SCOUTFS_OPEN_INO_MAP_BITS - 1) +#define SCOUTFS_OPEN_INO_MAP_LE64S (SCOUTFS_OPEN_INO_MAP_BITS / 64) + +/* + * The request and response conversation is as follows: + * + * client[init] -> server: + * group_nr = G + * req_id = 0 (I) + * server -> client[*] + * group_nr = G + * req_id = R + * client[*] -> server + * group_nr = G (I) + * req_id = R + * bits + * server -> client[init] + * group_nr = G (I) + * req_id = R (I) + * bits + * + * Many of the fields in individual messages are ignored ("I") because + * the net id or the omap req_id can be used to identify the + * conversation. We always include them on the wire to make inspected + * messages easier to follow. + */ +struct scoutfs_open_ino_map_args { + __le64 group_nr; + __le64 req_id; +}; + +struct scoutfs_open_ino_map { + struct scoutfs_open_ino_map_args args; + __le64 bits[SCOUTFS_OPEN_INO_MAP_LE64S]; +}; + #endif diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 42761a3f..cdade67d 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -33,6 +33,7 @@ #include "item.h" #include "client.h" #include "cmp.h" +#include "omap.h" /* * XXX @@ -82,6 +83,8 @@ static void scoutfs_inode_ctor(void *obj) init_waitqueue_head(&si->data_waitq.waitq); init_rwsem(&si->xattr_rwsem); RB_CLEAR_NODE(&si->writeback_node); + scoutfs_lock_init_coverage(&si->ino_lock_cov); + atomic_set(&si->inv_iput_count, 0); inode_init_once(&si->inode); } @@ -141,12 +144,15 @@ static void remove_writeback_inode(struct inode_sb_info *inf, void scoutfs_destroy_inode(struct inode *inode) { + struct scoutfs_inode_info *si = SCOUTFS_I(inode); DECLARE_INODE_SB_INFO(inode->i_sb, inf); spin_lock(&inf->writeback_lock); remove_writeback_inode(inf, SCOUTFS_I(inode)); spin_unlock(&inf->writeback_lock); + scoutfs_lock_del_coverage(inode->i_sb, &si->ino_lock_cov); + call_rcu(&inode->i_rcu, scoutfs_i_callback); } @@ -307,6 +313,8 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, if (ret == 0) { load_inode(inode, &sinode); atomic64_set(&si->last_refreshed, refresh_gen); + scoutfs_lock_add_coverage(sb, lock, &si->ino_lock_cov); + si->drop_invalidated = false; } } else { ret = 0; @@ -664,6 +672,28 @@ struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino) return ilookup5(sb, ino, scoutfs_iget_test, &ino); } +static int iget_test_nofreeing(struct inode *inode, void *arg) +{ + return !(inode->i_state & I_FREEING) && scoutfs_iget_test(inode, arg); +} + +/* + * There's a natural risk of a deadlock between lock invalidation and + * eviction. Invalidation blocks locks while looking up inodes and + * invalidating local caches. Inode eviction gets a lock to check final + * inode deletion while the inode is marked FREEING which blocks + * lookups. + * + * We have a lookup variant which doesn't return I_FREEING inodes + * instead of waiting on them. If an inode has made it to I_FREEING + * then it doesn't have any local caches that are reachable and the lock + * invalidation promise is kept. + */ +struct inode *scoutfs_ilookup_nofreeing(struct super_block *sb, u64 ino) +{ + return ilookup5(sb, ino, iget_test_nofreeing, &ino); +} + struct inode *scoutfs_iget(struct super_block *sb, u64 ino) { struct scoutfs_lock *lock = NULL; @@ -688,6 +718,8 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino) atomic64_set(&si->last_refreshed, 0); ret = scoutfs_inode_refresh(inode, lock, 0); + if (ret == 0) + ret = scoutfs_omap_inc(sb, ino); if (ret) { iget_failed(inode); inode = ERR_PTR(ret); @@ -1384,6 +1416,8 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, si->next_xattr_id = 0; si->have_item = false; atomic64_set(&si->last_refreshed, lock->refresh_gen); + scoutfs_lock_add_coverage(sb, lock, &si->ino_lock_cov); + si->drop_invalidated = false; si->flags = 0; scoutfs_inode_set_meta_seq(inode); @@ -1399,10 +1433,17 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, store_inode(&sinode, inode); init_inode_key(&key, scoutfs_ino(inode)); + ret = scoutfs_omap_inc(sb, ino); + if (ret < 0) + goto out; + ret = scoutfs_item_create(sb, &key, &sinode, sizeof(sinode), lock); + if (ret < 0) + scoutfs_omap_dec(sb, ino); +out: if (ret) { iput(inode); - return ERR_PTR(ret); + inode = ERR_PTR(ret); } return inode; @@ -1447,15 +1488,15 @@ int scoutfs_orphan_delete(struct super_block *sb, u64 ino) /* * Remove all the items associated with a given inode. This is only - * called once nlink has dropped to zero so we don't have to worry about - * dirents referencing the inode or link backrefs. Dropping nlink to 0 - * also created an orphan item. That orphan item will continue - * triggering attempts to finish previous partial deletion until all - * deletion is complete and the orphan item is removed. + * called once nlink has dropped to zero and nothing has the inode open + * so we don't have to worry about dirents referencing the inode or link + * backrefs. Dropping nlink to 0 also created an orphan item. That + * orphan item will continue triggering attempts to finish previous + * partial deletion until all deletion is complete and the orphan item + * is removed. */ -static int delete_inode_items(struct super_block *sb, u64 ino) +static int delete_inode_items(struct super_block *sb, u64 ino, struct scoutfs_lock *lock) { - struct scoutfs_lock *lock = NULL; struct scoutfs_inode sinode; struct scoutfs_key key; LIST_HEAD(ind_locks); @@ -1465,10 +1506,6 @@ static int delete_inode_items(struct super_block *sb, u64 ino) u64 size; int ret; - ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_WRITE, 0, ino, &lock); - if (ret) - return ret; - init_inode_key(&key, ino); ret = scoutfs_item_lookup_exact(sb, &key, &sinode, sizeof(sinode), @@ -1533,18 +1570,24 @@ out: if (release) scoutfs_release_trans(sb); scoutfs_inode_index_unlock(sb, &ind_locks); - scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + return ret; } /* * iput_final has already written out the dirty pages to the inode * before we get here. We're left with a clean inode that we have to - * tear down. If there are no more links to the inode then we also - * remove all its persistent structures. + * tear down. We use locking and open inode number bitmaps to decide if + * we should finally destroy an inode that is no longer open nor + * reachable through directory entries. */ void scoutfs_evict_inode(struct inode *inode) { + struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); + struct scoutfs_lock *lock; + int ret; + trace_scoutfs_evict_inode(inode->i_sb, scoutfs_ino(inode), inode->i_nlink, is_bad_inode(inode)); @@ -1553,19 +1596,45 @@ void scoutfs_evict_inode(struct inode *inode) truncate_inode_pages_final(&inode->i_data); - if (inode->i_nlink == 0) - delete_inode_items(inode->i_sb, scoutfs_ino(inode)); + ret = scoutfs_omap_should_delete(sb, inode, &lock); + if (ret > 0) { + ret = delete_inode_items(inode->i_sb, scoutfs_ino(inode), lock); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + } + if (ret < 0) + scoutfs_err(sb, "error %d while checking to delete inode nr %llu, it might linger.", + ret, ino); + + scoutfs_omap_dec(sb, ino); + clear: clear_inode(inode); } +/* + * We want to remove inodes from the cache as their count goes to 0 if + * they're no longer covered by a cluster lock or if while locked they + * were unlinked. + * + * We don't want unused cached inodes to linger outside of cluster + * locking so that they don't prevent final inode deletion on other + * nodes. We don't have specific per-inode or per-dentry locks which + * would otherwise remove the stale caches as they're invalidated. + * Stale cached inodes provide little value because they're going to be + * refreshed the next time they're locked. Populating the item cache + * and loading the inode item is a lot more expensive than initializing + * and inserting a newly allocated vfs inode. + */ int scoutfs_drop_inode(struct inode *inode) { - int ret = generic_drop_inode(inode); + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; - trace_scoutfs_drop_inode(inode->i_sb, scoutfs_ino(inode), - inode->i_nlink, inode_unhashed(inode)); - return ret; + trace_scoutfs_drop_inode(sb, scoutfs_ino(inode), inode->i_nlink, inode_unhashed(inode), + si->drop_invalidated); + + return si->drop_invalidated || !scoutfs_lock_is_covered(sb, &si->ino_lock_cov) || + generic_drop_inode(inode); } /* @@ -1582,8 +1651,10 @@ int scoutfs_scan_orphans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock = sbi->rid_lock; + struct scoutfs_lock *inode_lock = NULL; struct scoutfs_key key; struct scoutfs_key last; + u64 ino; int err = 0; int ret; @@ -1599,7 +1670,13 @@ int scoutfs_scan_orphans(struct super_block *sb) if (ret < 0) goto out; - ret = delete_inode_items(sb, le64_to_cpu(key.sko_ino)); + ino = le64_to_cpu(key.sko_ino); + + ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_WRITE, 0, ino, &inode_lock); + if (ret == 0) { + ret = delete_inode_items(sb, le64_to_cpu(key.sko_ino), inode_lock); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE); + } if (ret && ret != -ENOENT && !err) err = ret; diff --git a/kmod/src/inode.h b/kmod/src/inode.h index ad517752..f1884ad6 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -51,6 +51,13 @@ struct scoutfs_inode_info { struct rw_semaphore xattr_rwsem; struct rb_node writeback_node; + struct scoutfs_lock_coverage ino_lock_cov; + + /* drop if i_count hits 0, allows drop while invalidate holds coverage */ + bool drop_invalidated; + struct llist_node inv_iput_llnode; + atomic_t inv_iput_count; + struct inode inode; }; @@ -72,6 +79,7 @@ int scoutfs_orphan_inode(struct inode *inode); struct inode *scoutfs_iget(struct super_block *sb, u64 ino); struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino); +struct inode *scoutfs_ilookup_nofreeing(struct super_block *sb, u64 ino); void scoutfs_inode_init_index_key(struct scoutfs_key *key, u8 type, u64 major, u32 minor, u64 ino); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 10d95507..120bcdf7 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -34,6 +34,7 @@ #include "data.h" #include "xattr.h" #include "item.h" +#include "omap.h" /* * scoutfs uses a lock service to manage item cache consistency between @@ -74,6 +75,7 @@ struct lock_info { struct super_block *sb; spinlock_t lock; bool shutdown; + bool unmounting; struct rb_root lock_tree; struct rb_root lock_range_tree; struct shrinker shrinker; @@ -87,6 +89,9 @@ struct lock_info { struct work_struct shrink_work; struct list_head shrink_list; atomic64_t next_refresh_gen; + struct work_struct inv_iput_work; + struct llist_head inv_iput_llist; + struct dentry *tseq_dentry; struct scoutfs_tseq_tree tseq_tree; }; @@ -122,21 +127,81 @@ static bool lock_modes_match(int granted, int requested) } /* - * invalidate cached data associated with an inode whose lock is going - * away. + * Final iput can get into evict and perform final inode deletion which + * can delete a lot of items under locks and transactions. We really + * don't want to be doing all that in an iput during invalidation. When + * invalidation sees that iput might perform final deletion it puts them + * on a list and queues this work. + * + * Nothing stops multiple puts for multiple invalidations of an inode + * before the work runs so we can track multiple puts in flight. + */ +static void lock_inv_iput_worker(struct work_struct *work) +{ + struct lock_info *linfo = container_of(work, struct lock_info, inv_iput_work); + struct scoutfs_inode_info *si; + struct scoutfs_inode_info *tmp; + struct llist_node *inodes; + bool more; + + inodes = llist_del_all(&linfo->inv_iput_llist); + + llist_for_each_entry_safe(si, tmp, inodes, inv_iput_llnode) { + do { + more = atomic_dec_return(&si->inv_iput_count) > 0; + iput(&si->inode); + } while (more); + } +} + +/* + * Invalidate cached data associated with an inode whose lock is going + * away. We ignore indoes with I_FREEING instead of waiting on them to + * avoid a deadlock, if they're freeing then they won't be visible to + * future lock users and we don't need to invalidate them. + * + * We try to drop cached dentries and inodes covered by the lock if they + * aren't referenced. This removes them from the mount's open map and + * allows deletions to be performed by unlink without having to wait for + * remote cached inodes to be dropped. + * + * If the cached inode was already deferring final inode deletion then + * we can't perform that inline in invalidation. The locking alone + * deadlock, and it might also take multiple transactions to fully + * delete an inode with significant metadata. We only perform the iput + * inline if we know that possible eviction can't perform the final + * deletion, otherwise we kick it off to async work. */ static void invalidate_inode(struct super_block *sb, u64 ino) { + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_inode_info *si; struct inode *inode; - inode = scoutfs_ilookup(sb, ino); + inode = scoutfs_ilookup_nofreeing(sb, ino); if (inode) { + si = SCOUTFS_I(inode); + scoutfs_inc_counter(sb, lock_invalidate_inode); if (S_ISREG(inode->i_mode)) { truncate_inode_pages(inode->i_mapping, 0); scoutfs_data_wait_changed(inode); } - iput(inode); + + /* can't touch during unmount, dcache destroys w/o locks */ + if (!linfo->unmounting) + d_prune_aliases(inode); + + si->drop_invalidated = true; + if (scoutfs_lock_is_covered(sb, &si->ino_lock_cov) && inode->i_nlink > 0) { + iput(inode); + } else { + /* defer iput to work context so we don't evict inodes from invalidation */ + if (atomic_inc_return(&si->inv_iput_count) == 1) + llist_add(&si->inv_iput_llnode, &linfo->inv_iput_llist); + smp_wmb(); /* count and list visible before work executes */ + queue_work(linfo->workq, &linfo->inv_iput_work); + } } } @@ -172,6 +237,16 @@ static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, /* have to invalidate if we're not in the only usable case */ if (!(prev == SCOUTFS_LOCK_WRITE && mode == SCOUTFS_LOCK_READ)) { retry: + /* invalidate inodes before removing coverage */ + if (lock->start.sk_zone == SCOUTFS_FS_ZONE) { + ino = le64_to_cpu(lock->start.ski_ino); + last = le64_to_cpu(lock->end.ski_ino); + while (ino <= last) { + invalidate_inode(sb, ino); + ino++; + } + } + /* remove cov items to tell users that their cache is stale */ spin_lock(&lock->cov_list_lock); list_for_each_entry_safe(cov, tmp, &lock->cov_list, head) { @@ -187,15 +262,6 @@ retry: } spin_unlock(&lock->cov_list_lock); - if (lock->start.sk_zone == SCOUTFS_FS_ZONE) { - ino = le64_to_cpu(lock->start.ski_ino); - last = le64_to_cpu(lock->end.ski_ino); - while (ino <= last) { - invalidate_inode(sb, ino); - ino++; - } - } - scoutfs_item_invalidate(sb, &lock->start, &lock->end); } @@ -229,6 +295,7 @@ static void lock_free(struct lock_info *linfo, struct scoutfs_lock *lock) BUG_ON(!list_empty(&lock->shrink_head)); BUG_ON(!list_empty(&lock->cov_list)); + scoutfs_omap_free_lock_data(lock->omap_data); kfree(lock); } @@ -264,6 +331,7 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb, lock->mode = SCOUTFS_LOCK_NULL; atomic64_set(&lock->forest_bloom_nr, 0); + spin_lock_init(&lock->omap_spinlock); trace_scoutfs_lock_alloc(sb, lock); @@ -553,7 +621,7 @@ static void queue_grant_work(struct lock_info *linfo) { assert_spin_locked(&linfo->lock); - if (!list_empty(&linfo->grant_list) && !linfo->shutdown) + if (!list_empty(&linfo->grant_list)) queue_work(linfo->workq, &linfo->grant_work); } @@ -569,7 +637,7 @@ static void queue_inv_work(struct lock_info *linfo) { assert_spin_locked(&linfo->lock); - if (!list_empty(&linfo->inv_list) && !linfo->shutdown) + if (!list_empty(&linfo->inv_list)) mod_delayed_work(linfo->workq, &linfo->inv_dwork, 0); } @@ -800,8 +868,11 @@ static void lock_invalidate_worker(struct work_struct *work) nl = &lock->inv_nl; net_id = lock->inv_net_id; - ret = lock_invalidate(sb, lock, nl->old_mode, nl->new_mode); - BUG_ON(ret); + /* only lock protocol, inv can't call subsystems after shutdown */ + if (!linfo->shutdown) { + ret = lock_invalidate(sb, lock, nl->old_mode, nl->new_mode); + BUG_ON(ret); + } /* respond with the key and modes from the request */ ret = scoutfs_client_lock_response(sb, net_id, nl); @@ -991,7 +1062,7 @@ static int lock_key_range(struct super_block *sb, enum scoutfs_lock_mode mode, i lock_inc_count(lock->waiters, mode); for (;;) { - if (linfo->shutdown) { + if (WARN_ON_ONCE(linfo->shutdown)) { ret = -ESHUTDOWN; break; } @@ -1473,7 +1544,7 @@ restart: BUG_ON(lock->mode == SCOUTFS_LOCK_NULL); BUG_ON(!list_empty(&lock->shrink_head)); - if (linfo->shutdown || nr-- == 0) + if (nr-- == 0) break; __lock_del_lru(linfo, lock); @@ -1500,7 +1571,7 @@ out: return ret; } -void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr) +void scoutfs_free_unused_locks(struct super_block *sb) { struct lock_info *linfo = SCOUTFS_SB(sb)->lock_info; struct shrink_control sc = { @@ -1528,15 +1599,40 @@ static void lock_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent) } /* - * The caller is going to be calling _destroy soon and, critically, is - * about to shutdown networking before calling us so that we don't get - * any callbacks while we're destroying. We have to ensure that we - * won't call networking after this returns. + * shrink_dcache_for_umount() tears down dentries with no locking. We + * need to make sure that our invalidation won't touch dentries before + * we return and the caller calls the generic vfs unmount path. + */ +void scoutfs_lock_unmount_begin(struct super_block *sb) +{ + DECLARE_LOCK_INFO(sb, linfo); + + if (linfo) { + linfo->unmounting = true; + flush_delayed_work(&linfo->inv_dwork); + } +} + +/* + * The caller is going to be shutting down transactions and the client. + * We need to make sure that locking won't call either after we return. * - * Internal fs threads can be using locking, and locking can have async - * work pending. We use ->shutdown to force callers to return - * -ESHUTDOWN and to prevent the future queueing of work that could call - * networking. Locks whose work is stopped will be torn down by _destroy. + * At this point all fs callers and internal services that use locks + * should have stopped. We won't have any callers initiating lock + * transitions and sending requests. We set the shutdown flag to catch + * anyone who breaks this rule. + * + * We unregister the shrinker so that we won't try and send null + * requests in response to memory pressure. The locks will all be + * unceremoniously dropped once we get a farewell response from the + * server which indicates that they destroyed our locking state. + * + * We will still respond to invalidation requests that have to be + * processed to let unmount in other mounts acquire locks and make + * progress. However, we don't fully process the invalidation because + * we're shutting down. We only update the lock state and send the + * response. We shouldn't have any users of locking that require + * invalidation correctness at this point. */ void scoutfs_lock_shutdown(struct super_block *sb) { @@ -1549,19 +1645,18 @@ void scoutfs_lock_shutdown(struct super_block *sb) trace_scoutfs_lock_shutdown(sb, linfo); - spin_lock(&linfo->lock); + /* stop the shrinker from queueing work */ + unregister_shrinker(&linfo->shrinker); + flush_work(&linfo->shrink_work); + /* cause current and future lock calls to return errors */ + spin_lock(&linfo->lock); linfo->shutdown = true; for (node = rb_first(&linfo->lock_tree); node; node = rb_next(node)) { lock = rb_entry(node, struct scoutfs_lock, node); wake_up(&lock->waitq); } - spin_unlock(&linfo->lock); - - flush_work(&linfo->grant_work); - flush_delayed_work(&linfo->inv_dwork); - flush_work(&linfo->shrink_work); } /* @@ -1589,8 +1684,6 @@ void scoutfs_lock_destroy(struct super_block *sb) trace_scoutfs_lock_destroy(sb, linfo); - /* stop the shrinker from queueing work */ - unregister_shrinker(&linfo->shrinker); /* make sure that no one's actively using locks */ spin_lock(&linfo->lock); @@ -1636,8 +1729,10 @@ void scoutfs_lock_destroy(struct super_block *sb) __lock_del_lru(linfo, lock); if (!list_empty(&lock->grant_head)) list_del_init(&lock->grant_head); - if (!list_empty(&lock->inv_head)) + if (!list_empty(&lock->inv_head)) { list_del_init(&lock->inv_head); + lock->invalidate_pending = 0; + } if (!list_empty(&lock->shrink_head)) list_del_init(&lock->shrink_head); lock_remove(linfo, lock); @@ -1674,6 +1769,8 @@ int scoutfs_lock_setup(struct super_block *sb) INIT_WORK(&linfo->shrink_work, lock_shrink_worker); INIT_LIST_HEAD(&linfo->shrink_list); atomic64_set(&linfo->next_refresh_gen, 0); + INIT_WORK(&linfo->inv_iput_work, lock_inv_iput_worker); + init_llist_head(&linfo->inv_iput_llist); scoutfs_tseq_tree_init(&linfo->tseq_tree, lock_tseq_show); sbi->lock_info = linfo; diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 46dcdf96..40f8f5b9 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -10,6 +10,8 @@ #define SCOUTFS_LOCK_NR_MODES SCOUTFS_LOCK_INVALID +struct scoutfs_omap_lock; + /* * A few fields (start, end, refresh_gen, write_version, granted_mode) * are referenced by code outside lock.c. @@ -47,6 +49,10 @@ struct scoutfs_lock { /* the forest tracks which log tree last saw bloom bit updates */ atomic64_t forest_bloom_nr; + + /* open ino mapping has a valid map for a held write lock */ + spinlock_t omap_spinlock; + struct scoutfs_omap_lock_data *omap_data; }; struct scoutfs_lock_coverage { @@ -95,9 +101,10 @@ void scoutfs_lock_del_coverage(struct super_block *sb, bool scoutfs_lock_protected(struct scoutfs_lock *lock, struct scoutfs_key *key, enum scoutfs_lock_mode mode); -void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr); +void scoutfs_free_unused_locks(struct super_block *sb); int scoutfs_lock_setup(struct super_block *sb); +void scoutfs_lock_unmount_begin(struct super_block *sb); void scoutfs_lock_shutdown(struct super_block *sb); void scoutfs_lock_destroy(struct super_block *sb); diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index a2d96dbf..fb33fd80 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -20,10 +20,10 @@ #include "tseq.h" #include "spbm.h" #include "block.h" -#include "btree.h" #include "msg.h" #include "scoutfs_trace.h" #include "lock_server.h" +#include "recov.h" /* * The scoutfs server implements a simple lock service. Client mounts @@ -56,14 +56,11 @@ * Message requests and responses are reliably delivered in order across * reconnection. * - * The server maintains a persistent record of connected clients. A new - * server instance discovers these and waits for previously connected - * clients to reconnect and recover their state before proceeding. If - * clients don't reconnect they are forcefully prevented from unsafely - * accessing the shared persistent storage. (fenced, according to the - * rules of the platform.. could range from being powered off to having - * their switch port disabled to having their local block device set - * read-only.) + * As a new server comes up it recovers lock state from existing clients + * which were connected to a previous lock server. Recover requests are + * sent to clients as they connect and they respond with all there + * locks. Once all clients and locks are accounted for normal + * processing can resume. * * The lock server doesn't respond to memory pressure. The only way * locks are freed is if they are invalidated to null on behalf of a @@ -77,12 +74,8 @@ struct lock_server_info { struct super_block *sb; spinlock_t lock; - struct mutex mutex; struct rb_root locks_root; - struct scoutfs_spbm recovery_pending; - struct delayed_work recovery_dwork; - struct scoutfs_tseq_tree tseq_tree; struct dentry *tseq_dentry; @@ -430,7 +423,7 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 rid, goto out; } - /* XXX should always have a server lock here? recovery? */ + /* XXX should always have a server lock here? */ snode = get_server_lock(inf, &nl->key, NULL, false); if (!snode) { ret = -EINVAL; @@ -473,12 +466,9 @@ out: * so we unlock the snode mutex. * * All progress must wait for all clients to finish with recovery - * because we don't know which locks they'll hold. The unlocked - * recovery_pending test here is OK. It's filled by setup before - * anything runs. It's emptied by recovery completion. We can get a - * false nonempty result if we race with recovery completion, but that's - * OK because recovery completion processes all the locks that have - * requests after emptying, including the unlikely loser of that race. + * because we don't know which locks they'll hold. Once recover + * finishes the server calls us to kick all the locks that were waiting + * during recovery. */ static int process_waiting_requests(struct super_block *sb, struct server_lock_node *snode) @@ -496,7 +486,7 @@ static int process_waiting_requests(struct super_block *sb, /* processing waits for all invalidation responses or recovery */ if (!list_empty(&snode->invalidated) || - !scoutfs_spbm_empty(&inf->recovery_pending)) { + scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_LOCKS) != 0) { ret = 0; goto out; } @@ -569,89 +559,39 @@ out: return ret; } -static void init_lock_clients_key(struct scoutfs_key *key, u64 rid) -{ - *key = (struct scoutfs_key) { - .sk_zone = SCOUTFS_LOCK_CLIENTS_ZONE, - .sklc_rid = cpu_to_le64(rid), - }; -} - /* * The server received a greeting from a client for the first time. If - * the client had already talked to the server then we must find an - * existing record for it and should begin recovery. If it doesn't have - * a record then its timed out and we can't allow it to reconnect. If - * we're creating a new record for a client we can see EEXIST if the - * greeting is resent to a new server after the record was committed but - * before the response was received by the client. + * the client is in lock recovery then we send the initial lock request. * * This is running in concurrent client greeting processing contexts. */ -int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid, - bool should_exist) +int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid) { - DECLARE_LOCK_SERVER_INFO(sb, inf); - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_key key; int ret; - init_lock_clients_key(&key, rid); - - mutex_lock(&inf->mutex); - if (should_exist) { - ret = scoutfs_btree_lookup(sb, &super->lock_clients, &key, - &iref); - if (ret == 0) - scoutfs_btree_put_iref(&iref); - } else { - ret = scoutfs_btree_insert(sb, inf->alloc, inf->wri, - &super->lock_clients, - &key, NULL, 0); - if (ret == -EEXIST) - ret = 0; - } - mutex_unlock(&inf->mutex); - - if (should_exist && ret == 0) { + if (scoutfs_recov_is_pending(sb, rid, SCOUTFS_RECOV_LOCKS)) { scoutfs_key_set_zeros(&key); ret = scoutfs_server_lock_recover_request(sb, rid, &key); - if (ret) - goto out; + } else { + ret = 0; } -out: return ret; } /* - * A client sent their last recovery response and can exit recovery. If - * they were the last client in recovery then we can process all the - * server locks that had requests. + * All clients have finished lock recovery, we can make forward process + * on all the queued requests that were waiting on recovery. */ -static int finished_recovery(struct super_block *sb, u64 rid, bool cancel) +int scoutfs_lock_server_finished_recovery(struct super_block *sb) { DECLARE_LOCK_SERVER_INFO(sb, inf); struct server_lock_node *snode; struct scoutfs_key key; - bool still_pending; int ret = 0; - spin_lock(&inf->lock); - scoutfs_spbm_clear(&inf->recovery_pending, rid); - still_pending = !scoutfs_spbm_empty(&inf->recovery_pending); - spin_unlock(&inf->lock); - if (still_pending) - return 0; - - if (cancel) - cancel_delayed_work_sync(&inf->recovery_dwork); - scoutfs_key_set_zeros(&key); - - scoutfs_info(sb, "all lock clients recovered"); - while ((snode = get_server_lock(inf, &key, NULL, true))) { key = snode->key; @@ -695,16 +635,15 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid, int i; /* client must be in recovery */ - spin_lock(&inf->lock); - if (!scoutfs_spbm_test(&inf->recovery_pending, rid)) + if (!scoutfs_recov_is_pending(sb, rid, SCOUTFS_RECOV_LOCKS)) { ret = -EINVAL; - spin_unlock(&inf->lock); - if (ret) goto out; + } /* client has sent us all their locks */ if (nlr->nr == 0) { - ret = finished_recovery(sb, rid, true); + scoutfs_server_recov_finish(sb, rid, SCOUTFS_RECOV_LOCKS); + ret = 0; goto out; } @@ -755,101 +694,15 @@ out: return ret; } -static int get_rid_and_put_ref(struct scoutfs_btree_item_ref *iref, u64 *rid) -{ - int ret; - - if (iref->val_len == 0) { - *rid = le64_to_cpu(iref->key->sklc_rid); - ret = 0; - } else { - ret = -EIO; - } - scoutfs_btree_put_iref(iref); - return ret; -} - -/* - * This work executes if enough time passes without all of the clients - * finishing with recovery and canceling the work. We walk through the - * client records and find any that still have their recovery pending. - */ -static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) -{ - struct lock_server_info *inf = container_of(work, - struct lock_server_info, - recovery_dwork.work); - struct super_block *sb = inf->sb; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_key key; - bool timed_out; - u64 rid; - int ret; - - ret = scoutfs_server_hold_commit(sb); - if (ret) - goto out; - - /* we enter recovery if there are any client records */ - for (rid = 0; ; rid++) { - init_lock_clients_key(&key, rid); - ret = scoutfs_btree_next(sb, &super->lock_clients, &key, &iref); - if (ret == -ENOENT) { - ret = 0; - break; - } - if (ret == 0) - ret = get_rid_and_put_ref(&iref, &rid); - if (ret < 0) - break; - - spin_lock(&inf->lock); - if (scoutfs_spbm_test(&inf->recovery_pending, rid)) { - scoutfs_spbm_clear(&inf->recovery_pending, rid); - timed_out = true; - } else { - timed_out = false; - } - spin_unlock(&inf->lock); - - if (!timed_out) - continue; - - scoutfs_err(sb, "client rid %016llx lock recovery timed out", - rid); - - init_lock_clients_key(&key, rid); - ret = scoutfs_btree_delete(sb, inf->alloc, inf->wri, - &super->lock_clients, &key); - if (ret) - break; - } - - ret = scoutfs_server_apply_commit(sb, ret); -out: - /* force processing all pending lock requests */ - if (ret == 0) - ret = finished_recovery(sb, 0, false); - - if (ret < 0) { - scoutfs_err(sb, "lock server saw err %d while timing out clients, shutting down", ret); - scoutfs_server_abort(sb); - } -} - /* * A client is leaving the lock service. They aren't using locks and * won't send any more requests. We tear down all the state we had for * them. This can be called multiple times for a given client as their * farewell is resent to new servers. It's OK to not find any state. - * If we fail to delete a persistent entry then we have to shut down and - * hope that the next server has more luck. */ int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid) { DECLARE_LOCK_SERVER_INFO(sb, inf); - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct client_lock_entry *clent; struct client_lock_entry *tmp; struct server_lock_node *snode; @@ -858,20 +711,7 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid) bool freed; int ret = 0; - mutex_lock(&inf->mutex); - init_lock_clients_key(&key, rid); - ret = scoutfs_btree_delete(sb, inf->alloc, inf->wri, - &super->lock_clients, &key); - mutex_unlock(&inf->mutex); - if (ret == -ENOENT) { - ret = 0; - goto out; - } - if (ret < 0) - goto out; - scoutfs_key_set_zeros(&key); - while ((snode = get_server_lock(inf, &key, NULL, true))) { freed = false; @@ -956,23 +796,14 @@ static void lock_server_tseq_show(struct seq_file *m, /* * Setup the lock server. This is called before networking can deliver - * requests. If we find existing client records then we enter recovery. - * Lock request processing is deferred until recovery is resolved for - * all the existing clients, either they reconnect and replay locks or - * we time them out. + * requests. */ int scoutfs_lock_server_setup(struct super_block *sb, struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, u64 max_vers) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct lock_server_info *inf; - SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_key key; - unsigned int nr; - u64 rid; - int ret; inf = kzalloc(sizeof(struct lock_server_info), GFP_KERNEL); if (!inf) @@ -980,11 +811,7 @@ int scoutfs_lock_server_setup(struct super_block *sb, inf->sb = sb; spin_lock_init(&inf->lock); - mutex_init(&inf->mutex); inf->locks_root = RB_ROOT; - scoutfs_spbm_init(&inf->recovery_pending); - INIT_DELAYED_WORK(&inf->recovery_dwork, - scoutfs_lock_server_recovery_timeout); scoutfs_tseq_tree_init(&inf->tseq_tree, lock_server_tseq_show); inf->alloc = alloc; inf->wri = wri; @@ -999,36 +826,7 @@ int scoutfs_lock_server_setup(struct super_block *sb, sbi->lock_server_info = inf; - /* we enter recovery if there are any client records */ - nr = 0; - for (rid = 0; ; rid++) { - init_lock_clients_key(&key, rid); - ret = scoutfs_btree_next(sb, &super->lock_clients, &key, &iref); - if (ret == -ENOENT) - break; - if (ret == 0) - ret = get_rid_and_put_ref(&iref, &rid); - if (ret < 0) - goto out; - - ret = scoutfs_spbm_set(&inf->recovery_pending, rid); - if (ret) - goto out; - nr++; - - if (rid == U64_MAX) - break; - } - ret = 0; - - if (nr) { - schedule_delayed_work(&inf->recovery_dwork, - msecs_to_jiffies(LOCK_SERVER_RECOVERY_MS)); - scoutfs_info(sb, "waiting for %u lock clients to recover", nr); - } - -out: - return ret; + return 0; } /* @@ -1046,8 +844,6 @@ void scoutfs_lock_server_destroy(struct super_block *sb) LIST_HEAD(list); if (inf) { - cancel_delayed_work_sync(&inf->recovery_dwork); - debugfs_remove(inf->tseq_dentry); rbtree_postorder_for_each_entry_safe(snode, stmp, @@ -1066,8 +862,6 @@ void scoutfs_lock_server_destroy(struct super_block *sb) kfree(snode); } - scoutfs_spbm_destroy(&inf->recovery_pending); - kfree(inf); sbi->lock_server_info = NULL; } diff --git a/kmod/src/lock_server.h b/kmod/src/lock_server.h index 357fd5af..e77f116f 100644 --- a/kmod/src/lock_server.h +++ b/kmod/src/lock_server.h @@ -3,10 +3,10 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid, struct scoutfs_net_lock_recover *nlr); +int scoutfs_lock_server_finished_recovery(struct super_block *sb); int scoutfs_lock_server_request(struct super_block *sb, u64 rid, u64 net_id, struct scoutfs_net_lock *nl); -int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid, - bool should_exist); +int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid); int scoutfs_lock_server_response(struct super_block *sb, u64 rid, struct scoutfs_net_lock *nl); int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid); diff --git a/kmod/src/omap.c b/kmod/src/omap.c new file mode 100644 index 00000000..bb3ac8c4 --- /dev/null +++ b/kmod/src/omap.c @@ -0,0 +1,1046 @@ +/* + * Copyright (C) 2021 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "counters.h" +#include "cmp.h" +#include "inode.h" +#include "client.h" +#include "server.h" +#include "omap.h" +#include "recov.h" +#include "scoutfs_trace.h" + +/* + * As a client removes an inode from its cache with an nlink of 0 it + * needs to decide if it is the last client using the inode and should + * fully delete all its items. It needs to know if other mounts still + * have the inode in use. + * + * We need a way to communicate between mounts that an inode is open. + * We don't want to pay the synchronous per-file locking round trip + * costs associated with per-inode open locks that you'd typically see + * in systems to solve this problem. + * + * Instead clients maintain open bitmaps that cover groups of inodes. + * As inodes enter the cache their bit is set, and as the inode is + * evicted the bit is cleared. As an inode is evicted messages are sent + * around the cluster to get the current bitmaps for that inode's group + * from all active mounts. If the inode's bit is clear then it can be + * deleted. + * + * We associate the open bitmaps with our cluster locking of inode + * groups to cache these open bitmaps. As long as we have the lock then + * nlink can't be changed on any remote mounts. Specifically, it can't + * increase from 0 so any clear bits can gain references on remote + * mounts. As long as we have the lock, all clear bits in the group for + * inodes with 0 nlink can be deleted. + * + * This layer maintains a list of client rids to send messages to. The + * server calls us as clients enter and leave the cluster. We can't + * process requests until all clients are present as a server starts up + * so we hook into recovery and delay processing until all previously + * existing clients are recovered or fenced. + */ + +struct omap_rid_list { + int nr_rids; + struct list_head head; +}; + +struct omap_rid_entry { + struct list_head head; + u64 rid; +}; + +struct omap_info { + /* client */ + struct rhashtable group_ht; + + /* server */ + struct rhashtable req_ht; + struct llist_head requests; + spinlock_t lock; + struct omap_rid_list rids; + atomic64_t next_req_id; +}; + +#define DECLARE_OMAP_INFO(sb, name) \ + struct omap_info *name = SCOUTFS_SB(sb)->omap_info + +/* + * The presence of an inode in the inode cache increases the count of + * its inode number's position within its lock group. These structs + * track the counts for all the inodes in a lock group and maintain a + * bitmap whose bits are set for each non-zero count. + * + * We don't want to add additional global synchronization of inode cache + * maintenance so these are tracked in an rcu hash table. Once their + * total count reaches zero they're removed from the hash and queued for + * freeing and readers should ignore them. + */ +struct omap_group { + struct super_block *sb; + struct rhash_head ht_head; + struct rcu_head rcu; + u64 nr; + spinlock_t lock; + unsigned int total; + unsigned int *counts; + __le64 bits[SCOUTFS_OPEN_INO_MAP_LE64S]; +}; + +#define trace_group(sb, which, group, bit_nr) \ +do { \ + __typeof__(group) _grp = (group); \ + __typeof__(bit_nr) _nr = (bit_nr); \ + \ + trace_scoutfs_omap_group_##which(sb, _grp, _grp->nr, _grp->total, _nr, \ + _nr < 0 ? -1 : _grp->counts[_nr]); \ +} while (0) + +/* + * Each request is initialized with the rids of currently mounted + * clients. As each responds we remove their rid and send the response + * once everyone has contributed. + * + * The request frequency will typically be low, but in a mass rm -rf + * load we will see O(groups * clients) messages flying around. + */ +struct omap_request { + struct llist_node llnode; + struct rhash_head ht_head; + struct rcu_head rcu; + spinlock_t lock; + u64 client_rid; + u64 client_id; + struct omap_rid_list rids; + struct scoutfs_open_ino_map map; +}; + +/* + * In each inode group cluster lock we store data to track the open ino + * map which tracks all the inodes that the cluster lock covers. When + * the version shows that the map is stale we send a request to update + * it. + */ +struct scoutfs_omap_lock_data { + u64 version; + bool req_in_flight; + wait_queue_head_t waitq; + struct scoutfs_open_ino_map map; +}; + +static inline void init_rid_list(struct omap_rid_list *list) +{ + INIT_LIST_HEAD(&list->head); + list->nr_rids = 0; +} + +/* + * Negative searches almost never happen. + */ +static struct omap_rid_entry *find_rid(struct omap_rid_list *list, u64 rid) +{ + struct omap_rid_entry *entry; + + list_for_each_entry(entry, &list->head, head) { + if (rid == entry->rid) + return entry; + } + + return NULL; +} + +static int free_rid(struct omap_rid_list *list, struct omap_rid_entry *entry) +{ + int nr; + + list_del(&entry->head); + nr = --list->nr_rids; + + kfree(entry); + return nr; +} + +static int copy_rids(struct omap_rid_list *to, struct omap_rid_list *from, spinlock_t *from_lock) +{ + struct omap_rid_entry *entry; + struct omap_rid_entry *src; + struct omap_rid_entry *dst; + int nr; + + spin_lock(from_lock); + + while (to->nr_rids != from->nr_rids) { + nr = from->nr_rids; + spin_unlock(from_lock); + + while (to->nr_rids < nr) { + entry = kmalloc(sizeof(struct omap_rid_entry), GFP_NOFS); + if (!entry) + return -ENOMEM; + + list_add_tail(&entry->head, &to->head); + to->nr_rids++; + } + + while (to->nr_rids > nr) { + entry = list_first_entry(&to->head, struct omap_rid_entry, head); + list_del(&entry->head); + kfree(entry); + to->nr_rids--; + } + + spin_lock(from_lock); + } + + dst = list_first_entry(&to->head, struct omap_rid_entry, head); + list_for_each_entry(src, &from->head, head) { + dst->rid = src->rid; + dst = list_next_entry(dst, head); + } + + spin_unlock(from_lock); + + return 0; +} + +static void free_rids(struct omap_rid_list *list) +{ + struct omap_rid_entry *entry; + struct omap_rid_entry *tmp; + + list_for_each_entry_safe(entry, tmp, &list->head, head) { + list_del(&entry->head); + kfree(entry); + } +} + +static void calc_group_nrs(u64 ino, u64 *group_nr, int *bit_nr) +{ + *group_nr = ino >> SCOUTFS_OPEN_INO_MAP_SHIFT; + *bit_nr = ino & SCOUTFS_OPEN_INO_MAP_MASK; +} + +static struct omap_group *alloc_group(struct super_block *sb, u64 group_nr) +{ + struct omap_group *group; + + BUILD_BUG_ON((sizeof(group->counts[0]) * SCOUTFS_OPEN_INO_MAP_BITS) > PAGE_SIZE); + + group = kzalloc(sizeof(struct omap_group), GFP_NOFS); + if (group) { + group->sb = sb; + group->nr = group_nr; + spin_lock_init(&group->lock); + + group->counts = (void *)get_zeroed_page(GFP_NOFS); + if (!group->counts) { + kfree(group); + group = NULL; + } else { + trace_group(sb, alloc, group, -1); + } + } + + return group; +} + +static void free_group(struct super_block *sb, struct omap_group *group) +{ + trace_group(sb, free, group, -1); + free_page((unsigned long)group->counts); + kfree(group); +} + +static void free_group_rcu(struct rcu_head *rcu) +{ + struct omap_group *group = container_of(rcu, struct omap_group, rcu); + + free_group(group->sb, group); +} + +static const struct rhashtable_params group_ht_params = { + .key_len = member_sizeof(struct omap_group, nr), + .key_offset = offsetof(struct omap_group, nr), + .head_offset = offsetof(struct omap_group, ht_head), +}; + +/* + * Track an cached inode in its group. Our increment can be racing with + * a final decrement that removes the group from the hash, sets total to + * UINT_MAX, and calls rcu free. We can retry until the dead group is + * no longer visible in the hash table and we can insert a new allocated + * group. + */ +int scoutfs_omap_inc(struct super_block *sb, u64 ino) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct omap_group *group; + u64 group_nr; + int bit_nr; + bool found; + int ret = 0; + + calc_group_nrs(ino, &group_nr, &bit_nr); + +retry: + found = false; + rcu_read_lock(); + group = rhashtable_lookup(&ominf->group_ht, &group_nr, group_ht_params); + if (group) { + spin_lock(&group->lock); + if (group->total < UINT_MAX) { + found = true; + if (group->counts[bit_nr]++ == 0) { + set_bit_le(bit_nr, group->bits); + group->total++; + } + } + trace_group(sb, inc, group, bit_nr); + spin_unlock(&group->lock); + } + rcu_read_unlock(); + + if (!found) { + group = alloc_group(sb, group_nr); + if (group) { + ret = rhashtable_lookup_insert_fast(&ominf->group_ht, &group->ht_head, + group_ht_params); + if (ret < 0) + free_group(sb, group); + if (ret == -EEXIST) + ret = 0; + if (ret == -EBUSY) { + /* wait for rehash to finish */ + synchronize_rcu(); + ret = 0; + } + if (ret == 0) + goto retry; + } else { + ret = -ENOMEM; + } + } + + return ret; +} + +/* + * Decrement a previously incremented ino count. Not finding a count + * implies imbalanced inc/dec or bugs freeing groups. We only free + * groups here as the last dec drops the group's total count to 0. + */ +void scoutfs_omap_dec(struct super_block *sb, u64 ino) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct omap_group *group; + u64 group_nr; + int bit_nr; + + calc_group_nrs(ino, &group_nr, &bit_nr); + + rcu_read_lock(); + group = rhashtable_lookup(&ominf->group_ht, &group_nr, group_ht_params); + if (group) { + spin_lock(&group->lock); + WARN_ON_ONCE(group->counts[bit_nr] == 0); + WARN_ON_ONCE(group->total == 0); + WARN_ON_ONCE(group->total == UINT_MAX); + if (--group->counts[bit_nr] == 0) { + clear_bit_le(bit_nr, group->bits); + if (--group->total == 0) { + group->total = UINT_MAX; + rhashtable_remove_fast(&ominf->group_ht, &group->ht_head, + group_ht_params); + call_rcu(&group->rcu, free_group_rcu); + } + } + trace_group(sb, dec, group, bit_nr); + spin_unlock(&group->lock); + } + rcu_read_unlock(); + + WARN_ON_ONCE(!group); +} + +/* + * The server adds rids as it discovers clients. We add them to the + * list of rids to send map requests to. + */ +int scoutfs_omap_add_rid(struct super_block *sb, u64 rid) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct omap_rid_entry *entry; + struct omap_rid_entry *found; + + entry = kmalloc(sizeof(struct omap_rid_entry), GFP_NOFS); + if (!entry) + return -ENOMEM; + + spin_lock(&ominf->lock); + found = find_rid(&ominf->rids, rid); + if (!found) { + entry->rid = rid; + list_add_tail(&entry->head, &ominf->rids.head); + ominf->rids.nr_rids++; + } + spin_unlock(&ominf->lock); + + if (found) + kfree(entry); + + return 0; +} + +static void free_req(struct omap_request *req) +{ + free_rids(&req->rids); + kfree(req); +} + +static void free_req_rcu(struct rcu_head *rcu) +{ + struct omap_request *req = container_of(rcu, struct omap_request, rcu); + + free_req(req); +} + +static const struct rhashtable_params req_ht_params = { + .key_len = member_sizeof(struct omap_request, map.args.req_id), + .key_offset = offsetof(struct omap_request, map.args.req_id), + .head_offset = offsetof(struct omap_request, ht_head), +}; + +/* + * Remove a rid from all the pending requests. If it's the last rid we + * give the caller the details to send a response, they'll call back to + * keep removing. If their send fails they're going to shutdown the + * server so we can queue freeing the request as we give it to them. + */ +static int remove_rid_from_reqs(struct omap_info *ominf, u64 rid, u64 *resp_rid, u64 *resp_id, + struct scoutfs_open_ino_map *map) +{ + struct omap_rid_entry *entry; + struct rhashtable_iter iter; + struct omap_request *req; + int ret = 0; + + rhashtable_walk_enter(&ominf->req_ht, &iter); + rhashtable_walk_start(&iter); + + for (;;) { + req = rhashtable_walk_next(&iter); + if (req == NULL) + break; + if (req == ERR_PTR(-EAGAIN)) + continue; + + spin_lock(&req->lock); + entry = find_rid(&req->rids, rid); + if (entry && free_rid(&req->rids, entry) == 0) { + *resp_rid = req->client_rid; + *resp_id = req->client_id; + memcpy(map, &req->map, sizeof(struct scoutfs_open_ino_map)); + rhashtable_remove_fast(&ominf->req_ht, &req->ht_head, req_ht_params); + call_rcu(&req->rcu, free_req_rcu); + ret = 1; + } + spin_unlock(&req->lock); + if (ret > 0) + break; + } + + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); + + if (ret <= 0) { + *resp_rid = 0; + *resp_id = 0; + } + + return ret; +} + +/* + * A client has been evicted. Remove its rid from the list and walk + * through all the pending requests and remove its rids, sending the + * response if it was the last rid waiting for a response. + * + * If this returns an error then the server will shut down. + */ +int scoutfs_omap_remove_rid(struct super_block *sb, u64 rid) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct scoutfs_open_ino_map *map = NULL; + struct omap_rid_entry *entry; + u64 resp_rid = 0; + u64 resp_id = 0; + int ret; + + map = kmalloc(sizeof(struct scoutfs_open_ino_map), GFP_NOFS); + if (!map) { + ret = -ENOMEM; + goto out; + } + + spin_lock(&ominf->lock); + entry = find_rid(&ominf->rids, rid); + if (entry) + free_rid(&ominf->rids, entry); + spin_unlock(&ominf->lock); + + /* the server really shouldn't be removing a rid it never added */ + if (WARN_ON_ONCE(!entry)) { + ret = -ENOENT; + goto out; + } + + /* remove the rid from all pending requests, sending responses if it was final */ + for (;;) { + ret = remove_rid_from_reqs(ominf, rid, &resp_rid, &resp_id, map); + if (ret <= 0) + break; + ret = scoutfs_server_send_omap_response(sb, resp_rid, resp_id, map, 0); + if (ret < 0) + break; + } + +out: + kfree(map); + return ret; +} + +/* + * Handle a single incoming request in the server. This could have been + * delayed by recovery. This only returns an error if we couldn't send + * a processing error response to the client. + */ +static int handle_request(struct super_block *sb, struct omap_request *req) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct omap_rid_list priv_rids; + struct omap_rid_entry *entry; + int ret; + + init_rid_list(&priv_rids); + + ret = copy_rids(&priv_rids, &ominf->rids, &ominf->lock); + if (ret < 0) + goto out; + + /* don't send a request to the client who originated this request */ + entry = find_rid(&priv_rids, req->client_rid); + if (entry && free_rid(&priv_rids, entry) == 0) { + ret = scoutfs_server_send_omap_response(sb, req->client_rid, req->client_id, + &req->map, 0); + kfree(req); + req = NULL; + goto out; + } + + /* this lock isn't needed but sparse gave warnings with conditional locking */ + ret = copy_rids(&req->rids, &priv_rids, &ominf->lock); + if (ret < 0) + goto out; + + do { + ret = rhashtable_insert_fast(&ominf->req_ht, &req->ht_head, req_ht_params); + if (ret == -EBUSY) + synchronize_rcu(); /* wait for rehash to finish */ + } while (ret == -EBUSY); + + if (ret < 0) + goto out; + + /* + * We can start getting responses the moment we send the first response. After + * we send the last request the req can be freed. + */ + while ((entry = list_first_entry_or_null(&priv_rids.head, struct omap_rid_entry, head))) { + ret = scoutfs_server_send_omap_request(sb, entry->rid, &req->map.args); + if (ret < 0) { + rhashtable_remove_fast(&ominf->req_ht, &req->ht_head, req_ht_params); + goto out; + } + + free_rid(&priv_rids, entry); + } + + ret = 0; +out: + free_rids(&priv_rids); + if (ret < 0) { + ret = scoutfs_server_send_omap_response(sb, req->client_rid, req->client_id, + NULL, ret); + free_req(req); + } + + /* it's fine if we couldn't send to a client that left */ + if (ret == -ENOTCONN) + ret = 0; + + return ret; +} + +/* + * Handle all previously received omap requests from clients. Once + * we've finished recovery and can send requests to all clients we can + * handle all pending requests. The handling function frees the request + * and only returns an error if it couldn't send a response to the + * client. + */ +static int handle_requests(struct super_block *sb) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct llist_node *requests; + struct omap_request *req; + struct omap_request *tmp; + int ret; + int err; + + if (scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_GREETING)) + return 0; + + ret = 0; + requests = llist_del_all(&ominf->requests); + + llist_for_each_entry_safe(req, tmp, requests, llnode) { + err = handle_request(sb, req); + if (err < 0 && ret == 0) + ret = err; + } + + return ret; +} + +int scoutfs_omap_finished_recovery(struct super_block *sb) +{ + return handle_requests(sb); +} + +/* + * The server is receiving a request from a client for the bitmap of all + * open inodes around their ino. Queue it for processing which is + * typically immediate and inline but which can be deferred by recovery + * as the server first starts up. + */ +int scoutfs_omap_server_handle_request(struct super_block *sb, u64 rid, u64 id, + struct scoutfs_open_ino_map_args *args) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct omap_request *req; + + req = kzalloc(sizeof(struct omap_request), GFP_NOFS); + if (req == NULL) + return -ENOMEM; + + spin_lock_init(&req->lock); + req->client_rid = rid; + req->client_id = id; + init_rid_list(&req->rids); + req->map.args.group_nr = args->group_nr; + req->map.args.req_id = cpu_to_le64(atomic64_inc_return(&ominf->next_req_id)); + + llist_add(&req->llnode, &ominf->requests); + + return handle_requests(sb); +} + +/* + * The client is receiving a request from the server for its map for the + * given group. Look up the group and copy the bits to the map for + * non-zero open counts. + * + * The mount originating the request for this bitmap has the inode group + * write locked. We can't be adding links to any inodes in the group + * because that requires the lock. Inodes bits can be set and cleared + * while we're sampling the bitmap. These races are fine, they can't be + * adding cached inodes if nlink is 0 and we don't have the lock. If + * the caller is removing a set bit then they're about to try and delete + * the inode themselves and will first have to acquire the cluster lock + * themselves. + */ +int scoutfs_omap_client_handle_request(struct super_block *sb, u64 id, + struct scoutfs_open_ino_map_args *args) +{ + DECLARE_OMAP_INFO(sb, ominf); + u64 group_nr = le64_to_cpu(args->group_nr); + struct scoutfs_open_ino_map *map; + struct omap_group *group; + bool copied = false; + int ret; + + map = kmalloc(sizeof(struct scoutfs_open_ino_map), GFP_NOFS); + if (!map) + return -ENOMEM; + + map->args = *args; + + rcu_read_lock(); + group = rhashtable_lookup(&ominf->group_ht, &group_nr, group_ht_params); + if (group) { + spin_lock(&group->lock); + trace_group(sb, request, group, -1); + if (group->total > 0 && group->total < UINT_MAX) { + memcpy(map->bits, group->bits, sizeof(map->bits)); + copied = true; + } + spin_unlock(&group->lock); + } + rcu_read_unlock(); + + if (!copied) + memset(map->bits, 0, sizeof(map->bits)); + + ret = scoutfs_client_send_omap_response(sb, id, map); + kfree(map); + return ret; +} + +/* + * The server has received an open ino map response from a client. Find + * the original request that it's serving, or in the response's map, and + * send a reply if this was the last response from a client we were + * waiting for. + * + * We can get responses for requests we're no longer tracking if, for + * example, sending to a client gets an error. We'll have already sent + * the response to the requesting client so we drop these responses on + * the floor. + */ +int scoutfs_omap_server_handle_response(struct super_block *sb, u64 rid, + struct scoutfs_open_ino_map *resp_map) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct scoutfs_open_ino_map *map; + struct omap_rid_entry *entry; + bool send_response = false; + struct omap_request *req; + u64 resp_rid; + u64 resp_id; + int ret; + + map = kmalloc(sizeof(struct scoutfs_open_ino_map), GFP_NOFS); + if (!map) { + ret = -ENOMEM; + goto out; + } + + rcu_read_lock(); + req = rhashtable_lookup(&ominf->req_ht, &resp_map->args.req_id, req_ht_params); + if (req) { + spin_lock(&req->lock); + entry = find_rid(&req->rids, rid); + if (entry) { + bitmap_or((unsigned long *)req->map.bits, (unsigned long *)req->map.bits, + (unsigned long *)resp_map->bits, SCOUTFS_OPEN_INO_MAP_BITS); + if (free_rid(&req->rids, entry) == 0) + send_response = true; + } + spin_unlock(&req->lock); + + if (send_response) { + resp_rid = req->client_rid; + resp_id = req->client_id; + memcpy(map, &req->map, sizeof(struct scoutfs_open_ino_map)); + rhashtable_remove_fast(&ominf->req_ht, &req->ht_head, req_ht_params); + call_rcu(&req->rcu, free_req_rcu); + } + } + rcu_read_unlock(); + + if (send_response) + ret = scoutfs_server_send_omap_response(sb, resp_rid, resp_id, map, 0); + else + ret = 0; + kfree(map); +out: + return ret; +} + +/* + * The server is shutting down. Free all the server state associated + * with ongoing request processing. Clients who still have requests + * pending will resend them to the next server. + */ +void scoutfs_omap_server_shutdown(struct super_block *sb) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct rhashtable_iter iter; + struct llist_node *requests; + struct omap_request *req; + struct omap_request *tmp; + + rhashtable_walk_enter(&ominf->req_ht, &iter); + rhashtable_walk_start(&iter); + + for (;;) { + req = rhashtable_walk_next(&iter); + if (req == NULL) + break; + if (req == ERR_PTR(-EAGAIN)) + continue; + + if (req->rids.nr_rids != 0) { + free_rids(&req->rids); + rhashtable_remove_fast(&ominf->req_ht, &req->ht_head, req_ht_params); + call_rcu(&req->rcu, free_req_rcu); + } + } + + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); + + requests = llist_del_all(&ominf->requests); + llist_for_each_entry_safe(req, tmp, requests, llnode) + kfree(req); + + synchronize_rcu(); +} + +static bool omap_req_in_flight(struct scoutfs_lock *lock, struct scoutfs_omap_lock_data *ldata) +{ + bool in_flight; + + spin_lock(&lock->omap_spinlock); + in_flight = ldata->req_in_flight; + spin_unlock(&lock->omap_spinlock); + + return in_flight; +} + +/* + * Make sure the map covered by the cluster lock is current. The caller + * holds the cluster lock so once we store lock_data on the cluster lock + * it won't be freed and the write_version in the cluster lock won't + * change. + * + * The omap_spinlock protects the omap_data in the cluster lock. We + * have to drop it if we have to block to allocate lock_data, send a + * request for a new map, or wait for a request in flight to finish. + */ +static int get_current_lock_data(struct super_block *sb, struct scoutfs_lock *lock, + struct scoutfs_omap_lock_data **ldata_ret, u64 group_nr) +{ + struct scoutfs_omap_lock_data *ldata; + bool send_req; + int ret = 0; + + spin_lock(&lock->omap_spinlock); + + ldata = lock->omap_data; + if (ldata == NULL) { + spin_unlock(&lock->omap_spinlock); + ldata = kzalloc(sizeof(struct scoutfs_omap_lock_data), GFP_NOFS); + spin_lock(&lock->omap_spinlock); + + if (!ldata) { + ret = -ENOMEM; + goto out; + } + + if (lock->omap_data == NULL) { + ldata->version = lock->write_version - 1; /* ensure refresh */ + init_waitqueue_head(&ldata->waitq); + + lock->omap_data = ldata; + } else { + kfree(ldata); + ldata = lock->omap_data; + } + } + + while (ldata->version != lock->write_version) { + /* only one waiter sends a request at a time */ + if (!ldata->req_in_flight) { + ldata->req_in_flight = true; + send_req = true; + } else { + send_req = false; + } + + spin_unlock(&lock->omap_spinlock); + if (send_req) + ret = scoutfs_client_open_ino_map(sb, group_nr, &ldata->map); + else + wait_event(ldata->waitq, !omap_req_in_flight(lock, ldata)); + spin_lock(&lock->omap_spinlock); + + /* only sender can return error, other waiters retry */ + if (send_req) { + ldata->req_in_flight = false; + if (ret == 0) + ldata->version = lock->write_version; + wake_up(&ldata->waitq); + if (ret < 0) + goto out; + } + } + +out: + spin_unlock(&lock->omap_spinlock); + + if (ret == 0) + *ldata_ret = ldata; + else + *ldata_ret = NULL; + + return ret; +} + +/* + * Return 1 and give the caller a write inode lock if it is safe to be + * deleted. It's safe to be deleted when it is no longer reachable and + * nothing is referencing it. + * + * The inode is unreachable when nlink hits zero. Cluster locks protect + * modification and testing of nlink. We use the ino_lock_cov covrage + * to short circuit the common case of having a locked inode that hasn't + * been deleted. If it isn't locked, we have to acquire the lock to + * refresh the inode to see its current nlink. + * + * Then we use an open inode bitmap that covers all the inodes in the + * lock group to determine if the inode is present in any other mount's + * caches. We refresh it by asking the server for all clients' maps and + * then store it in the lock. As long as we hold the lock nothing can + * increase nlink from zero and let people get a reference to the inode. + */ +int scoutfs_omap_should_delete(struct super_block *sb, struct inode *inode, + struct scoutfs_lock **lock_ret) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct scoutfs_lock *lock = NULL; + const u64 ino = scoutfs_ino(inode); + struct scoutfs_omap_lock_data *ldata; + u64 group_nr; + int bit_nr; + int ret; + + /* lock group and omap constants are defined independently */ + BUILD_BUG_ON(SCOUTFS_OPEN_INO_MAP_BITS != SCOUTFS_LOCK_INODE_GROUP_NR); + + if (scoutfs_lock_is_covered(sb, &si->ino_lock_cov) && inode->i_nlink > 0) { + ret = 0; + goto out; + } + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, SCOUTFS_LKF_REFRESH_INODE, inode, &lock); + if (ret < 0) + goto out; + + if (inode->i_nlink > 0) { + ret = 0; + goto out; + } + + calc_group_nrs(ino, &group_nr, &bit_nr); + + /* only one request to refresh the map at a time */ + ret = get_current_lock_data(sb, lock, &ldata, group_nr); + if (ret < 0) + goto out; + + /* can delete caller's zero nlink inode if it's not cached in other mounts */ + ret = !test_bit_le(bit_nr, ldata->map.bits); +out: + trace_scoutfs_omap_should_delete(sb, ino, inode->i_nlink, ret); + + if (ret <= 0) { + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + lock = NULL; + } + + *lock_ret = lock; + return ret; +} + +void scoutfs_omap_free_lock_data(struct scoutfs_omap_lock_data *ldata) +{ + if (ldata) { + WARN_ON_ONCE(ldata->req_in_flight); + WARN_ON_ONCE(waitqueue_active(&ldata->waitq)); + kfree(ldata); + } +} + +int scoutfs_omap_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct omap_info *ominf; + int ret; + + ominf = kzalloc(sizeof(struct omap_info), GFP_KERNEL); + if (!ominf) { + ret = -ENOMEM; + goto out; + } + + ret = rhashtable_init(&ominf->group_ht, &group_ht_params); + if (ret < 0) { + kfree(ominf); + goto out; + } + + ret = rhashtable_init(&ominf->req_ht, &req_ht_params); + if (ret < 0) { + rhashtable_destroy(&ominf->group_ht); + kfree(ominf); + goto out; + } + + init_llist_head(&ominf->requests); + spin_lock_init(&ominf->lock); + init_rid_list(&ominf->rids); + atomic64_set(&ominf->next_req_id, 0); + + sbi->omap_info = ominf; + ret = 0; +out: + return ret; +} + +/* + * To get here the server must have shut down, freeing requests, and + * evict must have been called on all cached inodes so we can just + * synchronize all the pending group frees. + */ +void scoutfs_omap_destroy(struct super_block *sb) +{ + DECLARE_OMAP_INFO(sb, ominf); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct rhashtable_iter iter; + + if (ominf) { + synchronize_rcu(); + + /* double check that all the groups deced to 0 and were freed */ + rhashtable_walk_enter(&ominf->group_ht, &iter); + rhashtable_walk_start(&iter); + WARN_ON_ONCE(rhashtable_walk_peek(&iter) != NULL); + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); + + rhashtable_destroy(&ominf->group_ht); + rhashtable_destroy(&ominf->req_ht); + kfree(ominf); + sbi->omap_info = NULL; + } +} diff --git a/kmod/src/omap.h b/kmod/src/omap.h new file mode 100644 index 00000000..0e5fa29d --- /dev/null +++ b/kmod/src/omap.h @@ -0,0 +1,24 @@ +#ifndef _SCOUTFS_OMAP_H_ +#define _SCOUTFS_OMAP_H_ + +int scoutfs_omap_inc(struct super_block *sb, u64 ino); +void scoutfs_omap_dec(struct super_block *sb, u64 ino); +int scoutfs_omap_should_delete(struct super_block *sb, struct inode *inode, + struct scoutfs_lock **lock_ret); +void scoutfs_omap_free_lock_data(struct scoutfs_omap_lock_data *ldata); +int scoutfs_omap_client_handle_request(struct super_block *sb, u64 id, + struct scoutfs_open_ino_map_args *args); + +int scoutfs_omap_add_rid(struct super_block *sb, u64 rid); +int scoutfs_omap_remove_rid(struct super_block *sb, u64 rid); +int scoutfs_omap_finished_recovery(struct super_block *sb); +int scoutfs_omap_server_handle_request(struct super_block *sb, u64 rid, u64 id, + struct scoutfs_open_ino_map_args *args); +int scoutfs_omap_server_handle_response(struct super_block *sb, u64 rid, + struct scoutfs_open_ino_map *resp_map); +void scoutfs_omap_server_shutdown(struct super_block *sb); + +int scoutfs_omap_setup(struct super_block *sb); +void scoutfs_omap_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/recov.c b/kmod/src/recov.c new file mode 100644 index 00000000..b0d894c2 --- /dev/null +++ b/kmod/src/recov.c @@ -0,0 +1,280 @@ +/* + * Copyright (C) 2021 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "recov.h" + +/* + * There are a few server messages which can't be processed until they + * know that they have state for all possibly active clients. These + * little helpers track which clients have recovered what state and give + * those message handlers a call to check if recovery has completed. We + * track the timeout here, but all we do is call back into the server to + * take steps to evict timed out clients and then let us know that their + * recovery has finished. + */ + +struct recov_info { + struct super_block *sb; + spinlock_t lock; + struct list_head pending; + struct timer_list timer; + void (*timeout_fn)(struct super_block *); +}; + +#define DECLARE_RECOV_INFO(sb, name) \ + struct recov_info *name = SCOUTFS_SB(sb)->recov_info + +struct recov_pending { + struct list_head head; + u64 rid; + int which; +}; + +static struct recov_pending *find_pending(struct recov_info *recinf, u64 rid, int which) +{ + struct recov_pending *pend; + + list_for_each_entry(pend, &recinf->pending, head) { + if ((rid == 0 || pend->rid == rid) && (pend->which & which)) + return pend; + } + + return NULL; +} + +/* + * Record that we'll be waiting for a client to recover something. + * _finished will eventually be called for every _prepare, either + * because recovery naturally finished or because it timed out and the + * server evicted the client. + */ +int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct recov_pending *alloc; + struct recov_pending *pend; + + if (WARN_ON_ONCE(which & SCOUTFS_RECOV_INVALID)) + return -EINVAL; + + alloc = kmalloc(sizeof(*pend), GFP_NOFS); + if (!alloc) + return -ENOMEM; + + spin_lock(&recinf->lock); + + pend = find_pending(recinf, rid, SCOUTFS_RECOV_ALL); + if (pend) { + pend->which |= which; + } else { + swap(pend, alloc); + pend->rid = rid; + pend->which = which; + list_add(&pend->head, &recinf->pending); + } + + spin_unlock(&recinf->lock); + + kfree(alloc); + return 0; +} + +/* + * Recovery is only finished once we've begun (which sets the timer) and + * all clients have finished. If we didn't test the timer we could + * claim it finished prematurely as clients are being prepared. + */ +static int recov_finished(struct recov_info *recinf) +{ + return !!(recinf->timeout_fn != NULL && list_empty(&recinf->pending)); +} + +static void timer_callback(struct timer_list *timer) +{ + struct recov_info *recinf = from_timer(recinf, timer, timer); + + recinf->timeout_fn(recinf->sb); +} + +/* + * Begin waiting for recovery once we've prepared all the clients. If + * the timeout period elapses before _finish is called on all prepared + * clients then the timer will call the callback. + * + * Returns > 0 if all the prepared clients finish recovery before begin + * is called. + */ +int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_block *), + unsigned int timeout_ms) +{ + DECLARE_RECOV_INFO(sb, recinf); + int ret; + + spin_lock(&recinf->lock); + + recinf->timeout_fn = timeout_fn; + recinf->timer.expires = jiffies + msecs_to_jiffies(timeout_ms); + add_timer(&recinf->timer); + + ret = recov_finished(recinf); + + spin_unlock(&recinf->lock); + + if (ret > 0) + del_timer_sync(&recinf->timer); + + return ret; +} + +/* + * A given client has recovered the given state. If it's finished all + * recovery then we free it, and if all clients have finished recovery + * then we cancel the timeout timer. + * + * Returns > 0 if _begin has been called and all clients have finished. + * The caller will only see > 0 returned once. + */ +int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct recov_pending *pend; + int ret = 0; + + spin_lock(&recinf->lock); + + pend = find_pending(recinf, rid, which); + if (pend) { + pend->which &= ~which; + if (pend->which) { + pend = NULL; + } else { + list_del(&pend->head); + ret = recov_finished(recinf); + } + } + + spin_unlock(&recinf->lock); + + if (ret > 0) + del_timer_sync(&recinf->timer); + + kfree(pend); + + return ret; +} + +/* + * Returns true if the given client is still trying to recover + * the given state. + */ +bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which) +{ + DECLARE_RECOV_INFO(sb, recinf); + bool is_pending; + + spin_lock(&recinf->lock); + is_pending = find_pending(recinf, rid, which) != NULL; + spin_unlock(&recinf->lock); + + return is_pending; +} + +/* + * Returns 0 if there are no rids waiting for the given state to be + * recovered. Returns the rid of a client still waiting if there are + * any, in no specified order. + * + * This is inherently racey. Callers are responsible for resolving any + * actions taken based on pending with the recovery finishing, perhaps + * before we return. + */ +u64 scoutfs_recov_next_pending(struct super_block *sb, int which) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct recov_pending *pend; + u64 rid; + + spin_lock(&recinf->lock); + pend = find_pending(recinf, 0, which); + rid = pend ? pend->rid : 0; + spin_unlock(&recinf->lock); + + return rid; +} + +/* + * The server is shutting down and doesn't need to worry about recovery + * anymore. It'll be built up again by the next server, if needed. + */ +void scoutfs_recov_shutdown(struct super_block *sb) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct recov_pending *pend; + struct recov_pending *tmp; + LIST_HEAD(list); + + del_timer_sync(&recinf->timer); + + spin_lock(&recinf->lock); + list_splice_init(&recinf->pending, &list); + recinf->timeout_fn = NULL; + spin_unlock(&recinf->lock); + + list_for_each_entry_safe(pend, tmp, &recinf->pending, head) { + list_del(&pend->head); + kfree(pend); + } +} + +int scoutfs_recov_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct recov_info *recinf; + int ret; + + recinf = kzalloc(sizeof(struct recov_info), GFP_KERNEL); + if (!recinf) { + ret = -ENOMEM; + goto out; + } + + recinf->sb = sb; + spin_lock_init(&recinf->lock); + INIT_LIST_HEAD(&recinf->pending); + timer_setup(&recinf->timer, timer_callback, 0); + + sbi->recov_info = recinf; + ret = 0; +out: + return ret; +} + +void scoutfs_recov_destroy(struct super_block *sb) +{ + DECLARE_RECOV_INFO(sb, recinf); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (recinf) { + scoutfs_recov_shutdown(sb); + + kfree(recinf); + sbi->recov_info = NULL; + } +} diff --git a/kmod/src/recov.h b/kmod/src/recov.h new file mode 100644 index 00000000..cfdca30e --- /dev/null +++ b/kmod/src/recov.h @@ -0,0 +1,23 @@ +#ifndef _SCOUTFS_RECOV_H_ +#define _SCOUTFS_RECOV_H_ + +enum { + SCOUTFS_RECOV_GREETING = ( 1 << 0), + SCOUTFS_RECOV_LOCKS = ( 1 << 1), + + SCOUTFS_RECOV_INVALID = (~0 << 2), + SCOUTFS_RECOV_ALL = (~SCOUTFS_RECOV_INVALID), +}; + +int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which); +int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_block *), + unsigned int timeout_ms); +int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which); +bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which); +u64 scoutfs_recov_next_pending(struct super_block *sb, int which); +void scoutfs_recov_shutdown(struct super_block *sb); + +int scoutfs_recov_setup(struct super_block *sb); +void scoutfs_recov_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 91504e9c..7dce85f0 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -690,15 +690,16 @@ TRACE_EVENT(scoutfs_evict_inode, TRACE_EVENT(scoutfs_drop_inode, TP_PROTO(struct super_block *sb, __u64 ino, unsigned int nlink, - unsigned int unhashed), + unsigned int unhashed, bool drop_invalidated), - TP_ARGS(sb, ino, nlink, unhashed), + TP_ARGS(sb, ino, nlink, unhashed, drop_invalidated), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, ino) __field(unsigned int, nlink) __field(unsigned int, unhashed) + __field(unsigned int, drop_invalidated) ), TP_fast_assign( @@ -706,10 +707,12 @@ TRACE_EVENT(scoutfs_drop_inode, __entry->ino = ino; __entry->nlink = nlink; __entry->unhashed = unhashed; + __entry->drop_invalidated = !!drop_invalidated; ), - TP_printk(SCSBF" ino %llu nlink %u unhashed %d", SCSB_TRACE_ARGS, - __entry->ino, __entry->nlink, __entry->unhashed) + TP_printk(SCSBF" ino %llu nlink %u unhashed %d drop_invalidated %u", SCSB_TRACE_ARGS, + __entry->ino, __entry->nlink, __entry->unhashed, + __entry->drop_invalidated) ); TRACE_EVENT(scoutfs_inode_walk_writeback, @@ -2402,6 +2405,89 @@ TRACE_EVENT(scoutfs_item_invalidate_page, sk_trace_args(pg_start), sk_trace_args(pg_end), __entry->pgi) ); +DECLARE_EVENT_CLASS(scoutfs_omap_group_class, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(void *, grp) + __field(__u64, group_nr) + __field(unsigned int, group_total) + __field(int, bit_nr) + __field(int, bit_count) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->grp = grp; + __entry->group_nr = group_nr; + __entry->group_total = group_total; + __entry->bit_nr = bit_nr; + __entry->bit_count = bit_count; + ), + + TP_printk(SCSBF" grp %p group_nr %llu group_total %u bit_nr %d bit_count %d", + SCSB_TRACE_ARGS, __entry->grp, __entry->group_nr, __entry->group_total, + __entry->bit_nr, __entry->bit_count) +); + +DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_alloc, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count) +); +DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_free, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count) +); +DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_inc, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count) +); +DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_dec, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count) +); +DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_request, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count) +); +DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_destroy, + TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total, + int bit_nr, int bit_count), + TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count) +); + +TRACE_EVENT(scoutfs_omap_should_delete, + TP_PROTO(struct super_block *sb, u64 ino, unsigned int nlink, int ret), + + TP_ARGS(sb, ino, nlink, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(unsigned int, nlink) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->nlink = nlink; + __entry->ret = ret; + ), + + TP_printk(SCSBF" ino %llu nlink %u ret %d", + SCSB_TRACE_ARGS, __entry->ino, __entry->nlink, __entry->ret) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/server.c b/kmod/src/server.c index 6ecd8e85..369b2797 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -38,6 +38,8 @@ #include "srch.h" #include "alloc.h" #include "forest.h" +#include "recov.h" +#include "omap.h" /* * Every active mount can act as the server that listens on a net @@ -96,6 +98,9 @@ struct server_info { /* stable versions stored from commits, given in locks and rpcs */ seqcount_t roots_seqcount; struct scoutfs_net_roots roots; + + /* recovery timeout fences from work */ + struct work_struct fence_pending_recov_work; }; #define DECLARE_SERVER_INFO(sb, name) \ @@ -1016,6 +1021,60 @@ out: return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); } +/* The server is receiving an omap response from the client */ +static int open_ino_map_response(struct super_block *sb, struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, int error, void *data) +{ + u64 rid = scoutfs_net_client_rid(conn); + + if (resp_len != sizeof(struct scoutfs_open_ino_map)) + return -EINVAL; + + return scoutfs_omap_server_handle_response(sb, rid, resp); +} + +/* The server is sending an omap request to the client */ +int scoutfs_server_send_omap_request(struct super_block *sb, u64 rid, + struct scoutfs_open_ino_map_args *args) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + + return scoutfs_net_submit_request_node(sb, server->conn, rid, SCOUTFS_NET_CMD_OPEN_INO_MAP, + args, sizeof(*args), + open_ino_map_response, NULL, NULL); +} + +/* The server is sending an omap response to the client */ +int scoutfs_server_send_omap_response(struct super_block *sb, u64 rid, u64 id, + struct scoutfs_open_ino_map *map, int err) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + + return scoutfs_net_response_node(sb, server->conn, rid, + SCOUTFS_NET_CMD_OPEN_INO_MAP, id, err, + map, sizeof(*map)); +} + +/* The server is receiving an omap request from the client */ +static int server_open_ino_map(struct super_block *sb, struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + u64 rid = scoutfs_net_client_rid(conn); + int ret; + + if (arg_len != sizeof(struct scoutfs_open_ino_map_args)) { + ret = -EINVAL; + goto out; + } + + ret = scoutfs_omap_server_handle_request(sb, rid, id, arg); +out: + if (ret < 0) + return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); + + return 0; +} + static void init_mounted_client_key(struct scoutfs_key *key, u64 rid) { *key = (struct scoutfs_key) { @@ -1198,8 +1257,13 @@ static int server_greeting(struct super_block *sb, ret = scoutfs_server_apply_commit(sb, ret); queue_work(server->wq, &server->farewell_work); + if (ret < 0) + goto send_err; } + scoutfs_server_recov_finish(sb, le64_to_cpu(gr->rid), SCOUTFS_RECOV_GREETING); + ret = 0; + send_err: err = ret; @@ -1228,17 +1292,10 @@ send_err: scoutfs_net_server_greeting(sb, conn, le64_to_cpu(gr->rid), id, reconnecting, first_contact, farewell); - /* lock server might send recovery request */ + /* let layers know we have a client connecting for the first time */ if (le64_to_cpu(gr->server_term) != server->term) { - - /* we're now doing two commits per greeting, not great */ - ret = scoutfs_server_hold_commit(sb); - if (ret) - goto out; - - ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid), - gr->server_term != 0); - ret = scoutfs_server_apply_commit(sb, ret); + ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid)) ?: + scoutfs_omap_add_rid(sb, le64_to_cpu(gr->rid)); if (ret) goto out; } @@ -1259,6 +1316,25 @@ static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) sizeof(struct scoutfs_mounted_client_btree_val)); } +static int reclaim_rid(struct super_block *sb, u64 rid) +{ + int ret; + + ret = scoutfs_server_hold_commit(sb); + if (ret < 0) + return ret; + + /* delete mounted client last, client reconnect looks for it */ + ret = scoutfs_lock_server_farewell(sb, rid) ?: + remove_trans_seq(sb, rid) ?: + reclaim_log_trees(sb, rid) ?: + cancel_srch_compact(sb, rid) ?: + delete_mounted_client(sb, rid) ?: + scoutfs_omap_remove_rid(sb, rid); + + return scoutfs_server_apply_commit(sb, ret); +} + /* * This work processes farewell requests asynchronously. Requests from * quorum members can be held until only the final majority remains and @@ -1386,18 +1462,7 @@ static void farewell_worker(struct work_struct *work) /* process and send farewell responses */ list_for_each_entry_safe(fw, tmp, &send, entry) { - ret = scoutfs_server_hold_commit(sb); - if (ret) - goto out; - - /* delete mounted client last, client reconnect looks for it */ - ret = scoutfs_lock_server_farewell(sb, fw->rid) ?: - remove_trans_seq(sb, fw->rid) ?: - reclaim_log_trees(sb, fw->rid) ?: - cancel_srch_compact(sb, fw->rid) ?: - delete_mounted_client(sb, fw->rid); - - ret = scoutfs_server_apply_commit(sb, ret); + ret = reclaim_rid(sb, fw->rid); if (ret) goto out; } @@ -1499,6 +1564,7 @@ static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_LOCK] = server_lock, [SCOUTFS_NET_CMD_SRCH_GET_COMPACT] = server_srch_get_compact, [SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT] = server_srch_commit_compact, + [SCOUTFS_NET_CMD_OPEN_INO_MAP] = server_open_ino_map, [SCOUTFS_NET_CMD_FAREWELL] = server_farewell, }; @@ -1540,6 +1606,143 @@ static void server_notify_down(struct super_block *sb, } } +/* + * All clients have recovered all state. Now we can kick all the work + * that was waiting on recovery. + * + * It's a bit of a false dependency to have all work wait for completion + * before any work can make progress, but recovery is naturally + * concerned about in-memory state. It should all be quick to recover + * once a client arrives. + */ +static void finished_recovery(struct super_block *sb) +{ + DECLARE_SERVER_INFO(sb, server); + int ret = 0; + + scoutfs_info(sb, "all clients recovered"); + + ret = scoutfs_omap_finished_recovery(sb) ?: + scoutfs_lock_server_finished_recovery(sb); + if (ret < 0) { + scoutfs_err(sb, "error %d resuming after recovery finished, shutting down", ret); + stop_server(server); + } +} + +void scoutfs_server_recov_finish(struct super_block *sb, u64 rid, int which) +{ + if (scoutfs_recov_finish(sb, rid, which) > 0) + finished_recovery(sb); +} + +/* + * If the recovery timeout is too short we'll prematurely evict mounts + * that would have recovered. They need time to have their sockets + * timeout, reconnect to the current server, and fully recover their + * state. + * + * If it's too long we'll needlessly delay resuming operations after + * clients crash and will never recover. + */ +#define SERVER_RECOV_TIMEOUT_MS (30 * MSEC_PER_SEC) + +/* + * Not all clients recovered in time. We fence them and reclaim + * whatever resources they were using. If we see a rid here then we're + * going to fence it, regardless of if it manages to finish recovery + * while we're fencing it. + */ +static void fence_pending_recov_worker(struct work_struct *work) +{ + struct server_info *server = container_of(work, struct server_info, + fence_pending_recov_work); + struct super_block *sb = server->sb; + u64 rid; + int ret; + + while ((rid = scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_ALL)) > 0) { + scoutfs_err(sb, "%lu ms recovery timeout expired for client rid %016llx, fencing", + SERVER_RECOV_TIMEOUT_MS, rid); + + ret = reclaim_rid(sb, rid); + if (ret < 0) { + scoutfs_err(sb, "error %d reclaiming rid %016llx, shutting down", ret, rid); + stop_server(server); + break; + } + + scoutfs_server_recov_finish(sb, rid, SCOUTFS_RECOV_ALL); + } +} + +static void recovery_timeout(struct super_block *sb) +{ + DECLARE_SERVER_INFO(sb, server); + + if (!server->shutting_down) + queue_work(server->wq, &server->fence_pending_recov_work); +} + +/* + * As the server starts up it needs to start waiting for recovery from + * any clients which were previously still mounted in the last running + * server. This is done before networking is started so we won't + * receive any messages from clients until we've prepared them all. If + * the clients don't recover in time then they'll be fenced. + */ +static int start_recovery(struct super_block *sb) +{ + DECLARE_SERVER_INFO(sb, server); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + unsigned int nr = 0; + u64 rid; + int ret; + + for (rid = 0; ; rid++) { + init_mounted_client_key(&key, rid); + ret = scoutfs_btree_next(sb, &super->mounted_clients, &key, &iref); + if (ret == -ENOENT) { + ret = 0; + break; + } + if (ret == 0) { + rid = le64_to_cpu(iref.key->skmc_rid); + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) + goto out; + + ret = scoutfs_recov_prepare(sb, rid, SCOUTFS_RECOV_ALL); + if (ret < 0) { + scoutfs_err(sb, "error %d preparing recovery for client rid %016llx, shutting down", + ret, rid); + goto out; + } + + nr++; + } + + if (nr > 0) { + scoutfs_info(sb, "waiting for %u clients to recover", nr); + + ret = scoutfs_recov_begin(sb, recovery_timeout, SERVER_RECOV_TIMEOUT_MS); + if (ret > 0) { + finished_recovery(sb); + ret = 0; + } + } + +out: + if (ret < 0) { + scoutfs_err(sb, "error %d starting recovery, shutting down", ret); + stop_server(server); + } + return ret; +} + static void scoutfs_server_worker(struct work_struct *work) { struct server_info *server = container_of(work, struct server_info, @@ -1610,8 +1813,8 @@ static void scoutfs_server_worker(struct work_struct *work) goto shutdown; } - ret = scoutfs_lock_server_setup(sb, &server->alloc, &server->wri, - max_vers); + ret = scoutfs_lock_server_setup(sb, &server->alloc, &server->wri, max_vers) ?: + start_recovery(sb); if (ret) goto shutdown; @@ -1635,10 +1838,15 @@ shutdown: scoutfs_net_shutdown(sb, conn); server->conn = NULL; + /* stop tracking recovery, cancel timer, flush any fencing */ + scoutfs_recov_shutdown(sb); + flush_work(&server->fence_pending_recov_work); + /* wait for extra queues by requests, won't find waiters */ flush_work(&server->commit_work); scoutfs_lock_server_destroy(sb); + scoutfs_omap_server_shutdown(sb); out: scoutfs_net_free_conn(sb, conn); @@ -1724,6 +1932,7 @@ int scoutfs_server_setup(struct super_block *sb) mutex_init(&server->srch_mutex); mutex_init(&server->mounted_clients_mutex); seqcount_init(&server->roots_seqcount); + INIT_WORK(&server->fence_pending_recov_work, fence_pending_recov_worker); server->wq = alloc_workqueue("scoutfs_server", WQ_UNBOUND | WQ_NON_REENTRANT, 0); diff --git a/kmod/src/server.h b/kmod/src/server.h index 84e25ddb..8d31a271 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -64,6 +64,12 @@ int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, struct scoutfs_key *key); int scoutfs_server_hold_commit(struct super_block *sb); int scoutfs_server_apply_commit(struct super_block *sb, int err); +void scoutfs_server_recov_finish(struct super_block *sb, u64 rid, int which); + +int scoutfs_server_send_omap_request(struct super_block *sb, u64 rid, + struct scoutfs_open_ino_map_args *args); +int scoutfs_server_send_omap_response(struct super_block *sb, u64 rid, u64 id, + struct scoutfs_open_ino_map *map, int err); struct sockaddr_in; struct scoutfs_quorum_elected_info; diff --git a/kmod/src/super.c b/kmod/src/super.c index 1a795a69..b691b038 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -44,6 +44,8 @@ #include "srch.h" #include "item.h" #include "alloc.h" +#include "recov.h" +#include "omap.h" #include "scoutfs_trace.h" static struct dentry *scoutfs_debugfs_root; @@ -166,7 +168,7 @@ out: * try to free as many locks as possible. */ if (scoutfs_trigger(sb, STATFS_LOCK_PURGE)) - scoutfs_free_unused_locks(sb, -1UL); + scoutfs_free_unused_locks(sb); return ret; } @@ -243,25 +245,26 @@ static void scoutfs_put_super(struct super_block *sb) trace_scoutfs_put_super(sb); - sbi->shutdown = true; - - scoutfs_data_destroy(sb); scoutfs_srch_destroy(sb); scoutfs_unlock(sb, sbi->rid_lock, SCOUTFS_LOCK_WRITE); sbi->rid_lock = NULL; + scoutfs_lock_shutdown(sb); + scoutfs_shutdown_trans(sb); scoutfs_client_destroy(sb); scoutfs_inode_destroy(sb); scoutfs_item_destroy(sb); scoutfs_forest_destroy(sb); + scoutfs_data_destroy(sb); scoutfs_quorum_destroy(sb); - scoutfs_lock_shutdown(sb); scoutfs_server_destroy(sb); + scoutfs_recov_destroy(sb); scoutfs_net_destroy(sb); scoutfs_lock_destroy(sb); + scoutfs_omap_destroy(sb); scoutfs_block_destroy(sb); scoutfs_destroy_triggers(sb); @@ -591,8 +594,10 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_inode_setup(sb) ?: scoutfs_data_setup(sb) ?: scoutfs_setup_trans(sb) ?: + scoutfs_omap_setup(sb) ?: scoutfs_lock_setup(sb) ?: scoutfs_net_setup(sb) ?: + scoutfs_recov_setup(sb) ?: scoutfs_server_setup(sb) ?: scoutfs_quorum_setup(sb) ?: scoutfs_client_setup(sb) ?: @@ -643,6 +648,9 @@ static void scoutfs_kill_sb(struct super_block *sb) { trace_scoutfs_kill_sb(sb); + if (SCOUTFS_HAS_SBI(sb)) + scoutfs_lock_unmount_begin(sb); + kill_block_super(sb); } diff --git a/kmod/src/super.h b/kmod/src/super.h index 13912bdc..1790d40f 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -26,6 +26,8 @@ struct net_info; struct block_info; struct forest_info; struct srch_info; +struct recov_info; +struct omap_info; struct scoutfs_sb_info { struct super_block *sb; @@ -48,6 +50,7 @@ struct scoutfs_sb_info { struct block_info *block_info; struct forest_info *forest_info; struct srch_info *srch_info; + struct omap_info *omap_info; struct item_cache_info *item_cache_info; wait_queue_head_t trans_hold_wq; @@ -70,6 +73,7 @@ struct scoutfs_sb_info { struct lock_server_info *lock_server_info; struct client_info *client_info; struct server_info *server_info; + struct recov_info *recov_info; struct sysfs_info *sfsinfo; struct scoutfs_counters *counters; @@ -81,8 +85,6 @@ struct scoutfs_sb_info { struct dentry *debug_root; - bool shutdown; - unsigned long corruption_messages_once[SC_NR_LONGS]; }; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 742919f6..186239d4 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -564,8 +564,15 @@ int scoutfs_setup_trans(struct super_block *sb) } /* - * kill_sb calls sync before getting here so we know that dirty data - * should be in flight. We just have to wait for it to quiesce. + * While the vfs will have done an fs level sync before calling + * put_super, we may have done work down in our level after all the fs + * ops were done. An example is final inode deletion in iput, that's + * done in generic_shutdown_super after the sync and before calling our + * put_super. + * + * So we always try to write any remaining dirty transactions before + * shutting down. Typically there won't be any dirty data and the + * worker will just return. */ void scoutfs_shutdown_trans(struct super_block *sb) { @@ -573,13 +580,18 @@ void scoutfs_shutdown_trans(struct super_block *sb) DECLARE_TRANS_INFO(sb, tri); if (tri) { - scoutfs_block_writer_forget_all(sb, &tri->wri); if (sbi->trans_write_workq) { + /* immediately queues pending timer */ + flush_delayed_work(&sbi->trans_write_work); + /* prevents re-arming if it has to wait */ cancel_delayed_work_sync(&sbi->trans_write_work); destroy_workqueue(sbi->trans_write_workq); /* trans work schedules after shutdown see null */ sbi->trans_write_workq = NULL; } + + scoutfs_block_writer_forget_all(sb, &tri->wri); + kfree(tri); sbi->trans_info = NULL; } diff --git a/tests/funcs/filter.sh b/tests/funcs/filter.sh index 8226adad..5c793be4 100644 --- a/tests/funcs/filter.sh +++ b/tests/funcs/filter.sh @@ -52,8 +52,8 @@ t_filter_dmesg() # tests that drop unmount io triggers fencing re="$re|scoutfs .* error: fencing " - re="$re|scoutfs .*: waiting for .* lock clients" - re="$re|scoutfs .*: all lock clients recovered" + re="$re|scoutfs .*: waiting for .* clients" + re="$re|scoutfs .*: all clients recovered" re="$re|scoutfs .* error: client rid.*lock recovery timed out" # some tests mount w/o options diff --git a/tests/funcs/fs.sh b/tests/funcs/fs.sh index b4b17cf3..03fb9888 100644 --- a/tests/funcs/fs.sh +++ b/tests/funcs/fs.sh @@ -129,7 +129,7 @@ t_umount() test "$nr" -lt "$T_NR_MOUNTS" || \ t_fail "fs nr $nr invalid" - eval t_quiet umount \$T_M$i + eval t_quiet umount \$T_M$nr } # diff --git a/tests/golden/inode-deletion b/tests/golden/inode-deletion new file mode 100644 index 00000000..1e36ff37 --- /dev/null +++ b/tests/golden/inode-deletion @@ -0,0 +1,27 @@ +== basic unlink deletes +ino found in dseq index +ino not found in dseq index +== local open-unlink waits for close to delete +contents after rm: contents +ino found in dseq index +ino not found in dseq index +== multiple local opens are protected +contents after rm 1: contents +contents after rm 2: contents +ino found in dseq index +ino not found in dseq index +== remote unopened unlink deletes +ino not found in dseq index +ino not found in dseq index +== unlink wait for open on other mount +mount 0 contents after mount 1 rm: contents +ino found in dseq index +ino found in dseq index +stat: cannot stat ‘/mnt/test/test/inode-deletion/file’: No such file or directory +ino not found in dseq index +ino not found in dseq index +== lots of deletions use one open map +== open files survive remote scanning orphans +mount 0 contents after mount 1 remounted: contents +ino not found in dseq index +ino not found in dseq index diff --git a/tests/sequence b/tests/sequence index fe988b3a..e98afbad 100644 --- a/tests/sequence +++ b/tests/sequence @@ -30,4 +30,5 @@ mount-unmount-race.sh createmany-parallel-mounts.sh archive-light-cycle.sh block-stale-reads.sh +inode-deletion.sh xfstests.sh diff --git a/tests/tests/inode-deletion.sh b/tests/tests/inode-deletion.sh new file mode 100644 index 00000000..5af97354 --- /dev/null +++ b/tests/tests/inode-deletion.sh @@ -0,0 +1,98 @@ +# +# test deleting an inode once all its links and references are gone. +# + +t_require_commands cat scoutfs +t_require_mounts 2 + +FILE="$T_D0/file" + +check_ino_index() { + local ino="$1" + local dseq="$2" + local mnt="$3" + + t_sync_seq_index + + scoutfs walk-inodes -p "$mnt" -- data_seq $dseq $(($dseq + 1)) | + awk 'BEGIN { not = "not " } + ($4 == '$ino') { not = ""; exit; } + END { print "ino " not "found in dseq index" }' +} + +echo "== basic unlink deletes" +echo "contents" > "$FILE" +ino=$(stat -c "%i" "$FILE") +dseq=$(scoutfs stat -s data_seq "$FILE") +check_ino_index "$ino" "$dseq" "$T_M0" +rm -f "$FILE" +check_ino_index "$ino" "$dseq" "$T_M0" + +echo "== local open-unlink waits for close to delete" +echo "contents" > "$FILE" +ino=$(stat -c "%i" "$FILE") +dseq=$(scoutfs stat -s data_seq "$FILE") +exec {FD}<"$FILE" # open unused fd, assign to FD +rm -f "$FILE" +echo "contents after rm: $(cat <&$FD)" +check_ino_index "$ino" "$dseq" "$T_M0" +exec {FD}>&- # close +check_ino_index "$ino" "$dseq" "$T_M0" + +echo "== multiple local opens are protected" +echo "contents" > "$FILE" +ino=$(stat -c "%i" "$FILE") +dseq=$(scoutfs stat -s data_seq "$FILE") +exec {FD1}<"$FILE" +exec {FD2}<"$FILE" +rm -f "$FILE" +echo "contents after rm 1: $(cat <&$FD1)" +echo "contents after rm 2: $(cat <&$FD2)" +check_ino_index "$ino" "$dseq" "$T_M0" +exec {FD1}>&- # close +exec {FD2}>&- # close +check_ino_index "$ino" "$dseq" "$T_M0" + +echo "== remote unopened unlink deletes" +echo "contents" > "$T_D0/file" +ino=$(stat -c "%i" "$T_D0/file") +dseq=$(scoutfs stat -s data_seq "$T_D0/file") +rm -f "$T_D1/file" +check_ino_index "$ino" "$dseq" "$T_M0" +check_ino_index "$ino" "$dseq" "$T_M1" + +echo "== unlink wait for open on other mount" +echo "contents" > "$T_D0/file" +ino=$(stat -c "%i" "$T_D0/file") +dseq=$(scoutfs stat -s data_seq "$T_D0/file") +exec {FD}<"$T_D0/file" +rm -f "$T_D1/file" +echo "mount 0 contents after mount 1 rm: $(cat <&$FD)" +check_ino_index "$ino" "$dseq" "$T_M0" +check_ino_index "$ino" "$dseq" "$T_M1" +exec {FD}>&- # close +# we know that revalidating will unhash the remote dentry +stat "$T_D0/file" 2>&1 | t_filter_fs +check_ino_index "$ino" "$dseq" "$T_M0" +check_ino_index "$ino" "$dseq" "$T_M1" + +echo "== lots of deletions use one open map" +mkdir "$T_D0/dir" +touch "$T_D0/dir"/files-{1..5} +rm -f "$T_D0/dir"/files-* +rmdir "$T_D0/dir" + +echo "== open files survive remote scanning orphans" +echo "contents" > "$T_D0/file" +ino=$(stat -c "%i" "$T_D0/file") +dseq=$(scoutfs stat -s data_seq "$T_D0/file") +exec {FD}<"$T_D0/file" +rm -f "$T_D0/file" +t_umount 1 +t_mount 1 +echo "mount 0 contents after mount 1 remounted: $(cat <&$FD)" +exec {FD}>&- # close +check_ino_index "$ino" "$dseq" "$T_M0" +check_ino_index "$ino" "$dseq" "$T_M1" + +t_pass diff --git a/utils/src/print.c b/utils/src/print.c index 10140be6..c60c11d9 100644 --- a/utils/src/print.c +++ b/utils/src/print.c @@ -339,14 +339,6 @@ static int print_srch_root_item(struct scoutfs_key *key, void *val, return 0; } -static int print_lock_clients_entry(struct scoutfs_key *key, void *val, - unsigned val_len, void *arg) -{ - printf(" rid %016llx\n", le64_to_cpu(key->sklc_rid)); - - return 0; -} - static int print_trans_seqs_entry(struct scoutfs_key *key, void *val, unsigned val_len, void *arg) { @@ -876,7 +868,6 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno) " server_meta_avail[1]: "AL_HEAD_F"\n" " server_meta_freed[0]: "AL_HEAD_F"\n" " server_meta_freed[1]: "AL_HEAD_F"\n" - " lock_clients root: height %u blkno %llu seq %llu\n" " mounted_clients root: height %u blkno %llu seq %llu\n" " srch_root root: height %u blkno %llu seq %llu\n" " trans_seqs root: height %u blkno %llu seq %llu\n" @@ -896,9 +887,6 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno) AL_HEAD_A(&super->server_meta_avail[1]), AL_HEAD_A(&super->server_meta_freed[0]), AL_HEAD_A(&super->server_meta_freed[1]), - super->lock_clients.height, - le64_to_cpu(super->lock_clients.ref.blkno), - le64_to_cpu(super->lock_clients.ref.seq), super->mounted_clients.height, le64_to_cpu(super->mounted_clients.ref.blkno), le64_to_cpu(super->mounted_clients.ref.seq), @@ -947,11 +935,6 @@ static int print_volume(int fd) ret = print_quorum_blocks(fd, super); - err = print_btree(fd, super, "lock_clients", &super->lock_clients, - print_lock_clients_entry, NULL); - if (err && !ret) - ret = err; - err = print_btree(fd, super, "mounted_clients", &super->mounted_clients, print_mounted_client_entry, NULL); if (err && !ret)