Use count/scan objects shrinking interface

Move to the more recent interfaces for counting and scanning cached
objects to shrink.

Signed-off-by: Zach Brown <zab@versity.com>

ReleaseNotes.md

@@ -1,6 +1,126 @@
 Versity ScoutFS Release Notes
 =============================
 
+---
+v1.6
+\
+*Jul 7, 2022*
+
+* **Fix memory leaks in rare corner cases**
+\
+  Analysis tools found a few corner cases that leaked small structures,
+  generally around error handling or startup and shutdown.
+
+* **Add --skip-likely-huge scoutfs print command option**
+\
+  Add an option to scoutfs print to reduce the size of the output
+  so that it can be used to see system-wide metadata without being
+  overwhelmed by file-level details.
+
+---
+v1.5
+\
+*Jun 21, 2022*
+
+* **Fix persistent error during server startup**
+\
+  Fixed a case where the server would always hit a consistent error on
+  seartup, preventing the system from mounting.  This required a rare
+  but valid state across the clients.
+
+* **Fix a client hang that would lead to fencing**
+\
+  The client module's use of in-kernel networking was missing annotation
+  that could lead to communication hanging.  The server would fence the
+  client when it stopped communicating.  This could be identified by the
+  server fencing a client after it disconnected with no attempt by the
+  client to reconnect.
+
+---
+v1.4
+\
+*May 6, 2022*
+
+* **Fix possible client crash during server failover**
+\
+  Fixed a narrow window during server failover and lock recovery that
+  could cause a client mount to believe that it had an inconsistent item
+  cache and panic.  This required very specific lock state and messaging
+  patterns between multiple mounts and multiple servers which made it
+  unlikely to occur in the field.
+
+---
+v1.3
+\
+*Apr 7, 2022*
+
+* **Fix rare server instability under heavy load**
+\
+  Fixed a case of server instability under heavy load due to concurrent
+  work fully exhausting metadata block allocation pools reserved for a
+  single server transaction.  This would cause brief interruption as the
+  server shutdown and the next server started up and made progress as
+  pending work was retried.
+
+* **Fix slow fencing preventing server startup**
+\
+  If a server had to process many fence requests with a slow fencing
+  mechanism it could be interrupted before it finished.  The server
+  now makes sure heartbeat messages are sent while it is making progress
+  on fencing requests so that other quorum members don't interrupt the
+  process.
+
+* **Performance improvement in getxattr and setxattr**
+\
+  Kernel allocation patterns in the getxattr and setxattr
+  implementations were causing significant contention between CPUs.  Their
+  allocation strategy was changed so that concurrent tasks can call these
+  xattr methods without degrading performance.
+
+---
+v1.2
+\
+*Mar 14, 2022*
+
+* **Fix deadlock between fallocate() and read() system calls**
+\
+  Fixed a lock inversion that could cause two tasks to deadlock if they
+  performed fallocate() and read() on a file at the same time.   The
+  deadlock was uninterruptible so the machine needed to be rebooted.  This
+  was relatively rare as fallocate() is usually used to prepare files
+  before they're used.
+
+* **Fix instability from heavy file deletion workloads**
+\
+  Fixed rare circumstances under which background file deletion cleanup
+  tasks could try to delete a file while it is being deleted by another
+  task.  Heavy load across multiple nodes, either many files being deleted
+  or large files being deleted, increased the chances of this happening.
+  Heavy staging could cause this problem because staging can create many
+  internal temporary files that need to be deleted.
+
+---
+v1.1
+\
+*Feb 4, 2022*
+
+
+* **Add scoutfs(1) change-quorum-config command**
+\
+  Add a change-quorum-config command to scoutfs(1) to change the quorum
+  configuration stored in the metadata device while the file system is
+  unmounted.   This can be used to change the mounts that will
+  participate in quorum and the IP addresses they use.
+
+* **Fix Rare Risk of Item Cache Corruption**
+\
+  Code review found a rare potential source of item cache corruption.
+  If this happened it would look as though deleted parts of the filesystem
+  returned, but only at the time they were deleted.  Old deleted items are
+  not affected.  This problem only affected the item cache, never
+  persistent storage.  Unmounting and remounting would drop the bad item
+  cache and resync it with the correct persistent data.
+
 ---
 v1.0
 \

kmod/src/Makefile

@@ -46,6 +46,10 @@ scoutfs-y +=			\
 	volopt.o		\
 	xattr.o
 
+ifdef KC_BUILD_KERNELCOMPAT
+scoutfs-y += kernelcompat.o
+endif
+
 #
 # The raw types aren't available in userspace headers.  Make sure all
 # the types we use in the headers are the exported __ versions.

kmod/src/Makefile.kernelcompat

@@ -34,3 +34,52 @@ endif
 ifneq (,$(shell grep 'FMODE_KABI_ITERATE' include/linux/fs.h))
 ccflags-y += -DKC_FMODE_KABI_ITERATE
 endif
+
+#
+# v4.11-12447-g104b4e5139fe
+#
+# Renamed __percpu_counter_add to percpu_counter_add_batch to clarify
+# that the __ wasn't less safe, just took an extra parameter.
+#
+ifneq (,$(shell grep 'percpu_counter_add_batch' include/linux/percpu_counter.h))
+ccflags-y += -DKC_PERCPU_COUNTER_ADD_BATCH
+endif
+
+#
+# v4.11-4550-g7dea19f9ee63
+#
+# Introduced memalloc_nofs_{save,restore} preferred instead of _noio_.
+#
+ifneq (,$(shell grep 'memalloc_nofs_save' include/linux/sched/mm.h))
+ccflags-y += -DKC_MEMALLOC_NOFS_SAVE
+endif
+
+#
+# v4.7-12414-g1eff9d322a44
+#
+# Renamed bi_rw to bi_opf to force old code to catch up.  We use it as a
+# single switch between old and new bio structures.
+#
+ifneq (,$(shell grep 'bi_opf' include/linux/blk_types.h))
+ccflags-y += -DKC_BIO_BI_OPF
+endif
+
+#
+# v4.12-rc2-201-g4e4cbee93d56
+#
+# Moves to bi_status BLK_STS_ API instead of having a mix of error
+# end_io args or bi_error.
+#
+ifneq (,$(shell grep 'bi_status' include/linux/blk_types.h))
+ccflags-y += -DKC_BIO_BI_STATUS
+endif
+
+#
+# v3.11-8765-ga0b02131c5fc
+#
+# Remove the old ->shrink() API, ->{scan,count}_objects is preferred.
+#
+ifneq (,$(shell grep '(*shrink)' include/linux/shrinker.h))
+ccflags-y += -DKC_SHRINKER_SHRINK
+KC_BUILD_KERNELCOMPAT=1
+endif

kmod/src/alloc.c

@@ -84,6 +84,21 @@ static u64 smallest_order_length(u64 len)
 	return 1ULL << (free_extent_order(len) * 3);
 }
 
+/*
+ * An extent modification dirties three distinct leaves of an allocator
+ * btree as it adds and removes the blkno and size sorted items for the
+ * old and new lengths of the extent.  Dirtying the paths to these
+ * leaves can grow the tree and grow/shrink neighbours at each level.
+ * We over-estimate the number of blocks allocated and freed (the paths
+ * share a root, growth doesn't free) to err on the simpler and safer
+ * side.  The overhead is minimal given the relatively large list blocks
+ * and relatively short allocator trees.
+ */
+static u32 extent_mod_blocks(u32 height)
+{
+	return ((1 + height) * 2) * 3;
+}
+
 /*
  * Free extents don't have flags and are stored in two indexes sorted by
  * block location and by length order, largest first.  The location key
@@ -877,6 +892,14 @@ static int find_zone_extent(struct super_block *sb, struct scoutfs_alloc_root *r
  * -ENOENT is returned if we run out of extents in the source tree
  * before moving the total.
  *
+ * If meta_reserved is non-zero then -EINPROGRESS can be returned if the
+ * current meta allocator's avail blocks or room for freed blocks would
+ * have fallen under the reserved amount.  The could have been
+ * successfully dirtied in this case but the number of blocks moved is
+ * not returned.  The caller is expected to deal with the partial
+ * progress by commiting the dirty trees and examining the resulting
+ * modified trees to see if they need to continue moving extents.
+ *
  * The caller can specify that extents in the source tree should first
  * be found based on their zone bitmaps.  We'll first try to find
  * extents in the exclusive zones, then vacant zones, and then we'll
@@ -891,7 +914,7 @@ int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc,
 		       struct scoutfs_block_writer *wri,
 		       struct scoutfs_alloc_root *dst,
 		       struct scoutfs_alloc_root *src, u64 total,
-		       __le64 *exclusive, __le64 *vacant, u64 zone_blocks)
+		       __le64 *exclusive, __le64 *vacant, u64 zone_blocks, u64 meta_reserved)
 {
 	struct alloc_ext_args args = {
 		.alloc = alloc,
@@ -941,6 +964,14 @@ int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc,
 		if (ret < 0)
 			break;
 
+		if (meta_reserved != 0 &&
+		    scoutfs_alloc_meta_low(sb, alloc, meta_reserved +
+					   extent_mod_blocks(src->root.height) +
+					   extent_mod_blocks(dst->root.height))) {
+			ret = -EINPROGRESS;
+			break;
+		}
+
 		/* searching set start/len, finish initializing alloced extent */
 		ext.map = found.map ? ext.start - found.start + found.map : 0;
 		ext.flags = found.flags;
@@ -1065,15 +1096,6 @@ out:
  * than completely exhausting the avail list or overflowing the freed
  * list.
  *
- * An extent modification dirties three distinct leaves of an allocator
- * btree as it adds and removes the blkno and size sorted items for the
- * old and new lengths of the extent.  Dirtying the paths to these
- * leaves can grow the tree and grow/shrink neighbours at each level.
- * We over-estimate the number of blocks allocated and freed (the paths
- * share a root, growth doesn't free) to err on the simpler and safer
- * side.  The overhead is minimal given the relatively large list blocks
- * and relatively short allocator trees.
- *
  * The caller tells us how many extents they're about to modify and how
  * many other additional blocks they may cow manually.  And finally, the
  * caller could be the first to dirty the avail and freed blocks in the
@@ -1082,7 +1104,7 @@ out:
 static bool list_has_blocks(struct super_block *sb, struct scoutfs_alloc *alloc,
 			    struct scoutfs_alloc_root *root, u32 extents, u32 addl_blocks)
 {
-	u32 tree_blocks = (((1 + root->root.height) * 2) * 3) * extents;
+	u32 tree_blocks = extent_mod_blocks(root->root.height) * extents;
 	u32 most = 1 + tree_blocks + addl_blocks;
 
 	if (le32_to_cpu(alloc->avail.first_nr) < most) {
@@ -1318,6 +1340,17 @@ bool scoutfs_alloc_meta_low(struct super_block *sb,
 	return lo;
 }
 
+void scoutfs_alloc_meta_remaining(struct scoutfs_alloc *alloc, u32 *avail_total, u32 *freed_space)
+{
+	unsigned int seq;
+
+	do {
+		seq = read_seqbegin(&alloc->seqlock);
+		*avail_total = le32_to_cpu(alloc->avail.first_nr);
+		*freed_space = list_block_space(alloc->freed.first_nr);
+	} while (read_seqretry(&alloc->seqlock, seq));
+}
+
 bool scoutfs_alloc_test_flag(struct super_block *sb,
 			    struct scoutfs_alloc *alloc, u32 flag)
 {

kmod/src/alloc.h

@@ -131,7 +131,7 @@ int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc,
 		       struct scoutfs_block_writer *wri,
 		       struct scoutfs_alloc_root *dst,
 		       struct scoutfs_alloc_root *src, u64 total,
-		       __le64 *exclusive, __le64 *vacant, u64 zone_blocks);
+		       __le64 *exclusive, __le64 *vacant, u64 zone_blocks, u64 meta_reserved);
 int scoutfs_alloc_insert(struct super_block *sb, struct scoutfs_alloc *alloc,
 			 struct scoutfs_block_writer *wri, struct scoutfs_alloc_root *root,
 			 u64 start, u64 len);
@@ -158,6 +158,7 @@ int scoutfs_alloc_splice_list(struct super_block *sb,
 
 bool scoutfs_alloc_meta_low(struct super_block *sb,
 			    struct scoutfs_alloc *alloc, u32 nr);
+void scoutfs_alloc_meta_remaining(struct scoutfs_alloc *alloc, u32 *avail_total, u32 *freed_space);
 bool scoutfs_alloc_test_flag(struct super_block *sb,
 			    struct scoutfs_alloc *alloc, u32 flag);
 

kmod/src/block.c

@@ -21,6 +21,7 @@
 #include <linux/blkdev.h>
 #include <linux/rhashtable.h>
 #include <linux/random.h>
+#include <linux/sched/mm.h>
 
 #include "format.h"
 #include "super.h"
@@ -57,7 +58,7 @@ struct block_info {
 	atomic64_t access_counter;
 	struct rhashtable ht;
 	wait_queue_head_t waitq;
-	struct shrinker shrinker;
+	KC_DEFINE_SHRINKER(shrinker);
 	struct work_struct free_work;
 	struct llist_head free_llist;
 };
@@ -128,7 +129,7 @@ static __le32 block_calc_crc(struct scoutfs_block_header *hdr, u32 size)
 static struct block_private *block_alloc(struct super_block *sb, u64 blkno)
 {
 	struct block_private *bp;
-	unsigned int noio_flags;
+	unsigned int nofs_flags;
 
 	/*
 	 * If we had multiple blocks per page we'd need to be a little
@@ -156,9 +157,9 @@ static struct block_private *block_alloc(struct super_block *sb, u64 blkno)
 		 * spurious reclaim-on dependencies and warnings.
 		 */
 		lockdep_off();
-		noio_flags = memalloc_noio_save();
+		nofs_flags = memalloc_nofs_save();
 		bp->virt = __vmalloc(SCOUTFS_BLOCK_LG_SIZE, GFP_NOFS | __GFP_HIGHMEM, PAGE_KERNEL);
-		memalloc_noio_restore(noio_flags);
+		memalloc_nofs_restore(nofs_flags);
 		lockdep_on();
 
 		if (!bp->virt) {
@@ -436,11 +437,10 @@ static void block_remove_all(struct super_block *sb)
  * possible.  Final freeing, verifying checksums, and unlinking errored
  * blocks are all done by future users of the blocks.
  */
-static void block_end_io(struct super_block *sb, int rw,
+static void block_end_io(struct super_block *sb, unsigned int opf,
 			 struct block_private *bp, int err)
 {
 	DECLARE_BLOCK_INFO(sb, binf);
-	bool is_read = !(rw & WRITE);
 
 	if (err) {
 		scoutfs_inc_counter(sb, block_cache_end_io_error);
@@ -450,7 +450,7 @@ static void block_end_io(struct super_block *sb, int rw,
 	if (!atomic_dec_and_test(&bp->io_count))
 		return;
 
-	if (is_read && !test_bit(BLOCK_BIT_ERROR, &bp->bits))
+	if (!op_is_write(opf) && !test_bit(BLOCK_BIT_ERROR, &bp->bits))
 		set_bit(BLOCK_BIT_UPTODATE, &bp->bits);
 
 	clear_bit(BLOCK_BIT_IO_BUSY, &bp->bits);
@@ -463,13 +463,13 @@ static void block_end_io(struct super_block *sb, int rw,
 		wake_up(&binf->waitq);
 }
 
-static void block_bio_end_io(struct bio *bio, int err)
+static void KC_DECLARE_BIO_END_IO(block_bio_end_io, struct bio *bio)
 {
 	struct block_private *bp = bio->bi_private;
 	struct super_block *sb = bp->sb;
 
 	TRACE_BLOCK(end_io, bp);
-	block_end_io(sb, bio->bi_rw, bp, err);
+	block_end_io(sb, kc_bio_get_opf(bio), bp, kc_bio_get_errno(bio));
 	bio_put(bio);
 }
 
@@ -477,7 +477,7 @@ static void block_bio_end_io(struct bio *bio, int err)
  * Kick off IO for a single block.
  */
 static int block_submit_bio(struct super_block *sb, struct block_private *bp,
-			    int rw)
+			    unsigned int opf)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
 	struct bio *bio = NULL;
@@ -510,8 +510,9 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp,
 				break;
 			}
 
-			bio->bi_sector = sector + (off >> 9);
-			bio->bi_bdev = sbi->meta_bdev;
+			kc_bio_set_opf(bio, opf);
+			kc_bio_set_sector(bio, sector + (off >> 9));
+			bio_set_dev(bio, sbi->meta_bdev);
 			bio->bi_end_io = block_bio_end_io;
 			bio->bi_private = bp;
 
@@ -528,18 +529,18 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp,
 			BUG();
 
 		if (!bio_add_page(bio, page, PAGE_SIZE, 0)) {
-			submit_bio(rw, bio);
+			submit_bio(bio);
 			bio = NULL;
 		}
 	}
 
 	if (bio)
-		submit_bio(rw, bio);
+		submit_bio(bio);
 
 	blk_finish_plug(&plug);
 
 	/* let racing end_io know we're done */
-	block_end_io(sb, rw, bp, ret);
+	block_end_io(sb, opf, bp, ret);
 
 	return ret;
 }
@@ -640,7 +641,7 @@ static struct block_private *block_read(struct super_block *sb, u64 blkno)
 
 	if (!test_bit(BLOCK_BIT_UPTODATE, &bp->bits) &&
 	     test_and_clear_bit(BLOCK_BIT_NEW, &bp->bits)) {
-		ret = block_submit_bio(sb, bp, READ);
+		ret = block_submit_bio(sb, bp, REQ_OP_READ);
 		if (ret < 0)
 			goto out;
 	}
@@ -939,7 +940,7 @@ int scoutfs_block_writer_write(struct super_block *sb,
 		/* retry previous write errors */
 		clear_bit(BLOCK_BIT_ERROR, &bp->bits);
 
-		ret = block_submit_bio(sb, bp, WRITE);
+		ret = block_submit_bio(sb, bp, REQ_OP_WRITE);
 		if (ret < 0)
 			break;
 	}
@@ -1039,6 +1040,17 @@ u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb,
 	return wri->nr_dirty_blocks * SCOUTFS_BLOCK_LG_SIZE;
 }
 
+static unsigned long block_count_objects(struct shrinker *shrink, struct shrink_control *sc)
+{
+	struct block_info *binf = container_of(shrink, struct block_info, shrinker);
+	struct super_block *sb = binf->sb;
+
+	scoutfs_inc_counter(sb, block_cache_scan_objects);
+
+	return min_t(u64, (u64)atomic_read(&binf->total_inserted) * SCOUTFS_BLOCK_LG_PAGES_PER,
+		     ULONG_MAX / 2); /* magic numbers as we approach ~0UL :/ */
+}
+
 /*
  * Remove a number of cached blocks that haven't been used recently.
  *
@@ -1059,23 +1071,19 @@ u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb,
  * atomically remove blocks when the only references are ours and the
  * hash table.
  */
-static int block_shrink(struct shrinker *shrink, struct shrink_control *sc)
+static unsigned long block_scan_objects(struct shrinker *shrink, struct shrink_control *sc)
 {
-	struct block_info *binf = container_of(shrink, struct block_info,
-					       shrinker);
+	struct block_info *binf = container_of(shrink, struct block_info, shrinker);
 	struct super_block *sb = binf->sb;
 	struct rhashtable_iter iter;
 	struct block_private *bp;
+	unsigned long freed = 0;
 	unsigned long nr;
 	u64 recently;
 
-	nr = sc->nr_to_scan;
-	if (nr == 0)
-		goto out;
+	scoutfs_inc_counter(sb, block_cache_scan_objects);
 
-	scoutfs_inc_counter(sb, block_cache_shrink);
-
-	nr = DIV_ROUND_UP(nr, SCOUTFS_BLOCK_LG_PAGES_PER);
+	nr = DIV_ROUND_UP(sc->nr_to_scan, SCOUTFS_BLOCK_LG_PAGES_PER);
 
 restart:
 	recently = accessed_recently(binf);
@@ -1118,6 +1126,7 @@ restart:
 			if (block_remove_solo(sb, bp)) {
 				scoutfs_inc_counter(sb, block_cache_shrink_remove);
 				TRACE_BLOCK(shrink, bp);
+				freed++;
 				nr--;
 			}
 			block_put(sb, bp);
@@ -1126,9 +1135,8 @@ restart:
 
 	rhashtable_walk_stop(&iter);
 	rhashtable_walk_exit(&iter);
-out:
-	return min_t(u64, (u64)atomic_read(&binf->total_inserted) * SCOUTFS_BLOCK_LG_PAGES_PER,
-		     INT_MAX);
+
+	return freed;
 }
 
 struct sm_block_completion {
@@ -1136,11 +1144,11 @@ struct sm_block_completion {
 	int err;
 };
 
-static void sm_block_bio_end_io(struct bio *bio, int err)
+static void KC_DECLARE_BIO_END_IO(sm_block_bio_end_io, struct bio *bio)
 {
 	struct sm_block_completion *sbc = bio->bi_private;
 
-	sbc->err = err;
+	sbc->err = kc_bio_get_errno(bio);
 	complete(&sbc->comp);
 	bio_put(bio);
 }
@@ -1155,9 +1163,8 @@ static void sm_block_bio_end_io(struct bio *bio, int err)
  * only layer that sees the full block buffer so we pass the calculated
  * crc to the caller for them to check in their context.
  */
-static int sm_block_io(struct super_block *sb, struct block_device *bdev, int rw, u64 blkno,
-		       struct scoutfs_block_header *hdr, size_t len,
-		       __le32 *blk_crc)
+static int sm_block_io(struct super_block *sb, struct block_device *bdev, unsigned int opf,
+		       u64 blkno, struct scoutfs_block_header *hdr, size_t len, __le32 *blk_crc)
 {
 	struct scoutfs_block_header *pg_hdr;
 	struct sm_block_completion sbc;
@@ -1171,7 +1178,7 @@ static int sm_block_io(struct super_block *sb, struct block_device *bdev, int rw
 		return -EIO;
 
 	if (WARN_ON_ONCE(len > SCOUTFS_BLOCK_SM_SIZE) ||
-	    WARN_ON_ONCE(!(rw & WRITE) && !blk_crc))
+	    WARN_ON_ONCE(!op_is_write(opf) && !blk_crc))
 		return -EINVAL;
 
 	page = alloc_page(GFP_NOFS);
@@ -1180,7 +1187,7 @@ static int sm_block_io(struct super_block *sb, struct block_device *bdev, int rw
 
 	pg_hdr = page_address(page);
 
-	if (rw & WRITE) {
+	if (op_is_write(opf)) {
 		memcpy(pg_hdr, hdr, len);
 		if (len < SCOUTFS_BLOCK_SM_SIZE)
 			memset((char *)pg_hdr + len, 0,
@@ -1194,8 +1201,9 @@ static int sm_block_io(struct super_block *sb, struct block_device *bdev, int rw
 		goto out;
 	}
 
-	bio->bi_sector = blkno << (SCOUTFS_BLOCK_SM_SHIFT - 9);
-	bio->bi_bdev = bdev;
+	bio->bi_opf = opf | REQ_SYNC;
+	kc_bio_set_sector(bio, blkno << (SCOUTFS_BLOCK_SM_SHIFT - 9));
+	bio_set_dev(bio, bdev);
 	bio->bi_end_io = sm_block_bio_end_io;
 	bio->bi_private = &sbc;
 	bio_add_page(bio, page, SCOUTFS_BLOCK_SM_SIZE, 0);
@@ -1203,12 +1211,12 @@ static int sm_block_io(struct super_block *sb, struct block_device *bdev, int rw
 	init_completion(&sbc.comp);
 	sbc.err = 0;
 
-	submit_bio((rw & WRITE) ? WRITE_SYNC : READ_SYNC, bio);
+	submit_bio(bio);
 
 	wait_for_completion(&sbc.comp);
 	ret = sbc.err;
 
-	if (ret == 0 && !(rw & WRITE)) {
+	if (ret == 0 && !op_is_write(opf)) {
 		memcpy(hdr, pg_hdr, len);
 		*blk_crc = block_calc_crc(pg_hdr, SCOUTFS_BLOCK_SM_SIZE);
 	}
@@ -1222,14 +1230,14 @@ int scoutfs_block_read_sm(struct super_block *sb,
 			  struct scoutfs_block_header *hdr, size_t len,
 			  __le32 *blk_crc)
 {
-	return sm_block_io(sb, bdev, READ, blkno, hdr, len, blk_crc);
+	return sm_block_io(sb, bdev, REQ_OP_READ, blkno, hdr, len, blk_crc);
 }
 
 int scoutfs_block_write_sm(struct super_block *sb,
 			   struct block_device *bdev, u64 blkno,
 			   struct scoutfs_block_header *hdr, size_t len)
 {
-	return sm_block_io(sb, bdev, WRITE, blkno, hdr, len, NULL);
+	return sm_block_io(sb, bdev, REQ_OP_WRITE, blkno, hdr, len, NULL);
 }
 
 int scoutfs_block_setup(struct super_block *sb)
@@ -1254,7 +1262,8 @@ int scoutfs_block_setup(struct super_block *sb)
 	atomic_set(&binf->total_inserted, 0);
 	atomic64_set(&binf->access_counter, 0);
 	init_waitqueue_head(&binf->waitq);
-	binf->shrinker.shrink = block_shrink;
+	KC_INIT_SHRINKER_FUNCS(struct block_info, shrinker,
+			       &binf->shrinker, block_count_objects, block_scan_objects);
 	binf->shrinker.seeks = DEFAULT_SEEKS;
 	register_shrinker(&binf->shrinker);
 	INIT_WORK(&binf->free_work, block_free_work);

kmod/src/btree.c

@@ -1233,10 +1233,6 @@ static int btree_walk(struct super_block *sb,
 	    WARN_ON_ONCE((flags & (BTW_GET_PAR|BTW_SET_PAR)) && !par_root))
 		return -EINVAL;
 
-	/* all ops come through walk and walk calls all reads */
-	if (scoutfs_forcing_unmount(sb))
-		return -EIO;
-
 	scoutfs_inc_counter(sb, btree_walk);
 
 restart:
@@ -2453,7 +2449,7 @@ int scoutfs_btree_free_blocks(struct super_block *sb,
 			      struct scoutfs_alloc *alloc,
 			      struct scoutfs_block_writer *wri,
 			      struct scoutfs_key *key,
-			      struct scoutfs_btree_root *root, int alloc_low)
+			      struct scoutfs_btree_root *root, int free_budget)
 {
 	u64 blknos[SCOUTFS_BTREE_MAX_HEIGHT];
 	struct scoutfs_block *bl = NULL;
@@ -2463,11 +2459,15 @@ int scoutfs_btree_free_blocks(struct super_block *sb,
 	struct scoutfs_avl_node *node;
 	struct scoutfs_avl_node *next;
 	struct scoutfs_key par_next;
+	int nr_freed = 0;
 	int nr_par;
 	int level;
 	int ret;
 	int i;
 
+	if (WARN_ON_ONCE(free_budget <= 0))
+		return -EINVAL;
+
 	if (WARN_ON_ONCE(root->height > ARRAY_SIZE(blknos)))
 		return -EIO; /* XXX corruption */
 
@@ -2542,8 +2542,7 @@ int scoutfs_btree_free_blocks(struct super_block *sb,
 		while (node) {
 
 			/* make sure we can always free parents after leaves */
-			if (scoutfs_alloc_meta_low(sb, alloc,
-						   alloc_low + nr_par + 1)) {
+			if ((nr_freed + 1 + nr_par) > free_budget) {
 				ret = 0;
 				goto out;
 			}
@@ -2557,6 +2556,7 @@ int scoutfs_btree_free_blocks(struct super_block *sb,
 						le64_to_cpu(ref.blkno));
 			if (ret < 0)
 				goto out;
+			nr_freed++;
 
 			node = scoutfs_avl_next(&bt->item_root, node);
 			if (node) {
@@ -2572,6 +2572,7 @@ int scoutfs_btree_free_blocks(struct super_block *sb,
 							       blknos[i]);
 			ret = scoutfs_free_meta(sb, alloc, wri, blknos[i]);
 			BUG_ON(ret); /* checked meta low, freed should fit */
+			nr_freed++;
 		}
 
 		/* restart walk past the subtree we just freed */

kmod/src/btree.h

@@ -125,7 +125,7 @@ int scoutfs_btree_free_blocks(struct super_block *sb,
 			      struct scoutfs_alloc *alloc,
 			      struct scoutfs_block_writer *wri,
 			      struct scoutfs_key *key,
-			      struct scoutfs_btree_root *root, int alloc_low);
+			      struct scoutfs_btree_root *root, int free_budget);
 
 void scoutfs_btree_put_iref(struct scoutfs_btree_item_ref *iref);
 

kmod/src/client.c

@@ -477,12 +477,15 @@ static void scoutfs_client_connect_worker(struct work_struct *work)
 	struct super_block *sb = client->sb;
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
 	struct scoutfs_super_block *super = &sbi->super;
-	struct mount_options *opts = &sbi->opts;
-	const bool am_quorum = opts->quorum_slot_nr >= 0;
+	struct scoutfs_mount_options opts;
 	struct scoutfs_net_greeting greet;
 	struct sockaddr_in sin;
+	bool am_quorum;
 	int ret;
 
+	scoutfs_options_read(sb, &opts);
+	am_quorum = opts.quorum_slot_nr >= 0;
+
 	/* can unmount once server farewell handling removes our item */
 	if (client->sending_farewell &&
 	    lookup_mounted_client_item(sb, sbi->rid) == 0) {
@@ -668,3 +671,11 @@ void scoutfs_client_destroy(struct super_block *sb)
 	kfree(client);
 	sbi->client_info = NULL;
 }
+
+void scoutfs_client_net_shutdown(struct super_block *sb)
+{
+	struct client_info *client = SCOUTFS_SB(sb)->client_info;
+
+	if (client && client->conn)
+		scoutfs_net_shutdown(sb, client->conn);
+}

kmod/src/client.h

@@ -35,6 +35,7 @@ int scoutfs_client_clear_volopt(struct super_block *sb, struct scoutfs_volume_op
 int scoutfs_client_resize_devices(struct super_block *sb, struct scoutfs_net_resize_devices *nrd);
 int scoutfs_client_statfs(struct super_block *sb, struct scoutfs_net_statfs *nst);
 
+void scoutfs_client_net_shutdown(struct super_block *sb);
 int scoutfs_client_setup(struct super_block *sb);
 void scoutfs_client_destroy(struct super_block *sb);
 

kmod/src/counters.h

@@ -30,6 +30,8 @@
 	EXPAND_COUNTER(block_cache_free)			\
 	EXPAND_COUNTER(block_cache_free_work)			\
 	EXPAND_COUNTER(block_cache_remove_stale)		\
+	EXPAND_COUNTER(block_cache_count_objects)		\
+	EXPAND_COUNTER(block_cache_scan_objects)		\
 	EXPAND_COUNTER(block_cache_shrink)			\
 	EXPAND_COUNTER(block_cache_shrink_next)			\
 	EXPAND_COUNTER(block_cache_shrink_recent)		\
@@ -152,11 +154,12 @@
 	EXPAND_COUNTER(net_recv_messages)			\
 	EXPAND_COUNTER(net_unknown_request)			\
 	EXPAND_COUNTER(orphan_scan)				\
+	EXPAND_COUNTER(orphan_scan_attempts)			\
 	EXPAND_COUNTER(orphan_scan_cached)			\
 	EXPAND_COUNTER(orphan_scan_error)			\
 	EXPAND_COUNTER(orphan_scan_item)			\
 	EXPAND_COUNTER(orphan_scan_omap_set)			\
-	EXPAND_COUNTER(orphan_scan_read)			\
+	EXPAND_COUNTER(quorum_candidate_server_stopping)	\
 	EXPAND_COUNTER(quorum_elected)				\
 	EXPAND_COUNTER(quorum_fence_error)			\
 	EXPAND_COUNTER(quorum_fence_leader)			\
@@ -234,12 +237,12 @@ struct scoutfs_counters {
 #define SCOUTFS_PCPU_COUNTER_BATCH (1 << 30)
 
 #define scoutfs_inc_counter(sb, which)					\
-	__percpu_counter_add(&SCOUTFS_SB(sb)->counters->which, 1,	\
-			     SCOUTFS_PCPU_COUNTER_BATCH)
+	percpu_counter_add_batch(&SCOUTFS_SB(sb)->counters->which, 1,	\
+				 SCOUTFS_PCPU_COUNTER_BATCH)
 
 #define scoutfs_add_counter(sb, which, cnt)				\
-	__percpu_counter_add(&SCOUTFS_SB(sb)->counters->which, cnt,	\
-			     SCOUTFS_PCPU_COUNTER_BATCH)
+	percpu_counter_add_batch(&SCOUTFS_SB(sb)->counters->which, cnt,	\
+				 SCOUTFS_PCPU_COUNTER_BATCH)
 
 void __init scoutfs_init_counters(void);
 int scoutfs_setup_counters(struct super_block *sb);

kmod/src/data.c

@@ -983,9 +983,6 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
 	u64 last;
 	s64 ret;
 
-	mutex_lock(&inode->i_mutex);
-	down_write(&si->extent_sem);
-
 	/* XXX support more flags */
         if (mode & ~(FALLOC_FL_KEEP_SIZE)) {
 		ret = -EOPNOTSUPP;
@@ -1003,18 +1000,22 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
 		goto out;
 	}
 
+	mutex_lock(&inode->i_mutex);
+
 	ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE,
 				 SCOUTFS_LKF_REFRESH_INODE, inode, &lock);
 	if (ret)
-		goto out;
+		goto out_mutex;
 
 	inode_dio_wait(inode);
 
+	down_write(&si->extent_sem);
+
 	if (!(mode & FALLOC_FL_KEEP_SIZE) &&
 	    (offset + len > i_size_read(inode))) {
                 ret = inode_newsize_ok(inode, offset + len);
                 if (ret)
-                        goto out;
+                        goto out_extent;
         }
 
 	iblock = offset >> SCOUTFS_BLOCK_SM_SHIFT;
@@ -1024,7 +1025,7 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
 
 		ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, true);
 		if (ret)
-			goto out;
+			goto out_extent;
 
 		ret = fallocate_extents(sb, inode, iblock, last, lock);
 
@@ -1050,17 +1051,19 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
 		}
 
 		if (ret <= 0)
-			goto out;
+			goto out_extent;
 
 		iblock += ret;
 		ret = 0;
 	}
 
-out:
-	scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE);
+out_extent:
 	up_write(&si->extent_sem);
+out_mutex:
+	scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE);
 	mutex_unlock(&inode->i_mutex);
 
+out:
 	trace_scoutfs_data_fallocate(sb, ino, mode, offset, len, ret);
 	return ret;
 }

kmod/src/dir.c

@@ -511,7 +511,7 @@ out:
 	else if (ino == 0)
 		inode = NULL;
 	else
-		inode = scoutfs_iget(sb, ino, 0);
+		inode = scoutfs_iget(sb, ino, 0, 0);
 
 	/*
 	 * We can't splice dir aliases into the dcache.  dir entries
@@ -720,7 +720,7 @@ static struct inode *lock_hold_create(struct inode *dir, struct dentry *dentry,
 				      struct list_head *ind_locks)
 {
 	struct super_block *sb = dir->i_sb;
-	struct inode *inode;
+	struct inode *inode = NULL;
 	u64 ind_seq;
 	int ret = 0;
 	u64 ino;
@@ -765,11 +765,9 @@ retry:
 	if (ret)
 		goto out_unlock;
 
-	inode = scoutfs_new_inode(sb, dir, mode, rdev, ino, *inode_lock);
-	if (IS_ERR(inode)) {
-		ret = PTR_ERR(inode);
+	ret = scoutfs_new_inode(sb, dir, mode, rdev, ino, *inode_lock, &inode);
+	if (ret < 0)
 		goto out;
-	}
 
 	ret = scoutfs_dirty_inode_item(dir, *dir_lock);
 out:
@@ -787,6 +785,8 @@ out_unlock:
 			*orph_lock = NULL;
 		}
 
+		if (!IS_ERR_OR_NULL(inode))
+			iput(inode);
 		inode = ERR_PTR(ret);
 	}
 
@@ -1319,11 +1319,11 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry,
 	insert_inode_hash(inode);
 	/* XXX need to set i_op/fop before here for sec callbacks */
 	d_instantiate(dentry, inode);
+	inode = NULL;
+	ret = 0;
 out:
 	if (ret < 0) {
 		/* XXX remove inode items */
-		if (!IS_ERR_OR_NULL(inode))
-			iput(inode);
 
 		symlink_item_ops(sb, SYM_DELETE, scoutfs_ino(inode), inode_lock,
 				 NULL, name_len);
@@ -1334,6 +1334,9 @@ out:
 	scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_WRITE);
 	scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE);
 
+	if (!IS_ERR_OR_NULL(inode))
+		iput(inode);
+
 	return ret;
 }
 
@@ -1615,8 +1618,9 @@ static int verify_ancestors(struct super_block *sb, u64 p1, u64 p2,
  * from using parent/child locking orders as two groups can have both
  * parent and child relationships to each other.
  */
-static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry,
-			  struct inode *new_dir, struct dentry *new_dentry)
+static int scoutfs_rename_common(struct inode *old_dir,
+				 struct dentry *old_dentry, struct inode *new_dir,
+				 struct dentry *new_dentry, unsigned int flags)
 {
 	struct super_block *sb = old_dir->i_sb;
 	struct inode *old_inode = old_dentry->d_inode;
@@ -1688,6 +1692,11 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry,
 	if (ret)
 		goto out_unlock;
 
+	if ((flags & RENAME_NOREPLACE) && (new_inode != NULL)) {
+		ret = -EEXIST;
+		goto out_unlock;
+	}
+
 	if (should_orphan(new_inode)) {
 		ret = scoutfs_lock_orphan(sb, SCOUTFS_LOCK_WRITE_ONLY, 0, scoutfs_ino(new_inode),
 					  &orph_lock);
@@ -1870,6 +1879,23 @@ out_unlock:
 	return ret;
 }
 
+static int scoutfs_rename(struct inode *old_dir,
+			  struct dentry *old_dentry, struct inode *new_dir,
+			  struct dentry *new_dentry)
+{
+	return scoutfs_rename_common(old_dir, old_dentry, new_dir, new_dentry, 0);
+}
+
+static int scoutfs_rename2(struct inode *old_dir,
+			  struct dentry *old_dentry, struct inode *new_dir,
+			  struct dentry *new_dentry, unsigned int flags)
+{
+	if (flags & ~RENAME_NOREPLACE)
+		return -EINVAL;
+
+	return scoutfs_rename_common(old_dir, old_dentry, new_dir, new_dentry, flags);
+}
+
 #ifdef KC_FMODE_KABI_ITERATE
 /* we only need this to set the iterate flag for kabi :/ */
 static int scoutfs_dir_open(struct inode *inode, struct file *file)
@@ -1900,10 +1926,8 @@ static int scoutfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mod
 	si = SCOUTFS_I(inode);
 
 	ret = scoutfs_inode_orphan_create(sb, scoutfs_ino(inode), orph_lock);
-	if (ret < 0) {
-		iput(inode);
+	if (ret < 0)
 		goto out; /* XXX returning error but items created */
-	}
 
 	inode->i_mtime = inode->i_atime = inode->i_ctime = CURRENT_TIME;
 	si->crtime = inode->i_mtime;
@@ -1916,7 +1940,6 @@ static int scoutfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mod
 	scoutfs_update_inode_item(inode, inode_lock, &ind_locks);
 	scoutfs_update_inode_item(dir, dir_lock, &ind_locks);
 	scoutfs_inode_index_unlock(sb, &ind_locks);
-	iput(inode);
 
 out:
 	scoutfs_release_trans(sb);
@@ -1925,6 +1948,9 @@ out:
 	scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE);
 	scoutfs_unlock(sb, orph_lock, SCOUTFS_LOCK_WRITE_ONLY);
 
+	if (!IS_ERR_OR_NULL(inode))
+		iput(inode);
+
 	return ret;
 }
 
@@ -1960,6 +1986,7 @@ const struct inode_operations_wrapper scoutfs_dir_iops = {
 	.permission	= scoutfs_permission,
 	},
 	.tmpfile	= scoutfs_tmpfile,
+	.rename2	= scoutfs_rename2,
 };
 
 void scoutfs_dir_exit(void)

kmod/src/export.c

@@ -81,7 +81,7 @@ static struct dentry *scoutfs_fh_to_dentry(struct super_block *sb,
 	trace_scoutfs_fh_to_dentry(sb, fh_type, sfid);
 
 	if (scoutfs_valid_fileid(fh_type))
-		inode = scoutfs_iget(sb, le64_to_cpu(sfid->ino), 0);
+		inode = scoutfs_iget(sb, le64_to_cpu(sfid->ino), 0, SCOUTFS_IGF_LINKED);
 
 	return d_obtain_alias(inode);
 }
@@ -100,7 +100,7 @@ static struct dentry *scoutfs_fh_to_parent(struct super_block *sb,
 
 	if (scoutfs_valid_fileid(fh_type) &&
 	    fh_type == FILEID_SCOUTFS_WITH_PARENT)
-		inode = scoutfs_iget(sb, le64_to_cpu(sfid->parent_ino), 0);
+		inode = scoutfs_iget(sb, le64_to_cpu(sfid->parent_ino), 0, SCOUTFS_IGF_LINKED);
 
 	return d_obtain_alias(inode);
 }
@@ -123,7 +123,7 @@ static struct dentry *scoutfs_get_parent(struct dentry *child)
 	scoutfs_dir_free_backref_path(sb, &list);
 	trace_scoutfs_get_parent(sb, inode, ino);
 
-	inode = scoutfs_iget(sb, ino, 0);
+	inode = scoutfs_iget(sb, ino, 0, SCOUTFS_IGF_LINKED);
 
 	return d_obtain_alias(inode);
 }

kmod/src/fence.c

@@ -395,12 +395,13 @@ int scoutfs_fence_wait_fenced(struct super_block *sb, long timeout_jiffies)
 int scoutfs_fence_setup(struct super_block *sb)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
-	struct mount_options *opts = &sbi->opts;
+	struct scoutfs_mount_options opts;
 	struct fence_info *fi;
 	int ret;
 
 	/* can only fence if we can be elected by quorum */
-	if (opts->quorum_slot_nr == -1) {
+	scoutfs_options_read(sb, &opts);
+	if (opts.quorum_slot_nr == -1) {
 		ret = 0;
 		goto out;
 	}

kmod/src/inode.c

@@ -66,10 +66,6 @@ struct inode_sb_info {
 
 	struct delayed_work orphan_scan_dwork;
 
-	/* serialize multiple inode ->evict trying to delete same ino's items */
-	spinlock_t deleting_items_lock;
-	struct list_head deleting_items_list;
-
 	struct work_struct iput_work;
 	struct llist_head iput_llist;
 };
@@ -276,7 +272,7 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode)
 	set_item_info(si, cinode);
 }
 
-static void init_inode_key(struct scoutfs_key *key, u64 ino)
+void scoutfs_inode_init_key(struct scoutfs_key *key, u64 ino)
 {
 	*key = (struct scoutfs_key) {
 		.sk_zone = SCOUTFS_FS_ZONE,
@@ -296,8 +292,7 @@ static void init_inode_key(struct scoutfs_key *key, u64 ino)
  * fields because they should have already had a locked refreshed inode
  * to be dereferencing its contents.
  */
-int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock,
-			  int flags)
+int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock)
 {
 	struct scoutfs_inode_info *si = SCOUTFS_I(inode);
 	struct super_block *sb = inode->i_sb;
@@ -317,7 +312,7 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock,
 	if (atomic64_read(&si->last_refreshed) == refresh_gen)
 		return 0;
 
-	init_inode_key(&key, scoutfs_ino(inode));
+	scoutfs_inode_init_key(&key, scoutfs_ino(inode));
 
 	mutex_lock(&si->item_mutex);
 	if (atomic64_read(&si->last_refreshed) < refresh_gen) {
@@ -663,22 +658,12 @@ void scoutfs_inode_get_onoff(struct inode *inode, s64 *on, s64 *off)
 	} while (read_seqcount_retry(&si->seqcount, seq));
 }
 
-/*
- * We have inversions between getting cluster locks while performing
- * final deletion on a freeing inode and waiting on a freeing inode
- * while holding a cluster lock.
- *
- * We can avoid these deadlocks by hiding freeing inodes in our hash
- * lookup function.  We're fine with either returning null or populating
- * a new inode overlapping with eviction freeing a previous instance of
- * the inode.
- */
 static int scoutfs_iget_test(struct inode *inode, void *arg)
 {
 	struct scoutfs_inode_info *si = SCOUTFS_I(inode);
 	u64 *ino = arg;
 
-	return (si->ino == *ino) && !(inode->i_state & I_FREEING);
+	return si->ino == *ino;
 }
 
 static int scoutfs_iget_set(struct inode *inode, void *arg)
@@ -692,49 +677,93 @@ static int scoutfs_iget_set(struct inode *inode, void *arg)
 	return 0;
 }
 
-struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino)
+/*
+ * There's a risk of a deadlock between lock invalidation and eviction.
+ * Invalidation blocks locks while looking up inodes.  Eviction blocks
+ * inode lookups while trying to get a lock.
+ *
+ * We have an inode lookup variant which will never block waiting for an
+ * inode.   This is more aggressive than base ilookup5_nowait() which
+ * will, you know, wait for inodes that are being freed.   We have our
+ * test function hide those inodes from find_inode so that it won't wait
+ * on them.
+ *
+ * These semantics are sufficiently weird that we use a big giant scary
+ * looking function name to deter use.
+ */
+static int ilookup_test_nonewfree(struct inode *inode, void *arg)
 {
-	return ilookup5(sb, ino, scoutfs_iget_test, &ino);
+	return scoutfs_iget_test(inode, arg) &&
+	       !(inode->i_state & (I_NEW | I_WILL_FREE | I_FREEING));
+}
+struct inode *scoutfs_ilookup_nowait_nonewfree(struct super_block *sb, u64 ino)
+{
+	return ilookup5_nowait(sb, ino, ilookup_test_nonewfree, &ino);
 }
 
-struct inode *scoutfs_iget(struct super_block *sb, u64 ino, int lkf)
+/*
+ * Final iput can delete an unused inode's items which can take multiple
+ * locked transactions.  iget (which can call iput in error cases) and
+ * iput must not be called with locks or transactions held.
+ */
+struct inode *scoutfs_iget(struct super_block *sb, u64 ino, int lkf, int igf)
 {
 	struct scoutfs_lock *lock = NULL;
 	struct scoutfs_inode_info *si;
-	struct inode *inode;
+	struct inode *inode = NULL;
 	int ret;
 
-	ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_READ, lkf, ino, &lock);
-	if (ret)
-		return ERR_PTR(ret);
-
-	inode = iget5_locked(sb, ino, scoutfs_iget_test, scoutfs_iget_set,
-			     &ino);
+	/* wait for vfs inode (I_FREEING in particular) before acquiring cluster lock */
+	inode = iget5_locked(sb, ino, scoutfs_iget_test, scoutfs_iget_set, &ino);
 	if (!inode) {
-		inode = ERR_PTR(-ENOMEM);
+		ret = -ENOMEM;
 		goto out;
 	}
 
+	ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_READ, lkf, ino, &lock);
+	if (ret < 0)
+		goto out;
+
 	if (inode->i_state & I_NEW) {
 		/* XXX ensure refresh, instead clear in drop_inode? */
 		si = SCOUTFS_I(inode);
 		atomic64_set(&si->last_refreshed, 0);
 		inode->i_version = 0;
-
-		ret = scoutfs_inode_refresh(inode, lock, 0);
-		if (ret == 0)
-			ret = scoutfs_omap_inc(sb, ino);
-		if (ret) {
-			iget_failed(inode);
-			inode = ERR_PTR(ret);
-		} else {
-			set_inode_ops(inode);
-			unlock_new_inode(inode);
-		}
 	}
 
+	ret = scoutfs_inode_refresh(inode, lock);
+	if (ret < 0)
+		goto out;
+
+	/* check nlink both for new and after refreshing */
+	if ((igf & SCOUTFS_IGF_LINKED) && inode->i_nlink == 0) {
+		ret = -ENOENT;
+		goto out;
+	}
+
+	if (inode->i_state & I_NEW) {
+		ret = scoutfs_omap_set(sb, ino);
+		if (ret < 0)
+			goto out;
+
+		set_inode_ops(inode);
+		unlock_new_inode(inode);
+	}
+
+	ret = 0;
 out:
 	scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ);
+
+	if (ret < 0) {
+		if (inode) {
+			if (inode->i_state & I_NEW)
+				iget_failed(inode);
+			else
+				iput(inode);
+		}
+		inode = ERR_PTR(ret);
+	}
+
 	return inode;
 }
 
@@ -803,7 +832,7 @@ int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock)
 
 	store_inode(&sinode, inode);
 
-	init_inode_key(&key, scoutfs_ino(inode));
+	scoutfs_inode_init_key(&key, scoutfs_ino(inode));
 
 	ret = scoutfs_item_update(sb, &key, &sinode, sizeof(sinode), lock);
 	if (!ret)
@@ -1022,7 +1051,7 @@ void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock,
 	ret = update_indices(sb, si, ino, inode->i_mode, &sinode, lock_list);
 	BUG_ON(ret);
 
-	init_inode_key(&key, ino);
+	scoutfs_inode_init_key(&key, ino);
 
 	err = scoutfs_item_update(sb, &key, &sinode, sizeof(sinode), lock);
 	if (err) {
@@ -1382,10 +1411,14 @@ out:
 /*
  * Allocate and initialize a new inode.  The caller is responsible for
  * creating links to it and updating it.  @dir can be null.
+ *
+ * This is called with locks and a transaction because it creates the
+ * inode item.   We can't call iput on the new inode on error.   We
+ * return the inode to the caller *including on error* for them to put
+ * once they've released the transaction.
  */
-struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir,
-				umode_t mode, dev_t rdev, u64 ino,
-				struct scoutfs_lock *lock)
+int scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev,
+		      u64 ino, struct scoutfs_lock *lock, struct inode **inode_ret)
 {
 	struct scoutfs_inode_info *si;
 	struct scoutfs_key key;
@@ -1394,8 +1427,10 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir,
 	int ret;
 
 	inode = new_inode(sb);
-	if (!inode)
-		return ERR_PTR(-ENOMEM);
+	if (!inode) {
+		ret = -ENOMEM;
+		goto out;
+	}
 
 	si = SCOUTFS_I(inode);
 	si->ino = ino;
@@ -1421,22 +1456,19 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir,
 	set_inode_ops(inode);
 
 	store_inode(&sinode, inode);
-	init_inode_key(&key, scoutfs_ino(inode));
+	scoutfs_inode_init_key(&key, scoutfs_ino(inode));
 
-	ret = scoutfs_omap_inc(sb, ino);
+	ret = scoutfs_omap_set(sb, ino);
 	if (ret < 0)
 		goto out;
 
 	ret = scoutfs_item_create(sb, &key, &sinode, sizeof(sinode), lock);
 	if (ret < 0)
-		scoutfs_omap_dec(sb, ino);
+		scoutfs_omap_clear(sb, ino);
 out:
-	if (ret) {
-		iput(inode);
-		inode = ERR_PTR(ret);
-	}
+	*inode_ret = inode;
 
-	return inode;
+	return ret;
 }
 
 static void init_orphan_key(struct scoutfs_key *key, u64 ino)
@@ -1471,44 +1503,6 @@ int scoutfs_inode_orphan_delete(struct super_block *sb, u64 ino, struct scoutfs_
 	return scoutfs_item_delete_force(sb, &key, lock);
 }
 
-struct deleting_ino_entry {
-	struct list_head head;
-	u64 ino;
-};
-
-static bool added_deleting_ino(struct inode_sb_info *inf, struct deleting_ino_entry *del, u64 ino)
-{
-	struct deleting_ino_entry *tmp;
-	bool added = true;
-
-	spin_lock(&inf->deleting_items_lock);
-
-	list_for_each_entry(tmp, &inf->deleting_items_list, head) {
-		if (tmp->ino == ino) {
-			added = false;
-			break;
-		}
-	}
-
-	if (added) {
-		del->ino = ino;
-		list_add_tail(&del->head, &inf->deleting_items_list);
-	}
-
-	spin_unlock(&inf->deleting_items_lock);
-
-	return added;
-}
-
-static void del_deleting_ino(struct inode_sb_info *inf, struct deleting_ino_entry *del)
-{
-	if (del->ino) {
-		spin_lock(&inf->deleting_items_lock);
-		list_del_init(&del->head);
-		spin_unlock(&inf->deleting_items_lock);
-	}
-}
-
 /*
  * Remove all the items associated with a given inode.  This is only
  * called once nlink has dropped to zero and nothing has the inode open
@@ -1517,22 +1511,10 @@ static void del_deleting_ino(struct inode_sb_info *inf, struct deleting_ino_entr
  * orphan item will continue triggering attempts to finish previous
  * partial deletion until all deletion is complete and the orphan item
  * is removed.
- *
- * Currently this can be called multiple times for multiple cached
- * inodes for a given ino number (ilookup avoids freeing inodes to avoid
- * cluster lock<->inode flag waiting inversions).  Some items are not
- * safe to delete concurrently, for example concurrent data truncation
- * could free extents multiple times.  We use a very silly list of inos
- * being deleted.  Duplicates just return success.  If the first
- * deletion ends up failing orphan deletion will come back around later
- * and retry.
  */
-static int delete_inode_items(struct super_block *sb, u64 ino, struct scoutfs_lock *lock,
-			      struct scoutfs_lock *orph_lock)
+static int delete_inode_items(struct super_block *sb, u64 ino, struct scoutfs_inode *sinode,
+			      struct scoutfs_lock *lock, struct scoutfs_lock *orph_lock)
 {
-	DECLARE_INODE_SB_INFO(sb, inf);
-	struct deleting_ino_entry del = {{NULL, }};
-	struct scoutfs_inode sinode;
 	struct scoutfs_key key;
 	LIST_HEAD(ind_locks);
 	bool release = false;
@@ -1541,30 +1523,10 @@ static int delete_inode_items(struct super_block *sb, u64 ino, struct scoutfs_lo
 	u64 size;
 	int ret;
 
-	if (!added_deleting_ino(inf, &del, ino)) {
-		ret = 0;
-		goto out;
-	}
+	scoutfs_inode_init_key(&key, ino);
 
-	init_inode_key(&key, ino);
-
-	ret = scoutfs_item_lookup_exact(sb, &key, &sinode, sizeof(sinode),
-					lock);
-	if (ret < 0) {
-		if (ret == -ENOENT)
-			ret = 0;
-		goto out;
-	}
-
-	/* XXX corruption, inode probably won't be freed without repair */
-	if (le32_to_cpu(sinode.nlink)) {
-		scoutfs_warn(sb, "Dangling orphan item for inode %llu.", ino);
-		ret = -EIO;
-		goto out;
-	}
-
-	mode = le32_to_cpu(sinode.mode);
-	size = le64_to_cpu(sinode.size);
+	mode = le32_to_cpu(sinode->mode);
+	size = le64_to_cpu(sinode->size);
 	trace_scoutfs_delete_inode(sb, ino, mode, size);
 
 	/* remove data items in their own transactions */
@@ -1582,7 +1544,7 @@ static int delete_inode_items(struct super_block *sb, u64 ino, struct scoutfs_lo
 	/* then delete the small known number of remaining inode items */
 retry:
 	ret = scoutfs_inode_index_start(sb, &ind_seq) ?:
-	      prepare_index_deletion(sb, &ind_locks, ino, mode, &sinode) ?:
+	      prepare_index_deletion(sb, &ind_locks, ino, mode, sinode) ?:
 	      scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, false);
 	if (ret > 0)
 		goto retry;
@@ -1591,7 +1553,7 @@ retry:
 
 	release = true;
 
-	ret = remove_index_items(sb, ino, &sinode, &ind_locks);
+	ret = remove_index_items(sb, ino, sinode, &ind_locks);
 	if (ret)
 		goto out;
 
@@ -1601,15 +1563,21 @@ retry:
 			goto out;
 	}
 
-	ret = scoutfs_item_delete(sb, &key, lock);
-	if (ret)
+	/* make sure inode item and orphan are deleted together */
+	ret = scoutfs_item_dirty(sb, &key, lock);
+	if (ret < 0)
 		goto out;
 
 	ret = scoutfs_inode_orphan_delete(sb, ino, orph_lock);
-	if (ret == 0)
-		scoutfs_forest_dec_inode_count(sb);
+	if (ret < 0)
+		goto out;
+
+	ret = scoutfs_item_delete(sb, &key, lock);
+	BUG_ON(ret != 0); /* dirtying should have guaranteed success */
+
+	scoutfs_forest_dec_inode_count(sb);
+
 out:
-	del_deleting_ino(inf, &del);
 	if (release)
 		scoutfs_release_trans(sb);
 	scoutfs_inode_index_unlock(sb, &ind_locks);
@@ -1617,48 +1585,192 @@ out:
 	return ret;
 }
 
+struct inode_deletion_lock_data {
+	wait_queue_head_t waitq;
+	atomic64_t seq;
+	struct scoutfs_open_ino_map map;
+	unsigned long trying[DIV_ROUND_UP(SCOUTFS_OPEN_INO_MAP_BITS, BITS_PER_LONG)];
+};
+
 /*
- * iput_final has already written out the dirty pages to the inode
- * before we get here.  We're left with a clean inode that we have to
- * tear down.  We use locking and open inode number bitmaps to decide if
- * we should finally destroy an inode that is no longer open nor
- * reachable through directory entries.
+ * Get a lock data struct that has the current omap from this hold of
+ * the lock.  The lock data is saved on the lock so it can be used
+ * multiple times until the lock is refreshed.  Only one task will send
+ * an omap request at a time, and errors are only returned by each task
+ * as it gets a response to its send.
+ */
+static int get_current_lock_data(struct super_block *sb, struct scoutfs_lock *lock,
+				 struct inode_deletion_lock_data **ldata_ret, u64 group_nr)
+{
+	struct inode_deletion_lock_data *ldata;
+	u64 seq;
+	int ret;
+
+	/* we're storing omap maps in locks, they need to cover the same number of inodes */
+	BUILD_BUG_ON(SCOUTFS_OPEN_INO_MAP_BITS != SCOUTFS_LOCK_INODE_GROUP_NR);
+
+	/* allocate a new lock data struct as needed */
+	while ((ldata = cmpxchg(&lock->inode_deletion_data, NULL, NULL)) == NULL) {
+		ldata = kzalloc(sizeof(struct inode_deletion_lock_data), GFP_NOFS);
+		if (!ldata) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		atomic64_set(&ldata->seq, lock->write_seq - 1); /* ensure refresh */
+		init_waitqueue_head(&ldata->waitq);
+
+		/* the lock kfrees the inode_deletion_data pointer along with the lock */
+		if (cmpxchg(&lock->inode_deletion_data, NULL, ldata) == NULL)
+			break;
+		else
+			kfree(ldata);
+	}
+
+	/* make sure that the lock's data is current */
+	while ((seq = atomic64_read(&ldata->seq)) != lock->write_seq) {
+		if (seq != U64_MAX && atomic64_cmpxchg(&ldata->seq, seq, U64_MAX) == seq) {
+			/* ask the server for current omap */
+			ret = scoutfs_client_open_ino_map(sb, group_nr, &ldata->map);
+			if (ret == 0)
+				atomic64_set(&ldata->seq, lock->write_seq);
+			else
+				atomic64_set(&ldata->seq, lock->write_seq - 1);
+			wake_up(&ldata->waitq);
+			if (ret < 0)
+				goto out;
+		} else {
+			/* wait for someone else who's sent a request */
+			wait_event(ldata->waitq, atomic64_read(&ldata->seq) != U64_MAX);
+		}
+	}
+
+	ret = 0;
+out:
+	if (ret < 0)
+		ldata = NULL;
+	*ldata_ret = ldata;
+	return ret;
+}
+
+/*
+ * Try to delete all the items for an unused inode number.  This is the
+ * relatively slow path that uses cluster locks, network requests, and
+ * IO to ensure correctness.  Callers should try hard to avoid calling
+ * when there's no work to do.
  *
- * Because lookup ignores freeing inodes we can get here from multiple
- * instances of an inode that is being deleted.  Orphan scanning in
- * particular can race with deletion.   delete_inode_items() resolves
- * concurrent attempts.
+ * Inode references are added under cluster locks.  In-memory vfs cache
+ * references are added under read cluster locks and are visible in omap
+ * bitmaps.  Directory entry references are added under write cluster
+ * locks and are visible in the inode's nlink.  Orphan items exist
+ * whenever nlink == 0 and are maintained under write cluster locks.
+ * Directory entries can be added to an inode with nlink == 0 to
+ * instantiate tmpfile inodes into the name space.  Cached inodes will
+ * not be created for inodes with an nlink of 0.
+ *
+ * Combining all this we know that it's safe to delete an inode's items
+ * when we hold an exclusive write cluster lock, the inode has nlink ==
+ * 0, and an omap request protected by the lock doesn't have the inode's
+ * bit set.
+ *
+ * This is called by orphan scanning and vfs inode cache eviction after
+ * they've checked that the inode could really be deleted.  We serialize
+ * on a bit in the lock data so that we only have one deletion attempt
+ * per inode under this mount's cluster lock.
+ */
+static int try_delete_inode_items(struct super_block *sb, u64 ino)
+{
+	struct inode_deletion_lock_data *ldata = NULL;
+	struct scoutfs_lock *orph_lock = NULL;
+	struct scoutfs_lock *lock = NULL;
+	struct scoutfs_inode sinode;
+	struct scoutfs_key key;
+	u64 group_nr;
+	int bit_nr;
+	int ret;
+
+	ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_WRITE, 0, ino, &lock);
+	if (ret < 0)
+		goto out;
+
+	scoutfs_omap_calc_group_nrs(ino, &group_nr, &bit_nr);
+
+	ret = get_current_lock_data(sb, lock, &ldata, group_nr);
+	if (ret < 0)
+		goto out;
+
+	/* only one local attempt per inode at a time */
+	if (test_and_set_bit(bit_nr, ldata->trying)) {
+		ret = 0;
+		goto out;
+	}
+
+	/* can't delete if it's cached in local or remote mounts */
+	if (scoutfs_omap_test(sb, ino) || test_bit_le(bit_nr, ldata->map.bits)) {
+		ret = 0;
+		goto out;
+	}
+
+	scoutfs_inode_init_key(&key, ino);
+	ret = scoutfs_item_lookup_exact(sb, &key, &sinode, sizeof(sinode), lock);
+	if (ret < 0) {
+		if (ret == -ENOENT)
+			ret = 0;
+		goto out;
+	}
+
+	if (le32_to_cpu(sinode.nlink) > 0) {
+		ret = 0;
+		goto out;
+	}
+
+	ret = scoutfs_lock_orphan(sb, SCOUTFS_LOCK_WRITE_ONLY, 0, ino, &orph_lock);
+	if (ret < 0)
+		goto out;
+
+	ret = delete_inode_items(sb, ino, &sinode, lock, orph_lock);
+out:
+	if (ldata)
+		clear_bit(bit_nr, ldata->trying);
+
+	scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE);
+	scoutfs_unlock(sb, orph_lock, SCOUTFS_LOCK_WRITE_ONLY);
+
+	return ret;
+}
+
+/*
+ * As we drop an inode we need to decide to try and delete its items or
+ * not, which is expensive.  The two common cases we want to get right
+ * both have cluster lock coverage and don't want to delete.   Dropping
+ * unused inodes during read lock invalidation has the current lock and
+ * sees a nonzero nlink and knows not to delete.  Final iput after a
+ * local unlink also has a lock, sees a zero nlink, and tries to perform
+ * item deletion in the task that dropped the last link, as users
+ * expect. 
+ *
+ * Evicting an inode outside of cluster locking is the odd slow path
+ * that involves lock contention during use the worst cross-mount
+ * open-unlink/delete case.
  */
 void scoutfs_evict_inode(struct inode *inode)
 {
 	struct super_block *sb = inode->i_sb;
+	struct scoutfs_inode_info *si = SCOUTFS_I(inode);
 	const u64 ino = scoutfs_ino(inode);
-	struct scoutfs_lock *orph_lock;
-	struct scoutfs_lock *lock;
-	int ret;
 
-	trace_scoutfs_evict_inode(inode->i_sb, scoutfs_ino(inode),
-				  inode->i_nlink, is_bad_inode(inode));
+	trace_scoutfs_evict_inode(sb, ino, inode->i_nlink, is_bad_inode(inode));
 
-	if (is_bad_inode(inode))
-		goto clear;
+	if (!is_bad_inode(inode)) {
+		truncate_inode_pages_final(&inode->i_data);
 
-	truncate_inode_pages_final(&inode->i_data);
+		/* clear before trying to delete tests */
+		scoutfs_omap_clear(sb, ino);
 
-	ret = scoutfs_omap_should_delete(sb, inode, &lock, &orph_lock);
-	if (ret > 0) {
-		ret = delete_inode_items(inode->i_sb, scoutfs_ino(inode), lock, orph_lock);
-		scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE);
-		scoutfs_unlock(sb, orph_lock, SCOUTFS_LOCK_WRITE_ONLY);
-	}
-	if (ret < 0) {
-		scoutfs_err(sb, "error %d while checking to delete inode nr %llu, it might linger.",
-			    ret, ino);
+		if (!scoutfs_lock_is_covered(sb, &si->ino_lock_cov) || inode->i_nlink == 0)
+			try_delete_inode_items(sb, scoutfs_ino(inode));
 	}
 
-	scoutfs_omap_dec(sb, ino);
-
-clear:
 	clear_inode(inode);
 }
 
@@ -1734,18 +1846,26 @@ void scoutfs_inode_queue_iput(struct inode *inode)
 /*
  * All mounts are performing this work concurrently.  We introduce
  * significant jitter between them to try and keep them from all
- * bunching up and working on the same inodes.
+ * bunching up and working on the same inodes.  We always try to delay
+ * for at least one jiffy if precision tricks us into calculating no
+ * delay.
  */
-static void schedule_orphan_dwork(struct inode_sb_info *inf)
+void scoutfs_inode_schedule_orphan_dwork(struct super_block *sb)
 {
-#define ORPHAN_SCAN_MIN_MS (10 * MSEC_PER_SEC)
-#define ORPHAN_SCAN_JITTER_MS (40 * MSEC_PER_SEC)
-	unsigned long delay = msecs_to_jiffies(ORPHAN_SCAN_MIN_MS +
-					       prandom_u32_max(ORPHAN_SCAN_JITTER_MS));
+	DECLARE_INODE_SB_INFO(sb, inf);
+	struct scoutfs_mount_options opts;
+	unsigned long low;
+	unsigned long high;
+	unsigned long delay;
+
 	if (!inf->stopped) {
-		delay = msecs_to_jiffies(ORPHAN_SCAN_MIN_MS +
-					 prandom_u32_max(ORPHAN_SCAN_JITTER_MS));
-		schedule_delayed_work(&inf->orphan_scan_dwork, delay);
+		scoutfs_options_read(sb, &opts);
+
+		low = (opts.orphan_scan_delay_ms * 80) / 100;
+		high = (opts.orphan_scan_delay_ms * 120) / 100;
+		delay = msecs_to_jiffies(low + prandom_u32_max(high - low)) ?: 1;
+
+		mod_delayed_work(system_wq, &inf->orphan_scan_dwork, delay);
 	}
 }
 
@@ -1753,11 +1873,10 @@ static void schedule_orphan_dwork(struct inode_sb_info *inf)
  * Find and delete inodes whose only remaining reference is the
  * persistent orphan item that was created as they were unlinked.
  *
- * Orphan items are created as the final directory entry referring to an
- * inode is deleted.  They're deleted as the final cached inode is
- * evicted and the inode items are destroyed.  They can linger if all
- * the cached inodes pinning the inode fail to delete as they are
- * evicted from the cache -- either through crashing or errors.
+ * Orphan items are maintained for inodes that have an nlink of 0.
+ * Typically this is from unlink, but tmpfiles are created with orphans.
+ * They're deleted as the final cached inode is evicted and the inode
+ * items are destroyed.
  *
  * This work runs in all mounts in the background looking for those
  * orphaned inodes that weren't fully deleted.
@@ -1766,20 +1885,16 @@ static void schedule_orphan_dwork(struct inode_sb_info *inf)
  * only find orphan items that made it to the fs root after being merged
  * from a mount's log btree.  This naturally avoids orphan items that
  * exist while inodes have been unlinked but are still cached, including
- * O_TMPFILE inodes that are actively used during normal operations.
+ * tmpfile inodes that are actively used during normal operations.
  * Scanning the read-only persistent fs root uses cached blocks and
  * avoids the lock contention we'd cause if we tried to use the
  * consistent item cache.  The downside is that it adds a bit of
- * latency.  If an orphan was created in error it'll take until the
- * mount's log btree is finalized and merged.  A crash will have the log
- * btree merged after it is fenced.
+ * latency.
  *
- * Once we find candidate orphan items we can first check our local
- * inode cache for inodes that are already on their way to eviction and
- * can be skipped.  Then we ask the server for the open map containing
- * the inode.  Only if we don't have it cached, and no one else does, do
- * we try and read it into our cache and evict it to trigger the final
- * inode deletion process.
+ * Once we find candidate orphan items we first check our local omap for
+ * a locally cached inode.  Then we ask the server for the open map
+ * containing the inode.  Only if we don't see any cached users do we do
+ * the expensive work of acquiring locks to try and delete the items.
  */
 static void inode_orphan_scan_worker(struct work_struct *work)
 {
@@ -1791,7 +1906,6 @@ static void inode_orphan_scan_worker(struct work_struct *work)
 	SCOUTFS_BTREE_ITEM_REF(iref);
 	struct scoutfs_key last;
 	struct scoutfs_key key;
-	struct inode *inode;
 	u64 group_nr;
 	int bit_nr;
 	u64 ino;
@@ -1830,17 +1944,14 @@ static void inode_orphan_scan_worker(struct work_struct *work)
 		scoutfs_inc_counter(sb, orphan_scan_item);
 		ino = le64_to_cpu(key.sko_ino);
 
-		/* locally cached inodes will already be deleted */
-		inode = scoutfs_ilookup(sb, ino);
-		if (inode) {
+		/* locally cached inodes will try to delete as they evict */
+		if (scoutfs_omap_test(sb, ino)) {
 			scoutfs_inc_counter(sb, orphan_scan_cached);
-			iput(inode);
 			continue;
 		}
 
 		/* get an omap that covers the orphaned ino */
-		group_nr = ino >> SCOUTFS_OPEN_INO_MAP_SHIFT;
-		bit_nr = ino & SCOUTFS_OPEN_INO_MAP_MASK;
+		scoutfs_omap_calc_group_nrs(ino, &group_nr, &bit_nr);
 
 		if (le64_to_cpu(omap.args.group_nr) != group_nr) {
 			ret = scoutfs_client_open_ino_map(sb, group_nr, &omap);
@@ -1848,25 +1959,15 @@ static void inode_orphan_scan_worker(struct work_struct *work)
 				goto out;
 		}
 
-		/* don't need to evict if someone else has it open (cached) */
+		/* remote cached inodes will also try to delete */
 		if (test_bit_le(bit_nr, omap.bits)) {
 			scoutfs_inc_counter(sb, orphan_scan_omap_set);
 			continue;
 		}
 
-		/* try to cached and evict unused inode to delete, can be racing */
-		inode = scoutfs_iget(sb, ino, 0);
-		if (IS_ERR(inode)) {
-			ret = PTR_ERR(inode);
-			if (ret == -ENOENT)
-				continue;
-			else
-				goto out;
-		}
-
-		scoutfs_inc_counter(sb, orphan_scan_read);
-		SCOUTFS_I(inode)->drop_invalidated = true;
-		iput(inode);
+		/* seemingly orphaned and unused, get locks and check for sure */
+		scoutfs_inc_counter(sb, orphan_scan_attempts);
+		ret = try_delete_inode_items(sb, ino);
 	}
 
 	ret = 0;
@@ -1875,7 +1976,7 @@ out:
 	if (ret < 0)
 		scoutfs_inc_counter(sb, orphan_scan_error);
 
-	schedule_orphan_dwork(inf);
+	scoutfs_inode_schedule_orphan_dwork(sb);
 }
 
 /*
@@ -1983,8 +2084,6 @@ int scoutfs_inode_setup(struct super_block *sb)
 	spin_lock_init(&inf->dir_ino_alloc.lock);
 	spin_lock_init(&inf->ino_alloc.lock);
 	INIT_DELAYED_WORK(&inf->orphan_scan_dwork, inode_orphan_scan_worker);
-	spin_lock_init(&inf->deleting_items_lock);
-	INIT_LIST_HEAD(&inf->deleting_items_list);
 	INIT_WORK(&inf->iput_work, iput_worker);
 	init_llist_head(&inf->iput_llist);
 
@@ -2000,9 +2099,7 @@ int scoutfs_inode_setup(struct super_block *sb)
  */
 void scoutfs_inode_start(struct super_block *sb)
 {
-	DECLARE_INODE_SB_INFO(sb, inf);
-
-	schedule_orphan_dwork(inf);
+	scoutfs_inode_schedule_orphan_dwork(sb);
 }
 
 /*

kmod/src/inode.h

@@ -80,9 +80,13 @@ int scoutfs_drop_inode(struct inode *inode);
 void scoutfs_evict_inode(struct inode *inode);
 void scoutfs_inode_queue_iput(struct inode *inode);
 
-struct inode *scoutfs_iget(struct super_block *sb, u64 ino, int lkf);
-struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino);
+#define SCOUTFS_IGF_LINKED (1 << 0) /* enoent if nlink == 0 */
+struct inode *scoutfs_iget(struct super_block *sb, u64 ino, int lkf, int igf);
+struct inode *scoutfs_ilookup_nowait(struct super_block *sb, u64 ino);
+struct inode *scoutfs_ilookup_nowait_nonewfree(struct super_block *sb, u64 ino);
 
+
+void scoutfs_inode_init_key(struct scoutfs_key *key, u64 ino);
 void scoutfs_inode_init_index_key(struct scoutfs_key *key, u8 type, u64 major,
 				  u32 minor, u64 ino);
 int scoutfs_inode_index_start(struct super_block *sb, u64 *seq);
@@ -102,9 +106,8 @@ void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock,
 			       struct list_head *ind_locks);
 
 int scoutfs_alloc_ino(struct super_block *sb, bool is_dir, u64 *ino_ret);
-struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir,
-				umode_t mode, dev_t rdev, u64 ino,
-				struct scoutfs_lock *lock);
+int scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev,
+		      u64 ino, struct scoutfs_lock *lock, struct inode **inode_ret);
 
 void scoutfs_inode_set_meta_seq(struct inode *inode);
 void scoutfs_inode_set_data_seq(struct inode *inode);
@@ -117,14 +120,14 @@ u64 scoutfs_inode_data_version(struct inode *inode);
 void scoutfs_inode_get_onoff(struct inode *inode, s64 *on, s64 *off);
 int scoutfs_complete_truncate(struct inode *inode, struct scoutfs_lock *lock);
 
-int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock,
-			  int flags);
+int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock);
 int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry,
 		    struct kstat *stat);
 int scoutfs_setattr(struct dentry *dentry, struct iattr *attr);
 
 int scoutfs_inode_orphan_create(struct super_block *sb, u64 ino, struct scoutfs_lock *lock);
 int scoutfs_inode_orphan_delete(struct super_block *sb, u64 ino, struct scoutfs_lock *lock);
+void scoutfs_inode_schedule_orphan_dwork(struct super_block *sb);
 
 void scoutfs_inode_queue_writeback(struct inode *inode);
 int scoutfs_inode_walk_writeback(struct super_block *sb, bool write);

kmod/src/ioctl.c

@@ -387,7 +387,7 @@ static long scoutfs_ioc_data_wait_err(struct file *file, unsigned long arg)
 	if (sblock > eblock)
 		return -EINVAL;
 
-	inode = scoutfs_ilookup(sb, args.ino);
+	inode = scoutfs_ilookup_nowait_nonewfree(sb, args.ino);
 	if (!inode) {
 		ret = -ESTALE;
 		goto out;
@@ -1320,6 +1320,84 @@ out:
 	return ret ?: count;
 }
 
+static long scoutfs_ioc_get_allocated_inos(struct file *file, unsigned long arg)
+{
+	struct super_block *sb = file_inode(file)->i_sb;
+	struct scoutfs_ioctl_get_allocated_inos __user *ugai = (void __user *)arg;
+	struct scoutfs_ioctl_get_allocated_inos gai;
+	struct scoutfs_lock *lock = NULL;
+	struct scoutfs_key key;
+	struct scoutfs_key end;
+	u64 __user *uinos;
+	u64 bytes;
+	u64 ino;
+	int nr;
+	int ret;
+
+	if (!(file->f_mode & FMODE_READ)) {
+		ret = -EBADF;
+		goto out;
+	}
+
+	if (!capable(CAP_SYS_ADMIN)) {
+		ret = -EPERM;
+		goto out;
+	}
+
+	if (copy_from_user(&gai, ugai, sizeof(gai))) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	if ((gai.inos_ptr & (sizeof(__u64) - 1)) || (gai.inos_bytes < sizeof(__u64))) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	scoutfs_inode_init_key(&key, gai.start_ino);
+	scoutfs_inode_init_key(&end, gai.start_ino | SCOUTFS_LOCK_INODE_GROUP_MASK);
+	uinos = (void __user *)gai.inos_ptr;
+	bytes = gai.inos_bytes;
+	nr = 0;
+
+	ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_READ, 0, gai.start_ino, &lock);
+	if (ret < 0)
+		goto out;
+
+	while (bytes >= sizeof(*uinos)) {
+
+		ret = scoutfs_item_next(sb, &key, &end, NULL, 0, lock);
+		if (ret < 0) {
+			if (ret == -ENOENT)
+				ret = 0;
+			break;
+		}
+
+		if (key.sk_zone != SCOUTFS_FS_ZONE) {
+			ret = 0;
+			break;
+		}
+
+		/* all fs items are owned by allocated inodes, and _first is always ino */
+		ino = le64_to_cpu(key._sk_first);
+		if (put_user(ino, uinos)) {
+			ret = -EFAULT;
+			break;
+		}
+
+		uinos++;
+		bytes -= sizeof(*uinos);
+		if (++nr == INT_MAX)
+			break;
+
+		scoutfs_inode_init_key(&key, ino + 1);
+	}
+
+	scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ);
+out:
+	return ret ?: nr;
+}
+
 long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
 	switch (cmd) {
@@ -1353,6 +1431,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 		return scoutfs_ioc_resize_devices(file, arg);
 	case SCOUTFS_IOC_READ_XATTR_TOTALS:
 		return scoutfs_ioc_read_xattr_totals(file, arg);
+	case SCOUTFS_IOC_GET_ALLOCATED_INOS:
+		return scoutfs_ioc_get_allocated_inos(file, arg);
 	}
 
 	return -ENOTTY;

kmod/src/ioctl.h

@@ -520,4 +520,43 @@ struct scoutfs_ioctl_xattr_total {
 #define SCOUTFS_IOC_READ_XATTR_TOTALS \
 	_IOW(SCOUTFS_IOCTL_MAGIC, 15, struct scoutfs_ioctl_read_xattr_totals)
 
+/*
+ * This fills the caller's inos array with inode numbers that are in use
+ * after the start ino, within an internal inode group.
+ *
+ * This only makes a promise about the state of the inode numbers within
+ * the first and last numbers returned by one call.  At one time, all of
+ * those inodes were still allocated.   They could have changed before
+ * the call returned.   And any numbers outside of the first and last
+ * (or single) are undefined.
+ *
+ * This doesn't iterate over all allocated inodes, it only probes a
+ * single group that the start inode is within.   This interface was
+ * first introduced to support tests that needed to find out about a
+ * specific inode, while having some other similarly niche uses.   It is
+ * unsuitable for a consistent iteration over all the inode numbers in
+ * use.
+ *
+ * This test of inode items doesn't serialize with the inode lifetime
+ * mechanism.   It only tells you the numbers of inodes that were once
+ * active in the system and haven't yet been fully deleted.  The inode
+ * numbers returned could have been in the process of being deleted and
+ * were already unreachable even before the call started.
+ *
+ * @start_ino: the first inode number that could be returned
+ * @inos_ptr: pointer to an aligned array of 64bit inode numbers
+ * @inos_bytes: the number of bytes available in the inos_ptr array
+ *
+ * Returns errors or the count of inode numbers returned, quite possibly
+ * including 0.
+ */
+struct scoutfs_ioctl_get_allocated_inos {
+	__u64 start_ino;
+	__u64 inos_ptr;
+	__u64 inos_bytes;
+};
+
+#define SCOUTFS_IOC_GET_ALLOCATED_INOS \
+	_IOW(SCOUTFS_IOCTL_MAGIC, 16, struct scoutfs_ioctl_get_allocated_inos)
+
 #endif

kmod/src/item.c

@@ -685,6 +685,12 @@ static void erase_page_items(struct cached_page *pg,
  * to the dirty list after the left page, and by adding items to the
  * tail of right's dirty list in key sort order.
  *
+ * The max_seq of the source page might be larger than all the items
+ * while protecting an erased item from being reclaimed while an older
+ * read is in flight.  We don't know where it might be in the source
+ * page so we have to assume that it's in the key range being moved and
+ * update the destination page's max_seq accordingly.
+ *
  * The caller is responsible for page locking and managing the lru.
  */
 static void move_page_items(struct super_block *sb,
@@ -726,6 +732,9 @@ static void move_page_items(struct super_block *sb,
 
 		erase_item(left, from);
 	}
+
+	if (left->max_seq > right->max_seq)
+		right->max_seq = left->max_seq;
 }
 
 enum page_intersection_type {

kmod/src/kernelcompat.c

@@ -0,0 +1,23 @@
+
+#include "kernelcompat.h"
+
+#ifdef KC_SHRINKER_SHRINK
+#include <linux/shrinker.h>
+/*
+ * If a target doesn't have that .{count,scan}_objects() interface then
+ * we have a .shrink() helper that performs the shrink work in terms of
+ * count/scan.
+ */
+int kc_shrink_wrapper(struct shrinker *shrink, struct shrink_control *sc)
+{
+	struct kc_shrinker_funcs *funcs = KC_SHRINKER_FUNCS(shrink);
+	unsigned long nr;
+
+	if (sc->nr_to_scan != 0)
+		funcs->scan_objects(shrink, sc);
+
+	nr = funcs->count_objects(shrink, sc);
+
+	return min_t(unsigned long, nr, INT_MAX);
+}
+#endif

kmod/src/kernelcompat.h

@@ -46,4 +46,81 @@ static inline int dir_emit_dots(struct file *file, void *dirent,
 }
 #endif
 
+#ifndef KC_DIR_EMIT_DOTS
+#define percpu_counter_add_batch __percpu_counter_add
+#endif
+
+#ifndef KC_MEMALLOC_NOFS_SAVE
+#define memalloc_nofs_save memalloc_noio_save
+#define memalloc_nofs_restore memalloc_noio_restore
+#endif
+
+#ifdef KC_BIO_BI_OPF
+#define kc_bio_get_opf(bio)		\
+({					\
+	(bio)->bi_opf;			\
+})
+#define kc_bio_set_opf(bio, opf)	\
+do {					\
+	(bio)->bi_opf = opf;		\
+} while (0)
+#define kc_bio_set_sector(bio, sect)	\
+do {					\
+	(bio)->bi_iter.bi_sector = sect;\
+} while (0)
+#else
+#define kc_bio_get_opf(bio)		\
+({					\
+	(bio)->bi_rw;			\
+})
+#define kc_bio_set_opf(bio, opf)	\
+do {					\
+	(bio)->bio_rw = opf;		\
+} while (0)
+#define kc_bio_set_sector(bio, sect)	\
+do {					\
+	(bio)->bi_sector = sect;	\
+} while (0)
+#endif
+
+#ifdef KC_BIO_BI_STATUS
+#define KC_DECLARE_BIO_END_IO(name, bio)	name(bio)
+#define kc_bio_get_errno(bio)			({ blk_status_to_errno((bio)->bi_status); })
+#else
+#define KC_DECLARE_BIO_END_IO(name, bio)	name(bio, int _error_arg)
+#define kc_bio_get_errno(bio)			({ (int)((void)(bio), _error_arg); })
+#endif
+
+#ifndef KC_SHRINKER_SHRINK
+#define KC_DEFINE_SHRINKER(name) struct shrinker name
+#define KC_INIT_SHRINKER_FUNCS(type, name, shrink, count, scan) do {	\
+	__typeof__(shrink) _shrink = (shrink);				\
+	_shrink->count_objects = count;					\
+	_shrink->scan_objects = scan;					\
+} while (0)
+#else
+#include <linux/shrinker.h>
+struct kc_shrinker_funcs {
+	unsigned long (*count_objects)(struct shrinker *, struct shrink_control *sc);
+	unsigned long (*scan_objects)(struct shrinker *, struct shrink_control *sc);
+};
+/* using adjacent member of an unnamed struct */
+#define KC_DEFINE_SHRINKER(name)				\
+	{							\
+		struct kc_shrinker_funcs shrinker_funcs;	\
+		struct shinker name;				\
+	}
+#define KC_SHRINKER_FUNCS(shrinker) \
+	((void *)((long)(shrink) - sizeof(struct kc_shrinker_funcs)))
+#define KC_INIT_SHRINKER_FUNCS(type, name, shrink, count, scan) do {				\
+	BUILD_BUG_ON(offsetof(cont, shrink_funcs) + sizeof(struct kc_shrinker_funcs)) !=	\
+		     offsetof(cont, name) + sizeof(struct kc_shrinker_funcs);			\
+	struct kc_shrinker_funcs *_funcs = KC_SHRINKER_FUNCS(shrink)				\
+	__typeof__(shrink) _shrink = (shrink);							\
+	_funcs->count_objects = count;								\
+	_funcs->scan_objects = scan;								\
+	_shrink->shrink = kc_shrink_wrapper;							\
+} while (0)
+#endif
+
 #endif

kmod/src/lock.c

@@ -142,7 +142,7 @@ static void invalidate_inode(struct super_block *sb, u64 ino)
 	struct scoutfs_inode_info *si;
 	struct inode *inode;
 
-	inode = scoutfs_ilookup(sb, ino);
+	inode = scoutfs_ilookup_nowait_nonewfree(sb, ino);
 	if (inode) {
 		si = SCOUTFS_I(inode);
 
@@ -255,7 +255,7 @@ static void lock_free(struct lock_info *linfo, struct scoutfs_lock *lock)
 	BUG_ON(!list_empty(&lock->shrink_head));
 	BUG_ON(!list_empty(&lock->cov_list));
 
-	scoutfs_omap_free_lock_data(lock->omap_data);
+	kfree(lock->inode_deletion_data);
 	kfree(lock);
 }
 
@@ -289,9 +289,9 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb,
 	lock->sb = sb;
 	init_waitqueue_head(&lock->waitq);
 	lock->mode = SCOUTFS_LOCK_NULL;
+	lock->invalidating_mode = SCOUTFS_LOCK_NULL;
 
 	atomic64_set(&lock->forest_bloom_nr, 0);
-	spin_lock_init(&lock->omap_spinlock);
 
 	trace_scoutfs_lock_alloc(sb, lock);
 
@@ -667,7 +667,9 @@ struct inv_req {
  *
  * Before we start invalidating the lock we set the lock to the new
  * mode, preventing further incompatible users of the old mode from
- * using the lock while we're invalidating.
+ * using the lock while we're invalidating.  We record the previously
+ * granted mode so that we can send lock recover responses with the old
+ * granted mode during invalidation.
  */
 static void lock_invalidate_worker(struct work_struct *work)
 {
@@ -692,7 +694,8 @@ static void lock_invalidate_worker(struct work_struct *work)
 		if (!lock_counts_match(nl->new_mode, lock->users))
 			continue;
 
-		/* set the new mode, no incompatible users during inval */
+		/* set the new mode, no incompatible users during inval, recov needs old */
+		lock->invalidating_mode = lock->mode;
 		lock->mode = nl->new_mode;
 
 		/* move everyone that's ready to our private list */
@@ -735,6 +738,8 @@ static void lock_invalidate_worker(struct work_struct *work)
 		list_del(&ireq->head);
 		kfree(ireq);
 
+		lock->invalidating_mode = SCOUTFS_LOCK_NULL;
+
 		if (list_empty(&lock->inv_list)) {
 			/* finish if another request didn't arrive */
 			list_del_init(&lock->inv_head);
@@ -825,6 +830,7 @@ int scoutfs_lock_recover_request(struct super_block *sb, u64 net_id,
 {
 	DECLARE_LOCK_INFO(sb, linfo);
 	struct scoutfs_net_lock_recover *nlr;
+	enum scoutfs_lock_mode mode;
 	struct scoutfs_lock *lock;
 	struct scoutfs_lock *next;
 	struct rb_node *node;
@@ -845,10 +851,15 @@ int scoutfs_lock_recover_request(struct super_block *sb, u64 net_id,
 
 	for (i = 0; lock && i < SCOUTFS_NET_LOCK_MAX_RECOVER_NR; i++) {
 
+		if (lock->invalidating_mode != SCOUTFS_LOCK_NULL)
+			mode = lock->invalidating_mode;
+		else
+			mode = lock->mode;
+
 		nlr->locks[i].key = lock->start;
 		nlr->locks[i].write_seq = cpu_to_le64(lock->write_seq);
-		nlr->locks[i].old_mode = lock->mode;
-		nlr->locks[i].new_mode = lock->mode;
+		nlr->locks[i].old_mode = mode;
+		nlr->locks[i].new_mode = mode;
 
 		node = rb_next(&lock->node);
 		if (node)
@@ -1050,7 +1061,7 @@ int scoutfs_lock_inode(struct super_block *sb, enum scoutfs_lock_mode mode, int
 		goto out;
 
 	if (flags & SCOUTFS_LKF_REFRESH_INODE) {
-		ret = scoutfs_inode_refresh(inode, *lock, flags);
+		ret = scoutfs_inode_refresh(inode, *lock);
 		if (ret < 0) {
 			scoutfs_unlock(sb, *lock, mode);
 			*lock = NULL;

kmod/src/lock.h

@@ -11,7 +11,7 @@
 
 #define SCOUTFS_LOCK_NR_MODES		SCOUTFS_LOCK_INVALID
 
-struct scoutfs_omap_lock;
+struct inode_deletion_lock_data;
 
 /*
  * A few fields (start, end, refresh_gen, write_seq, granted_mode)
@@ -39,6 +39,7 @@ struct scoutfs_lock {
 	struct list_head cov_list;
 
 	enum scoutfs_lock_mode mode;
+	enum scoutfs_lock_mode invalidating_mode;
 	unsigned int waiters[SCOUTFS_LOCK_NR_MODES];
 	unsigned int users[SCOUTFS_LOCK_NR_MODES];
 
@@ -47,9 +48,8 @@ struct scoutfs_lock {
 	/* the forest tracks which log tree last saw bloom bit updates */
 	atomic64_t forest_bloom_nr;
 
-	/* open ino mapping has a valid map for a held write lock */
-	spinlock_t omap_spinlock;
-	struct scoutfs_omap_lock_data *omap_data;
+	/* inode deletion tracks some state per lock */
+	struct inode_deletion_lock_data *inode_deletion_data;
 };
 
 struct scoutfs_lock_coverage {

kmod/src/lock_server.c

@@ -153,30 +153,30 @@ enum {
  */
 static void add_client_entry(struct server_lock_node *snode,
 			     struct list_head *list,
-			     struct client_lock_entry *clent)
+			     struct client_lock_entry *c_ent)
 {
 	WARN_ON_ONCE(!mutex_is_locked(&snode->mutex));
 
-	if (list_empty(&clent->head))
-		list_add_tail(&clent->head, list);
+	if (list_empty(&c_ent->head))
+		list_add_tail(&c_ent->head, list);
 	else
-		list_move_tail(&clent->head, list);
+		list_move_tail(&c_ent->head, list);
 
-	clent->on_list = list == &snode->granted ? OL_GRANTED :
+	c_ent->on_list = list == &snode->granted ? OL_GRANTED :
 			 list == &snode->requested ? OL_REQUESTED :
 			 OL_INVALIDATED;
 }
 
 static void free_client_entry(struct lock_server_info *inf,
 			      struct server_lock_node *snode,
-			      struct client_lock_entry *clent)
+			      struct client_lock_entry *c_ent)
 {
 	WARN_ON_ONCE(!mutex_is_locked(&snode->mutex));
 
-	if (!list_empty(&clent->head))
-		list_del_init(&clent->head);
-	scoutfs_tseq_del(&inf->tseq_tree, &clent->tseq_entry);
-	kfree(clent);
+	if (!list_empty(&c_ent->head))
+		list_del_init(&c_ent->head);
+	scoutfs_tseq_del(&inf->tseq_tree, &c_ent->tseq_entry);
+	kfree(c_ent);
 }
 
 static bool invalid_mode(u8 mode)
@@ -339,13 +339,13 @@ static struct client_lock_entry *find_entry(struct server_lock_node *snode,
 					    struct list_head *list,
 					    u64 rid)
 {
-	struct client_lock_entry *clent;
+	struct client_lock_entry *c_ent;
 
 	WARN_ON_ONCE(!mutex_is_locked(&snode->mutex));
 
-	list_for_each_entry(clent, list, head) {
-		if (clent->rid == rid)
-			return clent;
+	list_for_each_entry(c_ent, list, head) {
+		if (c_ent->rid == rid)
+			return c_ent;
 	}
 
 	return NULL;
@@ -364,7 +364,7 @@ int scoutfs_lock_server_request(struct super_block *sb, u64 rid,
 				u64 net_id, struct scoutfs_net_lock *nl)
 {
 	DECLARE_LOCK_SERVER_INFO(sb, inf);
-	struct client_lock_entry *clent;
+	struct client_lock_entry *c_ent;
 	struct server_lock_node *snode;
 	int ret;
 
@@ -376,29 +376,29 @@ int scoutfs_lock_server_request(struct super_block *sb, u64 rid,
 		goto out;
 	}
 
-	clent = kzalloc(sizeof(struct client_lock_entry), GFP_NOFS);
-	if (!clent) {
+	c_ent = kzalloc(sizeof(struct client_lock_entry), GFP_NOFS);
+	if (!c_ent) {
 		ret = -ENOMEM;
 		goto out;
 	}
 
-	INIT_LIST_HEAD(&clent->head);
-	clent->rid = rid;
-	clent->net_id = net_id;
-	clent->mode = nl->new_mode;
+	INIT_LIST_HEAD(&c_ent->head);
+	c_ent->rid = rid;
+	c_ent->net_id = net_id;
+	c_ent->mode = nl->new_mode;
 
 	snode = alloc_server_lock(inf, &nl->key);
 	if (snode == NULL) {
-		kfree(clent);
+		kfree(c_ent);
 		ret = -ENOMEM;
 		goto out;
 	}
 
 	snode->stats[SLT_REQUEST]++;
 
-	clent->snode = snode;
-	add_client_entry(snode, &snode->requested, clent);
-	scoutfs_tseq_add(&inf->tseq_tree, &clent->tseq_entry);
+	c_ent->snode = snode;
+	add_client_entry(snode, &snode->requested, c_ent);
+	scoutfs_tseq_add(&inf->tseq_tree, &c_ent->tseq_entry);
 
 	ret = process_waiting_requests(sb, snode);
 out:
@@ -417,7 +417,7 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 rid,
 				 struct scoutfs_net_lock *nl)
 {
 	DECLARE_LOCK_SERVER_INFO(sb, inf);
-	struct client_lock_entry *clent;
+	struct client_lock_entry *c_ent;
 	struct server_lock_node *snode;
 	int ret;
 
@@ -438,18 +438,18 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 rid,
 
 	snode->stats[SLT_RESPONSE]++;
 
-	clent = find_entry(snode, &snode->invalidated, rid);
-	if (!clent) {
+	c_ent = find_entry(snode, &snode->invalidated, rid);
+	if (!c_ent) {
 		put_server_lock(inf, snode);
 		ret = -EINVAL;
 		goto out;
 	}
 
 	if (nl->new_mode == SCOUTFS_LOCK_NULL) {
-		free_client_entry(inf, snode, clent);
+		free_client_entry(inf, snode, c_ent);
 	} else {
-		clent->mode = nl->new_mode;
-		add_client_entry(snode, &snode->granted, clent);
+		c_ent->mode = nl->new_mode;
+		add_client_entry(snode, &snode->granted, c_ent);
 	}
 
 	ret = process_waiting_requests(sb, snode);
@@ -632,7 +632,7 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid,
 {
 	DECLARE_LOCK_SERVER_INFO(sb, inf);
 	struct client_lock_entry *existing;
-	struct client_lock_entry *clent;
+	struct client_lock_entry *c_ent;
 	struct server_lock_node *snode;
 	struct scoutfs_key key;
 	int ret = 0;
@@ -652,35 +652,35 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid,
 	}
 
 	for (i = 0; i < le16_to_cpu(nlr->nr); i++) {
-		clent = kzalloc(sizeof(struct client_lock_entry), GFP_NOFS);
-		if (!clent) {
+		c_ent = kzalloc(sizeof(struct client_lock_entry), GFP_NOFS);
+		if (!c_ent) {
 			ret = -ENOMEM;
 			goto out;
 		}
 
-		INIT_LIST_HEAD(&clent->head);
-		clent->rid = rid;
-		clent->net_id = 0;
-		clent->mode = nlr->locks[i].new_mode;
+		INIT_LIST_HEAD(&c_ent->head);
+		c_ent->rid = rid;
+		c_ent->net_id = 0;
+		c_ent->mode = nlr->locks[i].new_mode;
 
 		snode = alloc_server_lock(inf, &nlr->locks[i].key);
 		if (snode == NULL) {
-			kfree(clent);
+			kfree(c_ent);
 			ret = -ENOMEM;
 			goto out;
 		}
 
 		existing = find_entry(snode, &snode->granted, rid);
 		if (existing) {
-			kfree(clent);
+			kfree(c_ent);
 			put_server_lock(inf, snode);
 			ret = -EEXIST;
 			goto out;
 		}
 
-		clent->snode = snode;
-		add_client_entry(snode, &snode->granted, clent);
-		scoutfs_tseq_add(&inf->tseq_tree, &clent->tseq_entry);
+		c_ent->snode = snode;
+		add_client_entry(snode, &snode->granted, c_ent);
+		scoutfs_tseq_add(&inf->tseq_tree, &c_ent->tseq_entry);
 
 		put_server_lock(inf, snode);
 
@@ -707,7 +707,7 @@ out:
 int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid)
 {
 	DECLARE_LOCK_SERVER_INFO(sb, inf);
-	struct client_lock_entry *clent;
+	struct client_lock_entry *c_ent;
 	struct client_lock_entry *tmp;
 	struct server_lock_node *snode;
 	struct scoutfs_key key;
@@ -724,9 +724,9 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid)
 			    (list == &snode->requested) ? &snode->invalidated :
 			    NULL) {
 
-			list_for_each_entry_safe(clent, tmp, list, head) {
-				if (clent->rid == rid) {
-					free_client_entry(inf, snode, clent);
+			list_for_each_entry_safe(c_ent, tmp, list, head) {
+				if (c_ent->rid == rid) {
+					free_client_entry(inf, snode, c_ent);
 					freed = true;
 				}
 			}
@@ -749,7 +749,7 @@ out:
 	if (ret < 0) {
 		scoutfs_err(sb, "lock server err %d during client rid %016llx farewell, shutting down",
 			    ret, rid);
-		scoutfs_server_abort(sb);
+		scoutfs_server_stop(sb);
 	}
 
 	return ret;
@@ -787,15 +787,15 @@ static char *lock_on_list_string(u8 on_list)
 static void lock_server_tseq_show(struct seq_file *m,
 				  struct scoutfs_tseq_entry *ent)
 {
-	struct client_lock_entry *clent = container_of(ent,
+	struct client_lock_entry *c_ent = container_of(ent,
 						       struct client_lock_entry,
 						       tseq_entry);
-	struct server_lock_node *snode = clent->snode;
+	struct server_lock_node *snode = c_ent->snode;
 
 	seq_printf(m, SK_FMT" %s %s rid %016llx net_id %llu\n",
-		   SK_ARG(&snode->key), lock_mode_string(clent->mode),
-		   lock_on_list_string(clent->on_list), clent->rid,
-		   clent->net_id);
+		   SK_ARG(&snode->key), lock_mode_string(c_ent->mode),
+		   lock_on_list_string(c_ent->on_list), c_ent->rid,
+		   c_ent->net_id);
 }
 
 static void stats_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent)
@@ -857,7 +857,7 @@ void scoutfs_lock_server_destroy(struct super_block *sb)
 	DECLARE_LOCK_SERVER_INFO(sb, inf);
 	struct server_lock_node *snode;
 	struct server_lock_node *stmp;
-	struct client_lock_entry *clent;
+	struct client_lock_entry *c_ent;
 	struct client_lock_entry *ctmp;
 	LIST_HEAD(list);
 
@@ -873,8 +873,8 @@ void scoutfs_lock_server_destroy(struct super_block *sb)
 			list_splice_init(&snode->invalidated, &list);
 
 			mutex_lock(&snode->mutex);
-			list_for_each_entry_safe(clent, ctmp, &list, head) {
-				free_client_entry(inf, snode, clent);
+			list_for_each_entry_safe(c_ent, ctmp, &list, head) {
+				free_client_entry(inf, snode, c_ent);
 			}
 			mutex_unlock(&snode->mutex);
 

kmod/src/net.c

@@ -355,6 +355,7 @@ static int submit_send(struct super_block *sb,
 		}
 		if (rid != 0) {
 			spin_unlock(&conn->lock);
+			kfree(msend);
 			return -ENOTCONN;
 		}
 	}
@@ -835,17 +836,9 @@ static void scoutfs_net_destroy_worker(struct work_struct *work)
 	if (conn->listening_conn && conn->notify_down)
 		conn->notify_down(sb, conn, conn->info, conn->rid);
 
-	/*
-	 * Usually networking is idle and we destroy pending sends, but when forcing unmount
-	 * we can have to wake up waiters by failing pending sends.
-	 */
 	list_splice_init(&conn->resend_queue, &conn->send_queue);
-	list_for_each_entry_safe(msend, tmp, &conn->send_queue, head) {
-		if (scoutfs_forcing_unmount(sb))
-			call_resp_func(sb, conn, msend->resp_func, msend->resp_data,
-				       NULL, 0, -ECONNABORTED);
+	list_for_each_entry_safe(msend, tmp, &conn->send_queue, head)
 		free_msend(ninf, msend);
-	}
 
 	/* accepted sockets are removed from their listener's list */
 	if (conn->listening_conn) {
@@ -999,6 +992,8 @@ static void scoutfs_net_listen_worker(struct work_struct *work)
 		if (ret < 0)
 			break;
 
+		acc_sock->sk->sk_allocation = GFP_NOFS;
+
 		/* inherit accepted request funcs from listening conn */
 		acc_conn = scoutfs_net_alloc_conn(sb, conn->notify_up,
 						  conn->notify_down,
@@ -1061,6 +1056,8 @@ static void scoutfs_net_connect_worker(struct work_struct *work)
 	if (ret)
 		goto out;
 
+	sock->sk->sk_allocation = GFP_NOFS;
+
 	/* caller specified connect timeout */
 	tv.tv_sec = conn->connect_timeout_ms / MSEC_PER_SEC;
 	tv.tv_usec = (conn->connect_timeout_ms % MSEC_PER_SEC) * USEC_PER_MSEC;
@@ -1134,9 +1131,11 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work)
 	struct net_info *ninf = SCOUTFS_SB(sb)->net_info;
 	struct scoutfs_net_connection *listener;
 	struct scoutfs_net_connection *acc_conn;
+	scoutfs_net_response_t resp_func;
 	struct message_send *msend;
 	struct message_send *tmp;
 	unsigned long delay;
+	void *resp_data;
 
 	trace_scoutfs_net_shutdown_work_enter(sb, 0, 0);
 	trace_scoutfs_conn_shutdown_start(conn);
@@ -1182,6 +1181,30 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work)
 	/* and wait for accepted conn shutdown work to finish */
 	wait_event(conn->waitq, empty_accepted_list(conn));
 
+	/*
+	 * Forced unmount will cause net submit to fail once it's
+	 * started and it calls shutdown to interrupt any previous
+	 * senders waiting for a response.   The response callbacks can
+	 * do quite a lot of work so we're careful to call them outside
+	 * the lock.
+	 */
+	if (scoutfs_forcing_unmount(sb)) {
+		spin_lock(&conn->lock);
+		list_splice_tail_init(&conn->send_queue, &conn->resend_queue);
+		while ((msend = list_first_entry_or_null(&conn->resend_queue,
+							struct message_send, head))) {
+			resp_func = msend->resp_func;
+			resp_data = msend->resp_data;
+			free_msend(ninf, msend);
+			spin_unlock(&conn->lock);
+
+			call_resp_func(sb, conn, resp_func, resp_data, NULL, 0, -ECONNABORTED);
+
+			spin_lock(&conn->lock);
+		}
+		spin_unlock(&conn->lock);
+	}
+
 	spin_lock(&conn->lock);
 
 	/* greetings aren't resent across sockets */
@@ -1274,7 +1297,7 @@ restart:
 				if (ret) {
 					scoutfs_err(sb, "client fence returned err %d, shutting down server",
 						    ret);
-					scoutfs_server_abort(sb);
+					scoutfs_server_stop(sb);
 				}
 			}
 			destroy_conn(acc);
@@ -1323,10 +1346,12 @@ scoutfs_net_alloc_conn(struct super_block *sb,
 	if (!conn)
 		return NULL;
 
-	conn->info = kzalloc(info_size, GFP_NOFS);
-	if (!conn->info) {
-		kfree(conn);
-		return NULL;
+	if (info_size) {
+		conn->info = kzalloc(info_size, GFP_NOFS);
+		if (!conn->info) {
+			kfree(conn);
+			return NULL;
+		}
 	}
 
 	conn->workq = alloc_workqueue("scoutfs_net_%s",
@@ -1432,6 +1457,8 @@ int scoutfs_net_bind(struct super_block *sb,
 	if (ret)
 		goto out;
 
+	sock->sk->sk_allocation = GFP_NOFS;
+
 	optval = 1;
 	ret = kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR,
 				(char *)&optval, sizeof(optval));
@@ -1754,23 +1781,6 @@ int scoutfs_net_response_node(struct super_block *sb,
 			   NULL, NULL, NULL);
 }
 
-/*
- * The response function that was submitted with the request is not
- * called if the request is canceled here.
- */
-void scoutfs_net_cancel_request(struct super_block *sb,
-				struct scoutfs_net_connection *conn,
-				u8 cmd, u64 id)
-{
-	struct message_send *msend;
-
-	spin_lock(&conn->lock);
-	msend = find_request(conn, cmd, id);
-	if (msend)
-		complete_send(conn, msend);
-	spin_unlock(&conn->lock);
-}
-
 struct sync_request_completion {
 	struct completion comp;
 	void *resp;

kmod/src/net.h

@@ -134,9 +134,6 @@ int scoutfs_net_submit_request_node(struct super_block *sb,
 				    u64 rid, u8 cmd, void *arg, u16 arg_len,
 				    scoutfs_net_response_t resp_func,
 				    void *resp_data, u64 *id_ret);
-void scoutfs_net_cancel_request(struct super_block *sb,
-				struct scoutfs_net_connection *conn,
-				u8 cmd, u64 id);
 int scoutfs_net_sync_request(struct super_block *sb,
 			     struct scoutfs_net_connection *conn,
 			     u8 cmd, void *arg, unsigned arg_len,

kmod/src/omap.c

@@ -30,27 +30,22 @@
 /*
  * As a client removes an inode from its cache with an nlink of 0 it
  * needs to decide if it is the last client using the inode and should
- * fully delete all its items.  It needs to know if other mounts still
- * have the inode in use.
+ * fully delete all the inode's items.  It needs to know if other mounts
+ * still have the inode in use.
  *
- * We need a way to communicate between mounts that an inode is open.
+ * We need a way to communicate between mounts that an inode is in use.
  * We don't want to pay the synchronous per-file locking round trip
  * costs associated with per-inode open locks that you'd typically see
- * in systems to solve this problem.
+ * in systems to solve this problem.  The first prototypes of this
+ * tracked open file handles so this was coined the open map, though it
+ * now tracks cached inodes.
  *
- * Instead clients maintain open bitmaps that cover groups of inodes.
- * As inodes enter the cache their bit is set, and as the inode is
- * evicted the bit is cleared.  As an inode is evicted messages are sent
- * around the cluster to get the current bitmaps for that inode's group
- * from all active mounts.  If the inode's bit is clear then it can be
- * deleted.
- *
- * We associate the open bitmaps with our cluster locking of inode
- * groups to cache these open bitmaps.  As long as we have the lock then
- * nlink can't be changed on any remote mounts.  Specifically, it can't
- * increase from 0 so any clear bits can gain references on remote
- * mounts.  As long as we have the lock, all clear bits in the group for
- * inodes with 0 nlink can be deleted.
+ * Clients maintain bitmaps that cover groups of inodes.  As inodes
+ * enter the cache their bit is set and as the inode is evicted the bit
+ * is cleared.  As deletion is attempted, either by scanning orphans or
+ * evicting an inode with an nlink of 0, messages are sent around the
+ * cluster to get the current bitmaps for that inode's group from all
+ * active mounts.  If the inode's bit is clear then it can be deleted.
  *
  * This layer maintains a list of client rids to send messages to.  The
  * server calls us as clients enter and leave the cluster.    We can't
@@ -85,14 +80,12 @@ struct omap_info {
 	struct omap_info *name = SCOUTFS_SB(sb)->omap_info
 
 /*
- * The presence of an inode in the inode cache increases the count of
- * its inode number's position within its lock group.  These structs
- * track the counts for all the inodes in a lock group and maintain a
- * bitmap whose bits are set for each non-zero count.
+ * The presence of an inode in the inode sets its bit in the lock
+ * group's bitmap.
  *
  * We don't want to add additional global synchronization of inode cache
  * maintenance so these are tracked in an rcu hash table.  Once their
- * total count reaches zero they're removed from the hash and queued for
+ * total reaches zero they're removed from the hash and queued for
  * freeing and readers should ignore them.
  */
 struct omap_group {
@@ -102,7 +95,6 @@ struct omap_group {
 	u64 nr;
 	spinlock_t lock;
 	unsigned int total;
-	unsigned int *counts;
 	__le64 bits[SCOUTFS_OPEN_INO_MAP_LE64S];
 };
 
@@ -111,8 +103,7 @@ do {											\
 	__typeof__(group) _grp = (group);						\
 	__typeof__(bit_nr) _nr = (bit_nr);						\
 											\
-	trace_scoutfs_omap_group_##which(sb, _grp, _grp->nr, _grp->total, _nr,		\
-				        _nr < 0 ? -1 : _grp->counts[_nr]);		\
+	trace_scoutfs_omap_group_##which(sb, _grp, _grp->nr, _grp->total, _nr);		\
 } while (0)
 
 /*
@@ -134,18 +125,6 @@ struct omap_request {
 	struct scoutfs_open_ino_map map;
 };
 
-/*
- * In each inode group cluster lock we store data to track the open ino
- * map which tracks all the inodes that the cluster lock covers.  When
- * the seq shows that the map is stale we send a request to update it.
- */
-struct scoutfs_omap_lock_data {
-	u64 seq;
-	bool req_in_flight;
-	wait_queue_head_t waitq;
-	struct scoutfs_open_ino_map map;
-};
-
 static inline void init_rid_list(struct omap_rid_list *list)
 {
 	INIT_LIST_HEAD(&list->head);
@@ -178,6 +157,15 @@ static int free_rid(struct omap_rid_list *list, struct omap_rid_entry *entry)
 	return nr;
 }
 
+static void free_rid_list(struct omap_rid_list *list)
+{
+	struct omap_rid_entry *entry;
+	struct omap_rid_entry *tmp;
+
+	list_for_each_entry_safe(entry, tmp, &list->head, head)
+		free_rid(list, entry);
+}
+
 static int copy_rids(struct omap_rid_list *to, struct omap_rid_list *from, spinlock_t *from_lock)
 {
 	struct omap_rid_entry *entry;
@@ -232,7 +220,7 @@ static void free_rids(struct omap_rid_list *list)
 	}
 }
 
-static void calc_group_nrs(u64 ino, u64 *group_nr, int *bit_nr)
+void scoutfs_omap_calc_group_nrs(u64 ino, u64 *group_nr, int *bit_nr)
 {
 	*group_nr = ino >> SCOUTFS_OPEN_INO_MAP_SHIFT;
 	*bit_nr = ino & SCOUTFS_OPEN_INO_MAP_MASK;
@@ -242,21 +230,13 @@ static struct omap_group *alloc_group(struct super_block *sb, u64 group_nr)
 {
 	struct omap_group *group;
 
-	BUILD_BUG_ON((sizeof(group->counts[0]) * SCOUTFS_OPEN_INO_MAP_BITS) > PAGE_SIZE);
-
 	group = kzalloc(sizeof(struct omap_group), GFP_NOFS);
 	if (group) {
 		group->sb = sb;
 		group->nr = group_nr;
 		spin_lock_init(&group->lock);
 
-		group->counts = (void *)get_zeroed_page(GFP_NOFS);
-		if (!group->counts) {
-			kfree(group);
-			group = NULL;
-		} else {
-			trace_group(sb, alloc, group, -1);
-		}
+		trace_group(sb, alloc, group, -1);
 	}
 
 	return group;
@@ -265,7 +245,6 @@ static struct omap_group *alloc_group(struct super_block *sb, u64 group_nr)
 static void free_group(struct super_block *sb, struct omap_group *group)
 {
 	trace_group(sb, free, group, -1);
-	free_page((unsigned long)group->counts);
 	kfree(group);
 }
 
@@ -283,13 +262,16 @@ static const struct rhashtable_params group_ht_params = {
 };
 
 /*
- * Track an cached inode in its group.  Our increment can be racing with
- * a final decrement that removes the group from the hash, sets total to
+ * Track an cached inode in its group.  Our set can be racing with a
+ * final clear that removes the group from the hash, sets total to
  * UINT_MAX, and calls rcu free.  We can retry until the dead group is
  * no longer visible in the hash table and we can insert a new allocated
  * group.
+ *
+ * The caller must ensure that the bit is clear, -EEXIST will be
+ * returned otherwise.
  */
-int scoutfs_omap_inc(struct super_block *sb, u64 ino)
+int scoutfs_omap_set(struct super_block *sb, u64 ino)
 {
 	DECLARE_OMAP_INFO(sb, ominf);
 	struct omap_group *group;
@@ -298,7 +280,7 @@ int scoutfs_omap_inc(struct super_block *sb, u64 ino)
 	bool found;
 	int ret = 0;
 
-	calc_group_nrs(ino, &group_nr, &bit_nr);
+	scoutfs_omap_calc_group_nrs(ino, &group_nr, &bit_nr);
 
 retry:
 	found = false;
@@ -308,10 +290,10 @@ retry:
 		spin_lock(&group->lock);
 		if (group->total < UINT_MAX) {
 			found = true;
-			if (group->counts[bit_nr]++ == 0) {
-				set_bit_le(bit_nr, group->bits);
+			if (WARN_ON_ONCE(test_and_set_bit_le(bit_nr, group->bits)))
+				ret = -EEXIST;
+			else
 				group->total++;
-			}
 		}
 		trace_group(sb, inc, group, bit_nr);
 		spin_unlock(&group->lock);
@@ -342,29 +324,50 @@ retry:
 	return ret;
 }
 
+bool scoutfs_omap_test(struct super_block *sb, u64 ino)
+{
+	DECLARE_OMAP_INFO(sb, ominf);
+	struct omap_group *group;
+	bool ret = false;
+	u64 group_nr;
+	int bit_nr;
+
+	scoutfs_omap_calc_group_nrs(ino, &group_nr, &bit_nr);
+
+	rcu_read_lock();
+	group = rhashtable_lookup(&ominf->group_ht, &group_nr, group_ht_params);
+	if (group) {
+		spin_lock(&group->lock);
+		ret = !!test_bit_le(bit_nr, group->bits);
+		spin_unlock(&group->lock);
+	}
+	rcu_read_unlock();
+
+	return ret;
+}
+
 /*
- * Decrement a previously incremented ino count.  Not finding a count
- * implies imbalanced inc/dec or bugs freeing groups.  We only free
- * groups here as the last dec drops the group's total count to 0.
+ * Clear a previously set ino bit.  Trying to clear a bit that's already
+ * clear implies imbalanced set/clear or bugs freeing groups.  We only
+ * free groups here as the last clear drops the group's total to 0.
  */
-void scoutfs_omap_dec(struct super_block *sb, u64 ino)
+void scoutfs_omap_clear(struct super_block *sb, u64 ino)
 {
 	DECLARE_OMAP_INFO(sb, ominf);
 	struct omap_group *group;
 	u64 group_nr;
 	int bit_nr;
 
-	calc_group_nrs(ino, &group_nr, &bit_nr);
+	scoutfs_omap_calc_group_nrs(ino, &group_nr, &bit_nr);
 
 	rcu_read_lock();
 	group = rhashtable_lookup(&ominf->group_ht, &group_nr, group_ht_params);
 	if (group) {
 		spin_lock(&group->lock);
-		WARN_ON_ONCE(group->counts[bit_nr] == 0);
+		WARN_ON_ONCE(!test_bit_le(bit_nr, group->bits));
 		WARN_ON_ONCE(group->total == 0);
 		WARN_ON_ONCE(group->total == UINT_MAX);
-		if (--group->counts[bit_nr] == 0) {
-			clear_bit_le(bit_nr, group->bits);
+		if (test_and_clear_bit_le(bit_nr, group->bits)) {
 			if (--group->total == 0) {
 				group->total = UINT_MAX;
 				rhashtable_remove_fast(&ominf->group_ht, &group->ht_head,
@@ -664,8 +667,7 @@ int scoutfs_omap_server_handle_request(struct super_block *sb, u64 rid, u64 id,
 
 /*
  * The client is receiving a request from the server for its map for the
- * given group.  Look up the group and copy the bits to the map for
- * non-zero open counts.
+ * given group.  Look up the group and copy the bits to the map.
  *
  * The mount originating the request for this bitmap has the inode group
  * write locked.  We can't be adding links to any inodes in the group
@@ -811,182 +813,13 @@ void scoutfs_omap_server_shutdown(struct super_block *sb)
 	llist_for_each_entry_safe(req, tmp, requests, llnode)
 		kfree(req);
 
+	spin_lock(&ominf->lock);
+	free_rid_list(&ominf->rids);
+	spin_unlock(&ominf->lock);
+
 	synchronize_rcu();
 }
 
-static bool omap_req_in_flight(struct scoutfs_lock *lock, struct scoutfs_omap_lock_data *ldata)
-{
-	bool in_flight;
-
-	spin_lock(&lock->omap_spinlock);
-	in_flight = ldata->req_in_flight;
-	spin_unlock(&lock->omap_spinlock);
-
-	return in_flight;
-}
-
-/*
- * Make sure the map covered by the cluster lock is current.  The caller
- * holds the cluster lock so once we store lock_data on the cluster lock
- * it won't be freed and the write_seq in the cluster lock won't change.
- *
- * The omap_spinlock protects the omap_data in the cluster lock.  We
- * have to drop it if we have to block to allocate lock_data, send a
- * request for a new map, or wait for a request in flight to finish.
- */
-static int get_current_lock_data(struct super_block *sb, struct scoutfs_lock *lock,
-				 struct scoutfs_omap_lock_data **ldata_ret, u64 group_nr)
-{
-	struct scoutfs_omap_lock_data *ldata;
-	bool send_req;
-	int ret = 0;
-
-	spin_lock(&lock->omap_spinlock);
-
-	ldata = lock->omap_data;
-	if (ldata == NULL) {
-		spin_unlock(&lock->omap_spinlock);
-		ldata = kzalloc(sizeof(struct scoutfs_omap_lock_data), GFP_NOFS);
-		spin_lock(&lock->omap_spinlock);
-
-		if (!ldata) {
-			ret = -ENOMEM;
-			goto out;
-		}
-
-		if (lock->omap_data == NULL) {
-			ldata->seq = lock->write_seq - 1; /* ensure refresh */
-			init_waitqueue_head(&ldata->waitq);
-
-			lock->omap_data = ldata;
-		} else {
-			kfree(ldata);
-			ldata = lock->omap_data;
-		}
-	}
-
-	while (ldata->seq != lock->write_seq) {
-		/* only one waiter sends a request at a time */
-		if (!ldata->req_in_flight) {
-			ldata->req_in_flight = true;
-			send_req = true;
-		} else {
-			send_req = false;
-		}
-
-		spin_unlock(&lock->omap_spinlock);
-		if (send_req)
-			ret = scoutfs_client_open_ino_map(sb, group_nr, &ldata->map);
-		else
-			wait_event(ldata->waitq, !omap_req_in_flight(lock, ldata));
-		spin_lock(&lock->omap_spinlock);
-
-		/* only sender can return error, other waiters retry */
-		if (send_req) {
-			ldata->req_in_flight = false;
-			if (ret == 0)
-				ldata->seq = lock->write_seq;
-			wake_up(&ldata->waitq);
-			if (ret < 0)
-				goto out;
-		}
-	}
-
-out:
-	spin_unlock(&lock->omap_spinlock);
-
-	if (ret == 0)
-		*ldata_ret = ldata;
-	else
-		*ldata_ret = NULL;
-
-	return ret;
-}
-
-/*
- * Return 1 and give the caller their locks when they should delete the
- * inode items.  It's safe to delete the inode items when it is no
- * longer reachable and nothing is referencing it.
- *
- * The inode is unreachable when nlink hits zero.  Cluster locks protect
- * modification and testing of nlink.  We use the ino_lock_cov covrage
- * to short circuit the common case of having a locked inode that hasn't
- * been deleted.  If it isn't locked, we have to acquire the lock to
- * refresh the inode to see its current nlink. 
- *
- * Then we use an open inode bitmap that covers all the inodes in the
- * lock group to determine if the inode is present in any other mount's
- * caches.  We refresh it by asking the server for all clients' maps and
- * then store it in the lock.  As long as we hold the lock nothing can
- * increase nlink from zero and let people get a reference to the inode.
- */
-int scoutfs_omap_should_delete(struct super_block *sb, struct inode *inode,
-			       struct scoutfs_lock **lock_ret, struct scoutfs_lock **orph_lock_ret)
-{
-	struct scoutfs_inode_info *si = SCOUTFS_I(inode);
-	struct scoutfs_lock *orph_lock = NULL;
-	struct scoutfs_lock *lock = NULL;
-	const u64 ino = scoutfs_ino(inode);
-	struct scoutfs_omap_lock_data *ldata;
-	u64 group_nr;
-	int bit_nr;
-	int ret;
-	int err;
-
-	/* lock group and omap constants are defined independently */
-	BUILD_BUG_ON(SCOUTFS_OPEN_INO_MAP_BITS != SCOUTFS_LOCK_INODE_GROUP_NR);
-
-	if (scoutfs_lock_is_covered(sb, &si->ino_lock_cov) && inode->i_nlink > 0) {
-		ret = 0;
-		goto out;
-	}
-
-	ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, SCOUTFS_LKF_REFRESH_INODE, inode, &lock);
-	if (ret < 0)
-		goto out;
-
-	if (inode->i_nlink > 0) {
-		ret = 0;
-		goto out;
-	}
-
-	calc_group_nrs(ino, &group_nr, &bit_nr);
-
-	/* only one request to refresh the map at a time */
-	ret = get_current_lock_data(sb, lock, &ldata, group_nr);
-	if (ret < 0)
-		goto out;
-
-	/* can delete caller's zero nlink inode if it's not cached in other mounts */
-	ret = !test_bit_le(bit_nr, ldata->map.bits);
-out:
-	trace_scoutfs_omap_should_delete(sb, ino, inode->i_nlink, ret);
-
-	if (ret > 0) {
-		err = scoutfs_lock_orphan(sb, SCOUTFS_LOCK_WRITE_ONLY, 0, ino, &orph_lock);
-		if (err < 0)
-			ret = err;
-	}
-
-	if (ret <= 0) {
-		scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE);
-		lock = NULL;
-	}
-
-	*lock_ret = lock;
-	*orph_lock_ret = orph_lock;
-	return ret;
-}
-
-void scoutfs_omap_free_lock_data(struct scoutfs_omap_lock_data *ldata)
-{
-	if (ldata) {
-		WARN_ON_ONCE(ldata->req_in_flight);
-		WARN_ON_ONCE(waitqueue_active(&ldata->waitq));
-		kfree(ldata);
-	}
-}
-
 int scoutfs_omap_setup(struct super_block *sb)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
@@ -1044,6 +877,10 @@ void scoutfs_omap_destroy(struct super_block *sb)
 		rhashtable_walk_stop(&iter);
 		rhashtable_walk_exit(&iter);
 
+		spin_lock(&ominf->lock);
+		free_rid_list(&ominf->rids);
+		spin_unlock(&ominf->lock);
+
 		rhashtable_destroy(&ominf->group_ht);
 		rhashtable_destroy(&ominf->req_ht);
 		kfree(ominf);

kmod/src/omap.h

@@ -1,13 +1,12 @@
 #ifndef _SCOUTFS_OMAP_H_
 #define _SCOUTFS_OMAP_H_
 
-int scoutfs_omap_inc(struct super_block *sb, u64 ino);
-void scoutfs_omap_dec(struct super_block *sb, u64 ino);
-int scoutfs_omap_should_delete(struct super_block *sb, struct inode *inode,
-			       struct scoutfs_lock **lock_ret, struct scoutfs_lock **orph_lock_ret);
-void scoutfs_omap_free_lock_data(struct scoutfs_omap_lock_data *ldata);
+int scoutfs_omap_set(struct super_block *sb, u64 ino);
+bool scoutfs_omap_test(struct super_block *sb, u64 ino);
+void scoutfs_omap_clear(struct super_block *sb, u64 ino);
 int scoutfs_omap_client_handle_request(struct super_block *sb, u64 id,
 				       struct scoutfs_open_ino_map_args *args);
+void scoutfs_omap_calc_group_nrs(u64 ino, u64 *group_nr, int *bit_nr);
 
 int scoutfs_omap_add_rid(struct super_block *sb, u64 rid);
 int scoutfs_omap_remove_rid(struct super_block *sb, u64 rid);

kmod/src/options.c

@@ -26,22 +26,30 @@
 #include "msg.h"
 #include "options.h"
 #include "super.h"
+#include "inode.h"
+
+enum {
+	Opt_metadev_path,
+	Opt_orphan_scan_delay_ms,
+	Opt_quorum_slot_nr,
+	Opt_err,
+};
 
 static const match_table_t tokens = {
-	{Opt_quorum_slot_nr, "quorum_slot_nr=%s"},
 	{Opt_metadev_path, "metadev_path=%s"},
+	{Opt_orphan_scan_delay_ms, "orphan_scan_delay_ms=%s"},
+	{Opt_quorum_slot_nr, "quorum_slot_nr=%s"},
 	{Opt_err, NULL}
 };
 
-struct options_sb_info {
-	struct dentry *debugfs_dir;
+struct options_info {
+	seqlock_t seqlock;
+	struct scoutfs_mount_options opts;
+	struct scoutfs_sysfs_attrs sysfs_attrs;
 };
 
-u32 scoutfs_option_u32(struct super_block *sb, int token)
-{
-	WARN_ON_ONCE(1);
-	return 0;
-}
+#define DECLARE_OPTIONS_INFO(sb, name) \
+	struct options_info *name = SCOUTFS_SB(sb)->options_info
 
 static int parse_bdev_path(struct super_block *sb, substring_t *substr,
 			      char **bdev_path_ret)
@@ -89,8 +97,29 @@ out:
 	return ret;
 }
 
-int scoutfs_parse_options(struct super_block *sb, char *options,
-			  struct mount_options *parsed)
+static void free_options(struct scoutfs_mount_options *opts)
+{
+	kfree(opts->metadev_path);
+}
+
+#define MIN_ORPHAN_SCAN_DELAY_MS	100UL
+#define DEFAULT_ORPHAN_SCAN_DELAY_MS	(10 * MSEC_PER_SEC)
+#define MAX_ORPHAN_SCAN_DELAY_MS	(60 * MSEC_PER_SEC)
+
+static void init_default_options(struct scoutfs_mount_options *opts)
+{
+	memset(opts, 0, sizeof(*opts));
+	opts->quorum_slot_nr = -1;
+	opts->orphan_scan_delay_ms = DEFAULT_ORPHAN_SCAN_DELAY_MS;
+}
+
+/*
+ * Parse the option string into our options struct.   This can allocate
+ * memory in the struct.  The caller is responsible for always calling
+ * free_options() when the struct is destroyed, including when we return
+ * an error.
+ */
+static int parse_options(struct super_block *sb, char *options, struct scoutfs_mount_options *opts)
 {
 	substring_t args[MAX_OPT_ARGS];
 	int nr;
@@ -98,49 +127,61 @@ int scoutfs_parse_options(struct super_block *sb, char *options,
 	char *p;
 	int ret;
 
-	/* Set defaults */
-	memset(parsed, 0, sizeof(*parsed));
-	parsed->quorum_slot_nr = -1;
-
 	while ((p = strsep(&options, ",")) != NULL) {
 		if (!*p)
 			continue;
 
 		token = match_token(p, tokens, args);
 		switch (token) {
-		case Opt_quorum_slot_nr:
 
-			if (parsed->quorum_slot_nr != -1) {
+		case Opt_metadev_path:
+			ret = parse_bdev_path(sb, &args[0], &opts->metadev_path);
+			if (ret < 0)
+				return ret;
+			break;
+
+		case Opt_orphan_scan_delay_ms:
+			if (opts->orphan_scan_delay_ms != -1) {
+				scoutfs_err(sb, "multiple orphan_scan_delay_ms options provided, only provide one.");
+				return -EINVAL;
+			}
+
+			ret = match_int(args, &nr);
+			if (ret < 0 ||
+			    nr < MIN_ORPHAN_SCAN_DELAY_MS || nr > MAX_ORPHAN_SCAN_DELAY_MS) {
+				scoutfs_err(sb, "invalid orphan_scan_delay_ms option, must be between %lu and %lu",
+					    MIN_ORPHAN_SCAN_DELAY_MS, MAX_ORPHAN_SCAN_DELAY_MS);
+				if (ret == 0)
+					ret = -EINVAL;
+				return ret;
+			}
+			opts->orphan_scan_delay_ms = nr;
+			break;
+
+		case Opt_quorum_slot_nr:
+			if (opts->quorum_slot_nr != -1) {
 				scoutfs_err(sb, "multiple quorum_slot_nr options provided, only provide one.");
 				return -EINVAL;
 			}
 
 			ret = match_int(args, &nr);
-			if (ret < 0 || nr < 0 ||
-			    nr >= SCOUTFS_QUORUM_MAX_SLOTS) {
+			if (ret < 0 || nr < 0 || nr >= SCOUTFS_QUORUM_MAX_SLOTS) {
 				scoutfs_err(sb, "invalid quorum_slot_nr option, must be between 0 and %u",
 					    SCOUTFS_QUORUM_MAX_SLOTS - 1);
 				if (ret == 0)
 					ret = -EINVAL;
 				return ret;
 			}
-			parsed->quorum_slot_nr = nr;
+			opts->quorum_slot_nr = nr;
 			break;
-		case Opt_metadev_path:
 
-			ret = parse_bdev_path(sb, &args[0],
-						 &parsed->metadev_path);
-			if (ret < 0)
-				return ret;
-			break;
 		default:
-			scoutfs_err(sb, "Unknown or malformed option, \"%s\"",
-				    p);
-			break;
+			scoutfs_err(sb, "Unknown or malformed option, \"%s\"", p);
+			return -EINVAL;
 		}
 	}
 
-	if (!parsed->metadev_path) {
+	if (!opts->metadev_path) {
 		scoutfs_err(sb, "Required mount option \"metadev_path\" not found");
 		return -EINVAL;
 	}
@@ -148,40 +189,181 @@ int scoutfs_parse_options(struct super_block *sb, char *options,
 	return 0;
 }
 
-int scoutfs_options_setup(struct super_block *sb)
+void scoutfs_options_read(struct super_block *sb, struct scoutfs_mount_options *opts)
+{
+	DECLARE_OPTIONS_INFO(sb, optinf);
+	unsigned int seq;
+
+	if (WARN_ON_ONCE(optinf == NULL)) {
+		/* trying to use options before early setup or after destroy */
+		init_default_options(opts);
+		return;
+	}
+
+	do {
+		seq = read_seqbegin(&optinf->seqlock);
+		memcpy(opts, &optinf->opts, sizeof(struct scoutfs_mount_options));
+	} while (read_seqretry(&optinf->seqlock, seq));
+}
+
+/*
+ * Early setup that parses and stores the options so that the rest of
+ * setup can use them.   Full options setup that relies on other
+ * components will be done later.
+ */
+int scoutfs_options_early_setup(struct super_block *sb, char *options)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
-	struct options_sb_info *osi;
+	struct scoutfs_mount_options opts;
+	struct options_info *optinf;
 	int ret;
 
-	osi = kzalloc(sizeof(struct options_sb_info), GFP_KERNEL);
-	if (!osi)
-		return -ENOMEM;
+	init_default_options(&opts);
 
-	sbi->options = osi;
+	ret = parse_options(sb, options, &opts);
+	if (ret < 0)
+		goto out;
 
-	osi->debugfs_dir = debugfs_create_dir("options", sbi->debug_root);
-	if (!osi->debugfs_dir) {
+	optinf = kzalloc(sizeof(struct options_info), GFP_KERNEL);
+	if (!optinf) {
 		ret = -ENOMEM;
 		goto out;
 	}
 
+	seqlock_init(&optinf->seqlock);
+	scoutfs_sysfs_init_attrs(sb, &optinf->sysfs_attrs);
+
+	write_seqlock(&optinf->seqlock);
+	optinf->opts = opts;
+	write_sequnlock(&optinf->seqlock);
+
+	sbi->options_info = optinf;
 	ret = 0;
 out:
-	if (ret)
+	if (ret < 0)
+		free_options(&opts);
+
+	return ret;
+}
+
+int scoutfs_options_show(struct seq_file *seq, struct dentry *root)
+{
+	struct super_block *sb = root->d_sb;
+	struct scoutfs_mount_options opts;
+
+	scoutfs_options_read(sb, &opts);
+
+	seq_printf(seq, ",metadev_path=%s", opts.metadev_path);
+	seq_printf(seq, ",orphan_scan_delay_ms=%u", opts.orphan_scan_delay_ms);
+	if (opts.quorum_slot_nr >= 0)
+		seq_printf(seq, ",quorum_slot_nr=%d", opts.quorum_slot_nr);
+
+	return 0;
+}
+
+static ssize_t metadev_path_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj);
+	struct scoutfs_mount_options opts;
+
+	scoutfs_options_read(sb, &opts);
+
+	return snprintf(buf, PAGE_SIZE, "%s", opts.metadev_path);
+}
+SCOUTFS_ATTR_RO(metadev_path);
+
+static ssize_t orphan_scan_delay_ms_show(struct kobject *kobj, struct kobj_attribute *attr,
+					 char *buf)
+{
+	struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj);
+	struct scoutfs_mount_options opts;
+
+	scoutfs_options_read(sb, &opts);
+
+	return snprintf(buf, PAGE_SIZE, "%u", opts.orphan_scan_delay_ms);
+}
+static ssize_t orphan_scan_delay_ms_store(struct kobject *kobj, struct kobj_attribute *attr,
+					  const char *buf, size_t count)
+{
+	struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj);
+	DECLARE_OPTIONS_INFO(sb, optinf);
+	char nullterm[20]; /* more than enough for octal -U32_MAX */
+	long val;
+	int len;
+	int ret;
+
+	len = min(count, sizeof(nullterm) - 1);
+	memcpy(nullterm, buf, len);
+	nullterm[len] = '\0';
+
+	ret = kstrtol(nullterm, 0, &val);
+	if (ret < 0 || val < MIN_ORPHAN_SCAN_DELAY_MS || val > MAX_ORPHAN_SCAN_DELAY_MS) {
+		scoutfs_err(sb, "invalid orphan_scan_delay_ms value written to options sysfs file, must be between %lu and %lu",
+			    MIN_ORPHAN_SCAN_DELAY_MS, MAX_ORPHAN_SCAN_DELAY_MS);
+		return -EINVAL;
+	}
+
+	write_seqlock(&optinf->seqlock);
+	optinf->opts.orphan_scan_delay_ms = val;
+	write_sequnlock(&optinf->seqlock);
+
+	scoutfs_inode_schedule_orphan_dwork(sb);
+
+	return count;
+}
+SCOUTFS_ATTR_RW(orphan_scan_delay_ms);
+
+static ssize_t quorum_slot_nr_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj);
+	struct scoutfs_mount_options opts;
+
+	scoutfs_options_read(sb, &opts);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", opts.quorum_slot_nr);
+}
+SCOUTFS_ATTR_RO(quorum_slot_nr);
+
+static struct attribute *options_attrs[] = {
+	SCOUTFS_ATTR_PTR(metadev_path),
+	SCOUTFS_ATTR_PTR(orphan_scan_delay_ms),
+	SCOUTFS_ATTR_PTR(quorum_slot_nr),
+	NULL,
+};
+
+int scoutfs_options_setup(struct super_block *sb)
+{
+	DECLARE_OPTIONS_INFO(sb, optinf);
+	int ret;
+
+	ret = scoutfs_sysfs_create_attrs(sb, &optinf->sysfs_attrs, options_attrs, "mount_options");
+	if (ret < 0)
 		scoutfs_options_destroy(sb);
 	return ret;
 }
 
+/*
+ * We remove the sysfs files early in unmount so that they can't try to call other subsystems
+ * as they're being destroyed.
+ */
+void scoutfs_options_stop(struct super_block *sb)
+{
+	DECLARE_OPTIONS_INFO(sb, optinf);
+
+	if (optinf)
+		scoutfs_sysfs_destroy_attrs(sb, &optinf->sysfs_attrs);
+}
+
 void scoutfs_options_destroy(struct super_block *sb)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
-	struct options_sb_info *osi = sbi->options;
+	DECLARE_OPTIONS_INFO(sb, optinf);
 
-	if (osi) {
-		if (osi->debugfs_dir)
-			debugfs_remove_recursive(osi->debugfs_dir);
-		kfree(osi);
-		sbi->options = NULL;
+	scoutfs_options_stop(sb);
+
+	if (optinf) {
+		free_options(&optinf->opts);
+		kfree(optinf);
+		sbi->options_info = NULL;
 	}
 }

kmod/src/options.h

@@ -5,23 +5,19 @@
 #include <linux/in.h>
 #include "format.h"
 
-enum scoutfs_mount_options {
-	Opt_quorum_slot_nr,
-	Opt_metadev_path,
-	Opt_err,
-};
-
-struct mount_options {
-	int quorum_slot_nr;
+struct scoutfs_mount_options {
 	char *metadev_path;
+	unsigned int orphan_scan_delay_ms;
+	int quorum_slot_nr;
+
 };
 
-int scoutfs_parse_options(struct super_block *sb, char *options,
-			  struct mount_options *parsed);
+void scoutfs_options_read(struct super_block *sb, struct scoutfs_mount_options *opts);
+int scoutfs_options_show(struct seq_file *seq, struct dentry *root);
+
+int scoutfs_options_early_setup(struct super_block *sb, char *options);
 int scoutfs_options_setup(struct super_block *sb);
+void scoutfs_options_stop(struct super_block *sb);
 void scoutfs_options_destroy(struct super_block *sb);
 
-u32 scoutfs_option_u32(struct super_block *sb, int token);
-#define scoutfs_option_bool scoutfs_option_u32
-
 #endif	/* _SCOUTFS_OPTIONS_H_ */

kmod/src/quorum.c

@@ -105,6 +105,8 @@ enum quorum_role { FOLLOWER, CANDIDATE, LEADER };
 struct quorum_status {
 	enum quorum_role role;
 	u64 term;
+	u64 server_start_term;
+	int server_event;
 	int vote_for;
 	unsigned long vote_bits;
 	ktime_t timeout;
@@ -116,7 +118,7 @@ struct quorum_info {
 	struct socket *sock;
 	bool shutdown;
 
-	unsigned long flags;
+	int our_quorum_slot_nr;
 	int votes_needed;
 
 	spinlock_t show_lock;
@@ -127,8 +129,6 @@ struct quorum_info {
 	struct scoutfs_sysfs_attrs ssa;
 };
 
-#define QINF_FLAG_SERVER 0
-
 #define DECLARE_QUORUM_INFO(sb, name) \
 	struct quorum_info *name = SCOUTFS_SB(sb)->quorum_info
 #define DECLARE_QUORUM_INFO_KOBJ(kobj, name) \
@@ -160,9 +160,7 @@ static ktime_t heartbeat_timeout(void)
 static int create_socket(struct super_block *sb)
 {
 	DECLARE_QUORUM_INFO(sb, qinf);
-	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
-	struct mount_options *opts = &sbi->opts;
-	struct scoutfs_super_block *super = &sbi->super;
+	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
 	struct socket *sock = NULL;
 	struct sockaddr_in sin;
 	int addrlen;
@@ -176,7 +174,7 @@ static int create_socket(struct super_block *sb)
 
 	sock->sk->sk_allocation = GFP_NOFS;
 
-	scoutfs_quorum_slot_sin(super, opts->quorum_slot_nr, &sin);
+	scoutfs_quorum_slot_sin(super, qinf->our_quorum_slot_nr, &sin);
 
 	addrlen = sizeof(sin);
 	ret = kernel_bind(sock, (struct sockaddr *)&sin, addrlen);
@@ -207,7 +205,6 @@ static void send_msg_members(struct super_block *sb, int type, u64 term,
 			     int only)
 {
 	DECLARE_QUORUM_INFO(sb, qinf);
-	struct mount_options *opts = &SCOUTFS_SB(sb)->opts;
 	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
 	ktime_t now;
 	int i;
@@ -216,7 +213,7 @@ static void send_msg_members(struct super_block *sb, int type, u64 term,
 		.fsid = super->hdr.fsid,
 		.term = cpu_to_le64(term),
 		.type = type,
-		.from = opts->quorum_slot_nr,
+		.from = qinf->our_quorum_slot_nr,
 	};
 	struct kvec kv =  {
 		.iov_base = &qmes,
@@ -238,7 +235,7 @@ static void send_msg_members(struct super_block *sb, int type, u64 term,
 
 	for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
 		if (!quorum_slot_present(super, i) ||
-		    (only >= 0 && i != only) || i == opts->quorum_slot_nr)
+		    (only >= 0 && i != only) || i == qinf->our_quorum_slot_nr)
 			continue;
 
 		scoutfs_quorum_slot_sin(super, i, &sin);
@@ -476,8 +473,8 @@ static int write_quorum_block(struct super_block *sb, u64 blkno, struct scoutfs_
  */
 static int update_quorum_block(struct super_block *sb, int event, u64 term, bool check_rid)
 {
-	struct mount_options *opts = &SCOUTFS_SB(sb)->opts;
-	u64 blkno = SCOUTFS_QUORUM_BLKNO + opts->quorum_slot_nr;
+	DECLARE_QUORUM_INFO(sb, qinf);
+	u64 blkno = SCOUTFS_QUORUM_BLKNO + qinf->our_quorum_slot_nr;
 	struct scoutfs_quorum_block blk;
 	int ret;
 
@@ -496,16 +493,6 @@ static int update_quorum_block(struct super_block *sb, int event, u64 term, bool
 	return ret;
 }
 
-/*
- * The calling server has fenced previous leaders and reclaimed their
- * resources.  We can now update our fence event with a greater term to
- * stop future leaders from doing the same.
- */
-int scoutfs_quorum_fence_complete(struct super_block *sb, u64 term)
-{
-	return update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_FENCE, term, true);
-}
-
 /*
  * The calling server has been elected and has started running but can't
  * yet assume that it has exclusive access to the metadata device.  We
@@ -595,15 +582,9 @@ int scoutfs_quorum_fence_leaders(struct super_block *sb, u64 term)
 	}
 
 out:
-	if (fence_started) {
-		err = scoutfs_fence_wait_fenced(sb, msecs_to_jiffies(SCOUTFS_QUORUM_FENCE_TO_MS));
-		if (ret == 0)
-			ret = err;
-	} else {
-		err = scoutfs_quorum_fence_complete(sb, term);
-		if (ret == 0)
-			ret = err;
-	}
+	err = scoutfs_fence_wait_fenced(sb, msecs_to_jiffies(SCOUTFS_QUORUM_FENCE_TO_MS));
+	if (ret == 0)
+		ret = err;
 
 	if (ret < 0)
 		scoutfs_inc_counter(sb, quorum_fence_error);
@@ -611,21 +592,34 @@ out:
 	return ret;
 }
 
+/*
+ * The main quorum task maintains its private status.  It seemed cleaner
+ * to occasionally copy the status for showing in sysfs/debugfs files
+ * than to have the two lock access to shared status.  The show copy is
+ * updated after being modified before the quorum task sleeps for a
+ * significant amount of time, either waiting on timeouts or interacting
+ * with the server.
+ */
+static void update_show_status(struct quorum_info *qinf, struct quorum_status *qst)
+{
+	spin_lock(&qinf->show_lock);
+	qinf->show_status = *qst;
+	spin_unlock(&qinf->show_lock);
+}
+
 /*
  * The quorum work always runs in the background of quorum member
  * mounts.  It's responsible for starting and stopping the server if
- * it's elected leader, and the server can call back into it to let it
- * know that it has shut itself down (perhaps due to error) so that the
- * work should stop sending heartbeats.
+ * it's elected leader.  While it's leader it sends heartbeats to
+ * suppress other quorum work from standing for election.
  */
 static void scoutfs_quorum_worker(struct work_struct *work)
 {
 	struct quorum_info *qinf = container_of(work, struct quorum_info, work);
 	struct super_block *sb = qinf->sb;
-	struct mount_options *opts = &SCOUTFS_SB(sb)->opts;
 	struct sockaddr_in unused;
 	struct quorum_host_msg msg;
-	struct quorum_status qst;
+	struct quorum_status qst = {0,};
 	int ret;
 	int err;
 
@@ -634,9 +628,7 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 
 	/* start out as a follower */
 	qst.role = FOLLOWER;
-	qst.term = 0;
 	qst.vote_for = -1;
-	qst.vote_bits = 0;
 
 	/* read our starting term from greatest in all events in all slots */
 	read_greatest_term(sb, &qst.term);
@@ -654,6 +646,8 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 
 	while (!(qinf->shutdown || scoutfs_forcing_unmount(sb))) {
 
+		update_show_status(qinf, &qst);
+
 		ret = recv_msg(sb, &msg, qst.timeout);
 		if (ret < 0) {
 			if (ret != -ETIMEDOUT && ret != -EAGAIN) {
@@ -670,24 +664,6 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 		    msg.term < qst.term)
 			msg.type = SCOUTFS_QUORUM_MSG_INVALID;
 
-		/* if the server has shutdown we become follower */
-		if (!test_bit(QINF_FLAG_SERVER, &qinf->flags) &&
-		    qst.role == LEADER) {
-			qst.role = FOLLOWER;
-			qst.vote_for = -1;
-			qst.vote_bits = 0;
-			qst.timeout = election_timeout();
-			scoutfs_inc_counter(sb, quorum_server_shutdown);
-
-			send_msg_others(sb, SCOUTFS_QUORUM_MSG_RESIGNATION,
-					qst.term);
-			scoutfs_inc_counter(sb, quorum_send_resignation);
-		}
-
-		spin_lock(&qinf->show_lock);
-		qinf->show_status = qst;
-		spin_unlock(&qinf->show_lock);
-
 		trace_scoutfs_quorum_loop(sb, qst.role, qst.term, qst.vote_for,
 					  qst.vote_bits,
 					  ktime_to_timespec64(qst.timeout));
@@ -698,7 +674,6 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 			if (qst.role == LEADER) {
 				scoutfs_warn(sb, "saw msg type %u from %u for term %llu while leader in term %llu, shutting down server.",
 					     msg.type, msg.from, msg.term, qst.term);
-				scoutfs_server_stop(sb);
 			}
 			qst.role = FOLLOWER;
 			qst.term = msg.term;
@@ -720,11 +695,18 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 		/* followers and candidates start new election on timeout */
 		if (qst.role != LEADER &&
 		    ktime_after(ktime_get(), qst.timeout)) {
+			/* .. but only if their server has stopped */
+			if (!scoutfs_server_is_down(sb)) {
+				qst.timeout = election_timeout();
+				scoutfs_inc_counter(sb, quorum_candidate_server_stopping);
+				continue;
+			}
+
 			qst.role = CANDIDATE;
 			qst.term++;
 			qst.vote_for = -1;
 			qst.vote_bits = 0;
-			set_bit(opts->quorum_slot_nr, &qst.vote_bits);
+			set_bit(qinf->our_quorum_slot_nr, &qst.vote_bits);
 			send_msg_others(sb, SCOUTFS_QUORUM_MSG_REQUEST_VOTE,
 					qst.term);
 			qst.timeout = election_timeout();
@@ -761,29 +743,69 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 					qst.term);
 			qst.timeout = heartbeat_interval();
 
+			update_show_status(qinf, &qst);
+
 			/* record that we've been elected before starting up server */
 			ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_ELECT, qst.term, true);
 			if (ret < 0)
 				goto out;
 
-			/* make very sure server is fully shut down */
-			scoutfs_server_stop(sb);
-			/* set server bit before server shutdown could clear */
-			set_bit(QINF_FLAG_SERVER, &qinf->flags);
+			qst.server_start_term = qst.term;
+			qst.server_event = SCOUTFS_QUORUM_EVENT_ELECT;
+			scoutfs_server_start(sb, qst.term);
+		}
 
-			ret = scoutfs_server_start(sb, qst.term);
-			if (ret < 0) {
-				clear_bit(QINF_FLAG_SERVER, &qinf->flags);
-				/* store our increased term */
-				err = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_STOP, qst.term,
-							  true);
-				if (err < 0) {
-					ret = err;
-					goto out;
-				}
-				ret = 0;
-				continue;
+		/*
+		 * This leader's server is up, having finished fencing
+		 * previous leaders.  We update the fence event with the
+		 * current term to let future leaders know that previous
+		 * servers have been fenced.
+		 */
+		if (qst.role == LEADER && qst.server_event != SCOUTFS_QUORUM_EVENT_FENCE &&
+		    scoutfs_server_is_up(sb)) {
+			ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_FENCE, qst.term, true);
+			if (ret < 0)
+				goto out;
+			qst.server_event = SCOUTFS_QUORUM_EVENT_FENCE;
+		}
+
+		/*
+		 * Stop a running server if we're no longer leader in
+		 * its term.
+		 */
+		if (!(qst.role == LEADER && qst.term == qst.server_start_term) &&
+		    scoutfs_server_is_running(sb)) {
+			scoutfs_server_stop(sb);
+		}
+
+		/*
+		 * A previously running server has stopped.  The quorum
+		 * protocol might have shut it down by changing roles or
+		 * it might have stopped on its own, perhaps on errors.
+		 * If we're still a leader then we become a follower and
+		 * send resignations to encourage the next election.
+		 * Always update the _STOP event to stop connections and
+		 * fencing.
+		 */
+		if (qst.server_start_term > 0 && scoutfs_server_is_down(sb)) {
+			if (qst.role == LEADER) {
+				qst.role = FOLLOWER;
+				qst.vote_for = -1;
+				qst.vote_bits = 0;
+				qst.timeout = election_timeout();
+				scoutfs_inc_counter(sb, quorum_server_shutdown);
+
+				send_msg_others(sb, SCOUTFS_QUORUM_MSG_RESIGNATION,
+						qst.server_start_term);
+				scoutfs_inc_counter(sb, quorum_send_resignation);
 			}
+
+			ret = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_STOP,
+						  qst.server_start_term, true);
+			if (ret < 0)
+				goto out;
+
+			qst.server_start_term = 0;
 		}
 
 		/* leaders regularly send heartbeats to delay elections */
@@ -820,12 +842,19 @@ static void scoutfs_quorum_worker(struct work_struct *work)
 		}
 	}
 
+	update_show_status(qinf, &qst);
+
 	/* always try to stop a running server as we stop */
-	if (test_bit(QINF_FLAG_SERVER, &qinf->flags)) {
-		scoutfs_server_stop(sb);
-		scoutfs_fence_stop(sb);
-		send_msg_others(sb, SCOUTFS_QUORUM_MSG_RESIGNATION,
-				qst.term);
+	if (scoutfs_server_is_running(sb)) {
+		scoutfs_server_stop_wait(sb);
+		send_msg_others(sb, SCOUTFS_QUORUM_MSG_RESIGNATION, qst.term);
+
+		if (qst.server_start_term > 0) {
+			err = update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_STOP,
+						  qst.server_start_term, true);
+			if (err < 0 && ret == 0)
+				ret = err;
+		}
 	}
 
 	/* record that this slot no longer has an active quorum */
@@ -837,21 +866,6 @@ out:
 	}
 }
 
-/*
- * The calling server has shutdown and is no longer using shared
- * resources.  Clear the bit so that we stop sending heartbeats and
- * allow the next server to be elected.  Update the stop event so that
- * it won't be considered available by clients or fenced by the next
- * leader.
- */
-void scoutfs_quorum_server_shutdown(struct super_block *sb, u64 term)
-{
-	DECLARE_QUORUM_INFO(sb, qinf);
-
-	clear_bit(QINF_FLAG_SERVER, &qinf->flags);
-	update_quorum_block(sb, SCOUTFS_QUORUM_EVENT_STOP, term, true);
-}
-
 /*
  * Clients read quorum blocks looking for the leader with a server whose
  * address it can try and connect to.
@@ -954,7 +968,6 @@ static ssize_t status_show(struct kobject *kobj, struct kobj_attribute *attr,
 			   char *buf)
 {
 	DECLARE_QUORUM_INFO_KOBJ(kobj, qinf);
-	struct mount_options *opts = &SCOUTFS_SB(qinf->sb)->opts;
 	struct quorum_status qst;
 	struct last_msg last;
 	struct timespec64 ts;
@@ -971,9 +984,11 @@ static ssize_t status_show(struct kobject *kobj, struct kobj_attribute *attr,
 	ret = 0;
 
 	snprintf_ret(buf, size, &ret, "quorum_slot_nr %u\n",
-		     opts->quorum_slot_nr);
+		     qinf->our_quorum_slot_nr);
 	snprintf_ret(buf, size, &ret, "term %llu\n",
 		     qst.term);
+	snprintf_ret(buf, size, &ret, "server_start_term %llu\n", qst.server_start_term);
+	snprintf_ret(buf, size, &ret, "server_event %d\n", qst.server_event);
 	snprintf_ret(buf, size, &ret, "role %d (%s)\n",
 		     qst.role, role_str(qst.role));
 	snprintf_ret(buf, size, &ret, "vote_for %d\n",
@@ -1048,7 +1063,6 @@ static inline bool valid_ipv4_port(__be16 port)
 static int verify_quorum_slots(struct super_block *sb)
 {
 	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
-	struct mount_options *opts = &SCOUTFS_SB(sb)->opts;
 	char slots[(SCOUTFS_QUORUM_MAX_SLOTS * 3) + 1];
 	DECLARE_QUORUM_INFO(sb, qinf);
 	struct sockaddr_in other;
@@ -1099,7 +1113,7 @@ static int verify_quorum_slots(struct super_block *sb)
 		return -EINVAL;
 	}
 
-	if (!quorum_slot_present(super, opts->quorum_slot_nr)) {
+	if (!quorum_slot_present(super, qinf->our_quorum_slot_nr)) {
 		char *str = slots;
 		*str = '\0';
 		for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
@@ -1114,7 +1128,7 @@ static int verify_quorum_slots(struct super_block *sb)
 			}
 		}
 		scoutfs_err(sb, "quorum_slot_nr=%u option references unused slot, must be one of the following configured slots:%s",
-			    opts->quorum_slot_nr, slots);
+			    qinf->our_quorum_slot_nr, slots);
 		return -EINVAL;
 	}
 
@@ -1137,11 +1151,12 @@ static int verify_quorum_slots(struct super_block *sb)
 int scoutfs_quorum_setup(struct super_block *sb)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
-	struct mount_options *opts = &sbi->opts;
+	struct scoutfs_mount_options opts;
 	struct quorum_info *qinf;
 	int ret;
 
-	if (opts->quorum_slot_nr < 0)
+	scoutfs_options_read(sb, &opts);
+	if (opts.quorum_slot_nr < 0)
 		return 0;
 
 	qinf = kzalloc(sizeof(struct quorum_info), GFP_KERNEL);
@@ -1153,6 +1168,8 @@ int scoutfs_quorum_setup(struct super_block *sb)
 	spin_lock_init(&qinf->show_lock);
 	INIT_WORK(&qinf->work, scoutfs_quorum_worker);
 	scoutfs_sysfs_init_attrs(sb, &qinf->ssa);
+	/* static for the lifetime of the mount */
+	qinf->our_quorum_slot_nr = opts.quorum_slot_nr;
 
 	sbi->quorum_info = qinf;
 	qinf->sb = sb;

kmod/src/quorum.h

@@ -2,14 +2,12 @@
 #define _SCOUTFS_QUORUM_H_
 
 int scoutfs_quorum_server_sin(struct super_block *sb, struct sockaddr_in *sin);
-void scoutfs_quorum_server_shutdown(struct super_block *sb, u64 term);
 
 u8 scoutfs_quorum_votes_needed(struct super_block *sb);
 void scoutfs_quorum_slot_sin(struct scoutfs_super_block *super, int i,
 			     struct sockaddr_in *sin);
 
 int scoutfs_quorum_fence_leaders(struct super_block *sb, u64 term);
-int scoutfs_quorum_fence_complete(struct super_block *sb, u64 term);
 
 int scoutfs_quorum_setup(struct super_block *sb);
 void scoutfs_quorum_shutdown(struct super_block *sb);

kmod/src/recov.c

@@ -262,7 +262,7 @@ void scoutfs_recov_shutdown(struct super_block *sb)
 	recinf->timeout_fn = NULL;
 	spin_unlock(&recinf->lock);
 
-	list_for_each_entry_safe(pend, tmp, &recinf->pending, head) {
+	list_for_each_entry_safe(pend, tmp, &list, head) {
 		list_del(&pend->head);
 		kfree(pend);
 	}

kmod/src/scoutfs_trace.h

@@ -1843,6 +1843,53 @@ DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_down,
 	TP_ARGS(sb, rid, nr_clients)
 );
 
+DECLARE_EVENT_CLASS(scoutfs_server_commit_users_class,
+        TP_PROTO(struct super_block *sb, int holding, int applying, int nr_holders,
+		 u32 avail_before, u32 freed_before, int exceeded),
+        TP_ARGS(sb, holding, applying, nr_holders, avail_before, freed_before, exceeded),
+        TP_STRUCT__entry(
+		SCSB_TRACE_FIELDS
+		__field(int, holding)
+		__field(int, applying)
+		__field(int, nr_holders)
+		__field(__u32, avail_before)
+		__field(__u32, freed_before)
+		__field(int, exceeded)
+        ),
+        TP_fast_assign(
+		SCSB_TRACE_ASSIGN(sb);
+		__entry->holding = !!holding;
+		__entry->applying = !!applying;
+		__entry->nr_holders = nr_holders;
+		__entry->avail_before = avail_before;
+		__entry->freed_before = freed_before;
+		__entry->exceeded = !!exceeded;
+        ),
+	TP_printk(SCSBF" holding %u applying %u nr %u avail_before %u freed_before %u exceeded %u",
+		  SCSB_TRACE_ARGS, __entry->holding, __entry->applying, __entry->nr_holders,
+		  __entry->avail_before, __entry->freed_before, __entry->exceeded)
+);
+DEFINE_EVENT(scoutfs_server_commit_users_class, scoutfs_server_commit_hold,
+        TP_PROTO(struct super_block *sb, int holding, int applying, int nr_holders,
+		 u32 avail_before, u32 freed_before, int exceeded),
+        TP_ARGS(sb, holding, applying, nr_holders, avail_before, freed_before, exceeded)
+);
+DEFINE_EVENT(scoutfs_server_commit_users_class, scoutfs_server_commit_apply,
+        TP_PROTO(struct super_block *sb, int holding, int applying, int nr_holders,
+		 u32 avail_before, u32 freed_before, int exceeded),
+        TP_ARGS(sb, holding, applying, nr_holders, avail_before, freed_before, exceeded)
+);
+DEFINE_EVENT(scoutfs_server_commit_users_class, scoutfs_server_commit_start,
+        TP_PROTO(struct super_block *sb, int holding, int applying, int nr_holders,
+		 u32 avail_before, u32 freed_before, int exceeded),
+        TP_ARGS(sb, holding, applying, nr_holders, avail_before, freed_before, exceeded)
+);
+DEFINE_EVENT(scoutfs_server_commit_users_class, scoutfs_server_commit_end,
+        TP_PROTO(struct super_block *sb, int holding, int applying, int nr_holders,
+		 u32 avail_before, u32 freed_before, int exceeded),
+        TP_ARGS(sb, holding, applying, nr_holders, avail_before, freed_before, exceeded)
+);
+
 #define slt_symbolic(mode)						\
 	__print_symbolic(mode,					\
 		{ SLT_CLIENT,		"client" },	\
@@ -2620,9 +2667,9 @@ TRACE_EVENT(scoutfs_item_invalidate_page,
 
 DECLARE_EVENT_CLASS(scoutfs_omap_group_class,
 	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
-		 int bit_nr, int bit_count),
+		 int bit_nr),
 
-	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count),
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr),
 
 	TP_STRUCT__entry(
 		SCSB_TRACE_FIELDS
@@ -2630,7 +2677,6 @@ DECLARE_EVENT_CLASS(scoutfs_omap_group_class,
 		__field(__u64, group_nr)
 		__field(unsigned int, group_total)
 		__field(int, bit_nr)
-		__field(int, bit_count)
 	),
 
 	TP_fast_assign(
@@ -2639,43 +2685,42 @@ DECLARE_EVENT_CLASS(scoutfs_omap_group_class,
 		__entry->group_nr = group_nr;
 		__entry->group_total = group_total;
 		__entry->bit_nr = bit_nr;
-		__entry->bit_count = bit_count;
 	),
 
-	TP_printk(SCSBF" grp %p group_nr %llu group_total %u bit_nr %d bit_count %d",
+	TP_printk(SCSBF" grp %p group_nr %llu group_total %u bit_nr %d",
 		  SCSB_TRACE_ARGS, __entry->grp, __entry->group_nr, __entry->group_total,
-		  __entry->bit_nr, __entry->bit_count)
+		  __entry->bit_nr)
 );
 
 DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_alloc,
 	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
-		 int bit_nr, int bit_count),
-	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count)
+		 int bit_nr),
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr)
 );
 DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_free,
 	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
-		 int bit_nr, int bit_count),
-	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count)
+		 int bit_nr),
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr)
 );
 DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_inc,
 	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
-		 int bit_nr, int bit_count),
-	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count)
+		 int bit_nr),
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr)
 );
 DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_dec,
 	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
-		 int bit_nr, int bit_count),
-	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count)
+		 int bit_nr),
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr)
 );
 DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_request,
 	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
-		 int bit_nr, int bit_count),
-	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count)
+		 int bit_nr),
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr)
 );
 DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_destroy,
 	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
-		 int bit_nr, int bit_count),
-	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count)
+		 int bit_nr),
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr)
 );
 
 TRACE_EVENT(scoutfs_omap_should_delete,

kmod/src/server.h

@@ -64,8 +64,6 @@ int scoutfs_server_lock_response(struct super_block *sb, u64 rid, u64 id,
 				 struct scoutfs_net_lock *nl);
 int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid,
 					struct scoutfs_key *key);
-void scoutfs_server_hold_commit(struct super_block *sb);
-int scoutfs_server_apply_commit(struct super_block *sb, int err);
 void scoutfs_server_recov_finish(struct super_block *sb, u64 rid, int which);
 
 int scoutfs_server_send_omap_request(struct super_block *sb, u64 rid,
@@ -77,9 +75,12 @@ u64 scoutfs_server_seq(struct super_block *sb);
 u64 scoutfs_server_next_seq(struct super_block *sb);
 void scoutfs_server_set_seq_if_greater(struct super_block *sb, u64 seq);
 
-int scoutfs_server_start(struct super_block *sb, u64 term);
-void scoutfs_server_abort(struct super_block *sb);
+void scoutfs_server_start(struct super_block *sb, u64 term);
 void scoutfs_server_stop(struct super_block *sb);
+void scoutfs_server_stop_wait(struct super_block *sb);
+bool scoutfs_server_is_running(struct super_block *sb);
+bool scoutfs_server_is_up(struct super_block *sb);
+bool scoutfs_server_is_down(struct super_block *sb);
 
 int scoutfs_server_setup(struct super_block *sb);
 void scoutfs_server_destroy(struct super_block *sb);

kmod/src/super.c

@@ -132,44 +132,6 @@ out:
 	return ret;
 }
 
-static int scoutfs_show_options(struct seq_file *seq, struct dentry *root)
-{
-	struct super_block *sb = root->d_sb;
-	struct mount_options *opts = &SCOUTFS_SB(sb)->opts;
-
-	if (opts->quorum_slot_nr >= 0)
-		seq_printf(seq, ",quorum_slot_nr=%d", opts->quorum_slot_nr);
-	seq_printf(seq, ",metadev_path=%s", opts->metadev_path);
-
-	return 0;
-}
-
-static ssize_t metadev_path_show(struct kobject *kobj,
-				 struct kobj_attribute *attr, char *buf)
-{
-	struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj);
-	struct mount_options *opts = &SCOUTFS_SB(sb)->opts;
-
-	return snprintf(buf, PAGE_SIZE, "%s", opts->metadev_path);
-}
-SCOUTFS_ATTR_RO(metadev_path);
-
-static ssize_t quorum_server_nr_show(struct kobject *kobj,
-			      struct kobj_attribute *attr, char *buf)
-{
-	struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj);
-	struct mount_options *opts = &SCOUTFS_SB(sb)->opts;
-
-	return snprintf(buf, PAGE_SIZE, "%d\n", opts->quorum_slot_nr);
-}
-SCOUTFS_ATTR_RO(quorum_server_nr);
-
-static struct attribute *mount_options_attrs[] = {
-	SCOUTFS_ATTR_PTR(metadev_path),
-	SCOUTFS_ATTR_PTR(quorum_server_nr),
-	NULL,
-};
-
 static int scoutfs_sync_fs(struct super_block *sb, int wait)
 {
 	trace_scoutfs_sync_fs(sb, wait);
@@ -246,13 +208,11 @@ static void scoutfs_put_super(struct super_block *sb)
 	scoutfs_destroy_triggers(sb);
 	scoutfs_fence_destroy(sb);
 	scoutfs_options_destroy(sb);
-	scoutfs_sysfs_destroy_attrs(sb, &sbi->mopts_ssa);
 	debugfs_remove(sbi->debug_root);
 	scoutfs_destroy_counters(sb);
 	scoutfs_destroy_sysfs(sb);
 	scoutfs_metadev_close(sb);
 
-	kfree(sbi->opts.metadev_path);
 	kfree(sbi);
 
 	sb->s_fs_info = NULL;
@@ -271,6 +231,8 @@ static void scoutfs_umount_begin(struct super_block *sb)
 
 	scoutfs_warn(sb, "forcing unmount, can return errors and lose unsynced data");
 	sbi->forced_unmount = true;
+
+	scoutfs_client_net_shutdown(sb);
 }
 
 static const struct super_operations scoutfs_super_ops = {
@@ -280,7 +242,7 @@ static const struct super_operations scoutfs_super_ops = {
 	.destroy_inode = scoutfs_destroy_inode,
 	.sync_fs = scoutfs_sync_fs,
 	.statfs = scoutfs_statfs,
-	.show_options = scoutfs_show_options,
+	.show_options = scoutfs_options_show,
 	.put_super = scoutfs_put_super,
 	.umount_begin = scoutfs_umount_begin,
 };
@@ -509,9 +471,9 @@ out:
 
 static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 {
-	struct scoutfs_sb_info *sbi;
-	struct mount_options opts;
+	struct scoutfs_mount_options opts;
 	struct block_device *meta_bdev;
+	struct scoutfs_sb_info *sbi;
 	struct inode *inode;
 	int ret;
 
@@ -534,18 +496,17 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 
 	ret = assign_random_id(sbi);
 	if (ret < 0)
-		return ret;
+		goto out;
 
 	spin_lock_init(&sbi->next_ino_lock);
 	spin_lock_init(&sbi->data_wait_root.lock);
 	sbi->data_wait_root.root = RB_ROOT;
-	scoutfs_sysfs_init_attrs(sb, &sbi->mopts_ssa);
 
-	ret = scoutfs_parse_options(sb, data, &opts);
-	if (ret)
+	/* parse options early for use during setup */
+	ret = scoutfs_options_early_setup(sb, data);
+	if (ret < 0)
 		goto out;
-
-	sbi->opts = opts;
+	scoutfs_options_read(sb, &opts);
 
 	ret = sb_set_blocksize(sb, SCOUTFS_BLOCK_SM_SIZE);
 	if (ret != SCOUTFS_BLOCK_SM_SIZE) {
@@ -554,9 +515,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 		goto out;
 	}
 
-	meta_bdev =
-		blkdev_get_by_path(sbi->opts.metadev_path,
-				   SCOUTFS_META_BDEV_MODE, sb);
+	meta_bdev = blkdev_get_by_path(opts.metadev_path, SCOUTFS_META_BDEV_MODE, sb);
 	if (IS_ERR(meta_bdev)) {
 		scoutfs_err(sb, "could not open metadev: error %ld",
 			    PTR_ERR(meta_bdev));
@@ -576,8 +535,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 	      scoutfs_setup_sysfs(sb) ?:
 	      scoutfs_setup_counters(sb) ?:
 	      scoutfs_options_setup(sb) ?:
-	      scoutfs_sysfs_create_attrs(sb, &sbi->mopts_ssa,
-				mount_options_attrs, "mount_options") ?:
 	      scoutfs_setup_triggers(sb) ?:
 	      scoutfs_fence_setup(sb) ?:
 	      scoutfs_block_setup(sb) ?:
@@ -599,7 +556,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 		goto out;
 
 	/* this interruptible iget lets hung mount be aborted with ctl-c */
-	inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO, SCOUTFS_LKF_INTERRUPTIBLE);
+	inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO, SCOUTFS_LKF_INTERRUPTIBLE, 0);
 	if (IS_ERR(inode)) {
 		ret = PTR_ERR(inode);
 		if (ret == -ERESTARTSYS)
@@ -650,6 +607,7 @@ static void scoutfs_kill_sb(struct super_block *sb)
 	}
 
 	if (SCOUTFS_HAS_SBI(sb)) {
+		scoutfs_options_stop(sb);
 		scoutfs_inode_orphan_stop(sb);
 		scoutfs_lock_unmount_begin(sb);
 	}

kmod/src/super.h

@@ -44,6 +44,7 @@ struct scoutfs_sb_info {
 
 	spinlock_t next_ino_lock;
 
+	struct options_info *options_info;
 	struct data_info *data_info;
 	struct inode_sb_info *inode_sb_info;
 	struct btree_info *btree_info;
@@ -74,10 +75,6 @@ struct scoutfs_sb_info {
 	struct scoutfs_counters *counters;
 	struct scoutfs_triggers *triggers;
 
-	struct mount_options opts;
-	struct options_sb_info *options;
-	struct scoutfs_sysfs_attrs mopts_ssa;
-
 	struct dentry *debug_root;
 
 	bool forced_unmount;

kmod/src/sysfs.c

@@ -37,6 +37,15 @@ struct attr_funcs {
 #define ATTR_FUNCS_RO(_name) \
 	static struct attr_funcs _name##_attr_funcs = __ATTR_RO(_name)
 
+static ssize_t data_device_maj_min_show(struct kobject *kobj, struct attribute *attr, char *buf)
+{
+	struct super_block *sb = KOBJ_TO_SB(kobj, sb_id_kobj);
+
+	return snprintf(buf, PAGE_SIZE, "%u:%u\n",
+			MAJOR(sb->s_bdev->bd_dev), MINOR(sb->s_bdev->bd_dev));
+}
+ATTR_FUNCS_RO(data_device_maj_min);
+
 static ssize_t format_version_show(struct kobject *kobj, struct attribute *attr,
 			 char *buf)
 {
@@ -101,6 +110,7 @@ static ssize_t attr_funcs_show(struct kobject *kobj, struct attribute *attr,
 
 
 static struct attribute *sb_id_attrs[] = {
+	&data_device_maj_min_attr_funcs.attr,
 	&format_version_attr_funcs.attr,
 	&fsid_attr_funcs.attr,
 	&rid_attr_funcs.attr,

kmod/src/trans.c

@@ -640,6 +640,7 @@ void scoutfs_shutdown_trans(struct super_block *sb)
 			tri->write_workq = NULL;
 		}
 
+		scoutfs_alloc_prepare_commit(sb, &tri->alloc, &tri->wri);
 		scoutfs_block_writer_forget_all(sb, &tri->wri);
 
 		kfree(tri);

kmod/src/xattr.c

@@ -57,12 +57,6 @@ static u32 xattr_names_equal(const char *a_name, unsigned int a_len,
 	return a_len == b_len && memcmp(a_name, b_name, a_len) == 0;
 }
 
-static unsigned int xattr_full_bytes(struct scoutfs_xattr *xat)
-{
-	return offsetof(struct scoutfs_xattr,
-		        name[xat->name_len + le16_to_cpu(xat->val_len)]);
-}
-
 static unsigned int xattr_nr_parts(struct scoutfs_xattr *xat)
 {
 	return SCOUTFS_XATTR_NR_PARTS(xat->name_len,
@@ -137,12 +131,29 @@ int scoutfs_xattr_parse_tags(const char *name, unsigned int name_len,
 }
 
 /*
- * Find the next xattr and copy the key, xattr header, and as much of
- * the name and value into the callers buffer as we can.  Returns the
- * number of bytes copied which include the header, name, and value and
- * can be limited by the xattr length or the callers buffer.  The caller
- * is responsible for comparing their lengths, the header, and the
- * returned length before safely using the xattr.
+ * xattrs are stored in multiple items.   The first item is a
+ * concatenation of an initial header, the name, and then as much of the
+ * value as fits in the remainder of the first item.  This return the
+ * size of the first item that'd store an xattr with the given name
+ * length and value payload size.
+ */
+static int first_item_bytes(int name_len, size_t size)
+{
+	if (WARN_ON_ONCE(name_len <= 0) ||
+	    WARN_ON_ONCE(name_len > SCOUTFS_XATTR_MAX_NAME_LEN))
+		return 0;
+
+	return min_t(int, sizeof(struct scoutfs_xattr) + name_len + size,
+			  SCOUTFS_XATTR_MAX_PART_SIZE);
+}
+
+/*
+ * Find the next xattr, set the caller's key, and copy as much of the
+ * first item into the callers buffer as we can.  Returns the number of
+ * bytes copied which can include the header, name, and start of the
+ * value from the first item.  The caller is responsible for comparing
+ * their lengths, the header, and the returned length before safely
+ * using the buffer.
  *
  * If a name is provided then we'll iterate over items with a matching
  * name_hash until we find a matching name.  If we don't find a matching
@@ -154,20 +165,17 @@ int scoutfs_xattr_parse_tags(const char *name, unsigned int name_len,
  * Returns -ENOENT if it didn't find a next item.
  */
 static int get_next_xattr(struct inode *inode, struct scoutfs_key *key,
-			  struct scoutfs_xattr *xat, unsigned int bytes,
+			  struct scoutfs_xattr *xat, unsigned int xat_bytes,
 			  const char *name, unsigned int name_len,
 			  u64 name_hash, u64 id, struct scoutfs_lock *lock)
 {
 	struct super_block *sb = inode->i_sb;
 	struct scoutfs_key last;
-	u8 last_part;
-	int total;
-	u8 part;
 	int ret;
 
 	/* need to be able to see the name we're looking for */
-	if (WARN_ON_ONCE(name_len > 0 && bytes < offsetof(struct scoutfs_xattr,
-							  name[name_len])))
+	if (WARN_ON_ONCE(name_len > 0 &&
+			 xat_bytes < offsetof(struct scoutfs_xattr, name[name_len])))
 		return -EINVAL;
 
 	if (name_len)
@@ -176,26 +184,15 @@ static int get_next_xattr(struct inode *inode, struct scoutfs_key *key,
 	init_xattr_key(key, scoutfs_ino(inode), name_hash, id);
 	init_xattr_key(&last, scoutfs_ino(inode), U32_MAX, U64_MAX);
 
-	last_part = 0;
-	part = 0;
-	total = 0;
-
 	for (;;) {
-		key->skx_part = part;
-		ret = scoutfs_item_next(sb, key, &last,
-					(void *)xat + total, bytes - total,
-					lock);
-		if (ret < 0) {
-			/* XXX corruption, ran out of parts */
-			if (ret == -ENOENT && part > 0)
-				ret = -EIO;
+		ret = scoutfs_item_next(sb, key, &last, xat, xat_bytes, lock);
+		if (ret < 0)
 			break;
-		}
 
 		trace_scoutfs_xattr_get_next_key(sb, key);
 
 		/* XXX corruption */
-		if (key->skx_part != part) {
+		if (key->skx_part != 0) {
 			ret = -EIO;
 			break;
 		}
@@ -205,8 +202,7 @@ static int get_next_xattr(struct inode *inode, struct scoutfs_key *key,
 		 * the first part and if the next xattr name fits in our
 		 * buffer then the item must have included it.
 		 */
-		if (part == 0 &&
-		    (ret < sizeof(struct scoutfs_xattr) ||
+		if ((ret < sizeof(struct scoutfs_xattr) ||
 		     (xat->name_len <= name_len &&
 		      ret < offsetof(struct scoutfs_xattr,
 				     name[xat->name_len])) ||
@@ -216,7 +212,7 @@ static int get_next_xattr(struct inode *inode, struct scoutfs_key *key,
 			break;
 		}
 
-		if (part == 0 && name_len) {
+		if (name_len > 0) {
 			/* ran out of names that could match */
 			if (le64_to_cpu(key->skx_name_hash) != name_hash) {
 				ret = -ENOENT;
@@ -224,64 +220,126 @@ static int get_next_xattr(struct inode *inode, struct scoutfs_key *key,
 			}
 
 			/* keep looking for our name */
-			if (!xattr_names_equal(name, name_len,
-					       xat->name, xat->name_len)) {
-				part = 0;
+			if (!xattr_names_equal(name, name_len, xat->name, xat->name_len)) {
 				le64_add_cpu(&key->skx_id, 1);
 				continue;
 			}
-
-			/* use the matching name we found */
-			last_part = xattr_nr_parts(xat) - 1;
 		}
 
-		total += ret;
-		if (total == bytes || part == last_part) {
-			/* copied as much as we could */
-			ret = total;
-			break;
-		}
-		part++;
+		/* found next name */
+		break;
 	}
 
 	return ret;
 }
 
+/*
+ * The caller has already read and verified the xattr's first item.
+ * Copy the value from the tail of the first item and from any future
+ * items into the destination buffer.
+ */
+static int copy_xattr_value(struct super_block *sb, struct scoutfs_key *xat_key,
+			    struct scoutfs_xattr *xat, int xat_bytes,
+			    char *buffer, size_t size,
+			    struct scoutfs_lock *lock)
+{
+	struct scoutfs_key key;
+	size_t copied = 0;
+	int val_tail;
+	int bytes;
+	int ret;
+	int i;
+
+	/* must have first item up to value */
+	if (WARN_ON_ONCE(xat_bytes < sizeof(struct scoutfs_xattr)) ||
+	    WARN_ON_ONCE(xat_bytes < offsetof(struct scoutfs_xattr, name[xat->name_len])))
+		return -EINVAL;
+
+	/* only ever copy up to the full value */
+	size = min_t(size_t, size, le16_to_cpu(xat->val_len));
+
+	/* must have full first item if caller needs value from second item */
+	val_tail = SCOUTFS_XATTR_MAX_PART_SIZE -
+		   offsetof(struct scoutfs_xattr, name[xat->name_len]);
+	if (WARN_ON_ONCE(size > val_tail && xat_bytes != SCOUTFS_XATTR_MAX_PART_SIZE))
+		return -EINVAL;
+
+	/* copy from tail of first item */
+	bytes = min_t(unsigned int, size, val_tail);
+	if (bytes > 0) {
+		memcpy(buffer, &xat->name[xat->name_len], bytes);
+		copied += bytes;
+	}
+
+	key = *xat_key;
+	for (i = 1; copied < size; i++) {
+		key.skx_part = i;
+		bytes = min_t(unsigned int, size - copied, SCOUTFS_XATTR_MAX_PART_SIZE);
+
+		ret = scoutfs_item_lookup(sb, &key, buffer + copied, bytes, lock);
+		if (ret >= 0 && ret != bytes)
+			ret = -EIO;
+		if (ret < 0)
+			return ret;
+
+		copied += ret;
+	}
+
+	return copied;
+}
+
+/*
+ * The caller is working with items that are either in the allocated
+ * first compound item or further items that are offsets into a value
+ * buffer.  Give them a pointer and length of the start of the item.
+ */
+static void xattr_item_part_buffer(void **buf, int *len, int part,
+				   struct scoutfs_xattr *xat, unsigned int xat_bytes,
+				   const char *value, size_t size)
+{
+	int off;
+
+	if (part == 0) {
+		*buf = xat;
+		*len = xat_bytes;
+	} else {
+		off = (part * SCOUTFS_XATTR_MAX_PART_SIZE) -
+		      offsetof(struct scoutfs_xattr, name[xat->name_len]);
+		BUG_ON(off >= size); /* calls limited by number of parts */
+		*buf = (void *)value + off;
+		*len = min_t(size_t, size - off, SCOUTFS_XATTR_MAX_PART_SIZE);
+	}
+}
+
 /*
  * Create all the items associated with the given xattr.  If this
  * returns an error it will have already cleaned up any items it created
  * before seeing the error.
  */
-static int create_xattr_items(struct inode *inode, u64 id,
-			      struct scoutfs_xattr *xat, unsigned int bytes,
+static int create_xattr_items(struct inode *inode, u64 id, struct scoutfs_xattr *xat,
+			      int xat_bytes, const char *value, size_t size, u8 new_parts,
 			      struct scoutfs_lock *lock)
 {
 	struct super_block *sb = inode->i_sb;
 	struct scoutfs_key key;
-	unsigned int part_bytes;
-	unsigned int total;
-	int ret;
+	int ret = 0;
+	void *buf;
+	int len;
+	int i;
 
 	init_xattr_key(&key, scoutfs_ino(inode),
 		       xattr_name_hash(xat->name, xat->name_len), id);
 
-	total = 0;
-	ret = 0;
-	while (total < bytes) {
-		part_bytes = min_t(unsigned int, bytes - total,
-				   SCOUTFS_XATTR_MAX_PART_SIZE);
+	for (i = 0; i < new_parts; i++) {
+		key.skx_part = i;
+		xattr_item_part_buffer(&buf, &len, i, xat, xat_bytes, value, size);
 
-		ret = scoutfs_item_create(sb, &key,
-					  (void *)xat + total, part_bytes,
-					  lock);
-		if (ret) {
+		ret = scoutfs_item_create(sb, &key, buf, len, lock);
+		if (ret < 0) {
 			while (key.skx_part-- > 0)
 				scoutfs_item_delete(sb, &key, lock);
 			break;
 		}
-
-		total += part_bytes;
-		key.skx_part++;
 	}
 
 	return ret;
@@ -329,20 +387,20 @@ out:
  * deleted items.
  */
 static int change_xattr_items(struct inode *inode, u64 id,
-			      struct scoutfs_xattr *new_xat,
-			      unsigned int new_bytes, u8 new_parts,
-			      u8 old_parts, struct scoutfs_lock *lock)
+			      struct scoutfs_xattr *xat, int xat_bytes,
+			      const char *value, size_t size,
+			      u8 new_parts, u8 old_parts, struct scoutfs_lock *lock)
 {
 	struct super_block *sb = inode->i_sb;
 	struct scoutfs_key key;
 	int last_created = -1;
-	int bytes;
-	int off;
+	void *buf;
+	int len;
 	int i;
 	int ret;
 
 	init_xattr_key(&key, scoutfs_ino(inode),
-		       xattr_name_hash(new_xat->name, new_xat->name_len), id);
+		       xattr_name_hash(xat->name, xat->name_len), id);
 
 	/* dirty existing old items */
 	for (i = 0; i < old_parts; i++) {
@@ -354,13 +412,10 @@ static int change_xattr_items(struct inode *inode, u64 id,
 
 	/* create any new items past the old */
 	for (i = old_parts; i < new_parts; i++) {
-		off = i * SCOUTFS_XATTR_MAX_PART_SIZE;
-		bytes = min_t(unsigned int, new_bytes - off,
-			      SCOUTFS_XATTR_MAX_PART_SIZE);
-
 		key.skx_part = i;
-		ret = scoutfs_item_create(sb, &key, (void *)new_xat + off,
-					  bytes, lock);
+		xattr_item_part_buffer(&buf, &len, i, xat, xat_bytes, value, size);
+
+		ret = scoutfs_item_create(sb, &key, buf, len, lock);
 		if (ret)
 			goto out;
 
@@ -369,13 +424,10 @@ static int change_xattr_items(struct inode *inode, u64 id,
 
 	/* update dirtied overlapping existing items, last partial first */
 	for (i = min(old_parts, new_parts) - 1; i >= 0; i--) {
-		off = i * SCOUTFS_XATTR_MAX_PART_SIZE;
-		bytes = min_t(unsigned int, new_bytes - off,
-			      SCOUTFS_XATTR_MAX_PART_SIZE);
-
 		key.skx_part = i;
-		ret = scoutfs_item_update(sb, &key, (void *)new_xat + off,
-					  bytes, lock);
+		xattr_item_part_buffer(&buf, &len, i, xat, xat_bytes, value, size);
+
+		ret = scoutfs_item_update(sb, &key, buf, len, lock);
 		/* only last partial can fail, then we unwind created */
 		if (ret < 0)
 			goto out;
@@ -412,7 +464,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer,
 	struct scoutfs_xattr *xat = NULL;
 	struct scoutfs_lock *lck = NULL;
 	struct scoutfs_key key;
-	unsigned int bytes;
+	unsigned int xat_bytes;
 	size_t name_len;
 	int ret;
 
@@ -423,9 +475,8 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer,
 	if (name_len > SCOUTFS_XATTR_MAX_NAME_LEN)
 		return -ENODATA;
 
-	/* only need enough for caller's name and value sizes */
-	bytes = sizeof(struct scoutfs_xattr) + name_len + size;
-	xat = __vmalloc(bytes, GFP_NOFS, PAGE_KERNEL);
+	xat_bytes = first_item_bytes(name_len, size);
+	xat = kmalloc(xat_bytes, GFP_NOFS);
 	if (!xat)
 		return -ENOMEM;
 
@@ -435,40 +486,32 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer,
 
 	down_read(&si->xattr_rwsem);
 
-	ret = get_next_xattr(inode, &key, xat, bytes,
-			     name, name_len, 0, 0, lck);
-
-	up_read(&si->xattr_rwsem);
-	scoutfs_unlock(sb, lck, SCOUTFS_LOCK_READ);
+	ret = get_next_xattr(inode, &key, xat, xat_bytes, name, name_len, 0, 0, lck);
 
 	if (ret < 0) {
 		if (ret == -ENOENT)
 			ret = -ENODATA;
-		goto out;
+		goto unlock;
 	}
 
 	/* the caller just wants to know the size */
 	if (size == 0) {
 		ret = le16_to_cpu(xat->val_len);
-		goto out;
+		goto unlock;
 	}
 
 	/* the caller's buffer wasn't big enough */
 	if (size < le16_to_cpu(xat->val_len)) {
 		ret = -ERANGE;
-		goto out;
+		goto unlock;
 	}
 
-	/* XXX corruption, the items didn't match the header */
-	if (ret < xattr_full_bytes(xat)) {
-		ret = -EIO;
-		goto out;
-	}
-
-	ret = le16_to_cpu(xat->val_len);
-	memcpy(buffer, &xat->name[xat->name_len], ret);
+	ret = copy_xattr_value(sb, &key, xat, xat_bytes, buffer, size, lck);
+unlock:
+	up_read(&si->xattr_rwsem);
+	scoutfs_unlock(sb, lck, SCOUTFS_LOCK_READ);
 out:
-	vfree(xat);
+	kfree(xat);
 	return ret;
 }
 
@@ -596,7 +639,8 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name,
 	bool undo_totl = false;
 	LIST_HEAD(ind_locks);
 	u8 found_parts;
-	unsigned int bytes;
+	unsigned int xat_bytes_totl;
+	unsigned int xat_bytes;
 	unsigned int val_len;
 	u64 ind_seq;
 	u64 total;
@@ -629,9 +673,12 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name,
 	if (tgs.totl && ((ret = parse_totl_key(&totl_key, name, name_len)) != 0))
 		return ret;
 
-	bytes = sizeof(struct scoutfs_xattr) + name_len + size;
-	/* alloc enough to read old totl value */
-	xat = __vmalloc(bytes + SCOUTFS_XATTR_MAX_TOTL_U64, GFP_NOFS, PAGE_KERNEL);
+	/* allocate enough to always read an existing xattr's totl */
+	xat_bytes_totl = first_item_bytes(name_len,
+					  max_t(size_t, size, SCOUTFS_XATTR_MAX_TOTL_U64));
+	/* but store partial first item that only includes the new xattr's value */
+	xat_bytes = first_item_bytes(name_len, size);
+	xat = kmalloc(xat_bytes_totl, GFP_NOFS);
 	if (!xat) {
 		ret = -ENOMEM;
 		goto out;
@@ -645,9 +692,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name,
 	down_write(&si->xattr_rwsem);
 
 	/* find an existing xattr to delete, including possible totl value */
-	ret = get_next_xattr(inode, &key, xat,
-			     sizeof(struct scoutfs_xattr) + name_len + SCOUTFS_XATTR_MAX_TOTL_U64,
-			     name, name_len, 0, 0, lck);
+	ret = get_next_xattr(inode, &key, xat, xat_bytes_totl, name, name_len, 0, 0, lck);
 	if (ret < 0 && ret != -ENOENT)
 		goto unlock;
 
@@ -683,7 +728,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name,
 		le64_add_cpu(&tval.total, -total);
 	}
 
-	/* prepare our xattr */
+	/* prepare the xattr header, name, and start of value in first item */
 	if (value) {
 		if (found_parts)
 			id = le64_to_cpu(key.skx_id);
@@ -693,7 +738,9 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name,
 		xat->val_len = cpu_to_le16(size);
 		memset(xat->__pad, 0, sizeof(xat->__pad));
 		memcpy(xat->name, name, name_len);
-		memcpy(&xat->name[xat->name_len], value, size);
+		memcpy(&xat->name[name_len], value,
+		       min(size, SCOUTFS_XATTR_MAX_PART_SIZE -
+			         offsetof(struct scoutfs_xattr, name[name_len])));
 
 		if (tgs.totl) {
 			ret = parse_totl_u64(value, size, &total);
@@ -741,14 +788,15 @@ retry:
 	}
 
 	if (found_parts && value)
-		ret = change_xattr_items(inode, id, xat, bytes,
+		ret = change_xattr_items(inode, id, xat, xat_bytes, value, size,
 					 xattr_nr_parts(xat), found_parts, lck);
 	else if (found_parts)
 		ret = delete_xattr_items(inode, le64_to_cpu(key.skx_name_hash),
 					 le64_to_cpu(key.skx_id), found_parts,
 					 lck);
 	else
-		ret = create_xattr_items(inode, id, xat, bytes, lck);
+		ret = create_xattr_items(inode, id, xat, xat_bytes, value, size,
+					 xattr_nr_parts(xat), lck);
 	if (ret < 0)
 		goto release;
 
@@ -778,7 +826,7 @@ unlock:
 	scoutfs_unlock(sb, lck, SCOUTFS_LOCK_WRITE);
 	scoutfs_unlock(sb, totl_lock, SCOUTFS_LOCK_WRITE_ONLY);
 out:
-	vfree(xat);
+	kfree(xat);
 
 	return ret;
 }
@@ -807,7 +855,7 @@ ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer,
 	struct scoutfs_xattr *xat = NULL;
 	struct scoutfs_lock *lck = NULL;
 	struct scoutfs_key key;
-	unsigned int bytes;
+	unsigned int xat_bytes;
 	ssize_t total = 0;
 	u32 name_hash = 0;
 	bool is_hidden;
@@ -820,8 +868,8 @@ ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer,
 		id = *id_pos;
 
 	/* need a buffer large enough for all possible names */
-	bytes = sizeof(struct scoutfs_xattr) + SCOUTFS_XATTR_MAX_NAME_LEN;
-	xat = kmalloc(bytes, GFP_NOFS);
+	xat_bytes = first_item_bytes(SCOUTFS_XATTR_MAX_NAME_LEN, 0);
+	xat = kmalloc(xat_bytes, GFP_NOFS);
 	if (!xat) {
 		ret = -ENOMEM;
 		goto out;
@@ -834,8 +882,7 @@ ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer,
 	down_read(&si->xattr_rwsem);
 
 	for (;;) {
-		ret = get_next_xattr(inode, &key, xat, bytes,
-				     NULL, 0, name_hash, id, lck);
+		ret = get_next_xattr(inode, &key, xat, xat_bytes, NULL, 0, name_hash, id, lck);
 		if (ret < 0) {
 			if (ret == -ENOENT)
 				ret = total;

tests/.gitignore

@@ -1,7 +1,9 @@
 src/*.d
 src/createmany
+src/dumb_renameat2
 src/dumb_setxattr
 src/handle_cat
+src/handle_fsetxattr
 src/bulk_create_paths
 src/find_xattrs
 src/stage_tmpfile

tests/Makefile

@@ -3,8 +3,10 @@ SHELL := /usr/bin/bash
 
 # each binary command is built from a single .c file
 BIN := src/createmany			\
+	src/dumb_renameat2		\
 	src/dumb_setxattr		\
 	src/handle_cat			\
+	src/handle_fsetxattr		\
 	src/bulk_create_paths		\
 	src/stage_tmpfile		\
 	src/find_xattrs			\

tests/fenced-local-force-unmount.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/bash
+
+#
+# This fencing script is used for testing clusters of multiple mounts on
+# a single host.  It finds mounts to fence by looking for their rids and
+# only knows how to "fence" by using forced unmount.
+#
+
+echo "$0 running rid '$SCOUTFS_FENCED_REQ_RID' ip '$SCOUTFS_FENCED_REQ_IP' args '$@'"
+
+log() {
+	echo "$@" > /dev/stderr
+	exit 1
+}
+
+echo_fail() {
+	echo "$@" > /dev/stderr
+	exit 1
+}
+
+rid="$SCOUTFS_FENCED_REQ_RID"
+
+for fs in /sys/fs/scoutfs/*; do
+	[ ! -d "$fs" ] && continue
+
+	fs_rid="$(cat $fs/rid)" || \
+		echo_fail "failed to get rid in $fs"
+	if [ "$fs_rid" != "$rid" ]; then
+		continue
+	fi
+
+	nr="$(cat $fs/data_device_maj_min)" || \
+		echo_fail "failed to get data device major:minor in $fs"
+
+	mnts=$(findmnt -l -n -t scoutfs -o TARGET -S $nr) || \
+		echo_fail "findmnt -t scoutfs -S $nr failed"
+	for mnt in $mnts; do
+		umount -f "$mnt" || \
+			echo_fail "umout -f $mnt failed"
+	done
+done
+
+exit 0

tests/funcs/filter.sh

@@ -56,8 +56,11 @@ t_filter_dmesg()
 	re="$re|scoutfs .*: all clients recovered"
 	re="$re|scoutfs .* error: client rid.*lock recovery timed out"
 
-	# some tests mount w/o options
+	# we test bad devices and options
 	re="$re|scoutfs .* error: Required mount option \"metadev_path\" not found"
+	re="$re|scoutfs .* error: meta_super META flag not set"
+	re="$re|scoutfs .* error: could not open metadev:.*"
+	re="$re|scoutfs .* error: Unknown or malformed option,.*"
 
 	# in debugging kernels we can slow things down a bit
 	re="$re|hrtimer: interrupt took .*"
@@ -72,6 +75,12 @@ t_filter_dmesg()
 	re="$re|scoutfs .* error reading quorum block"
 	re="$re|scoutfs .* error .* writing quorum block"
 	re="$re|scoutfs .* error .* while checking to delete inode"
+	re="$re|scoutfs .* error .*writing btree blocks.*"
+	re="$re|scoutfs .* error .*writing super block.*"
+	re="$re|scoutfs .* error .* freeing merged btree blocks.*.looping commit del.*upd freeing item"
+	re="$re|scoutfs .* error .* freeing merged btree blocks.*.final commit del.upd freeing item"
+	re="$re|scoutfs .* error .*reading quorum block.*to update event.*"
+	re="$re|scoutfs .* error.*server failed to bind to.*"
 
 	egrep -v "($re)" 
 }

tests/funcs/fs.sh

@@ -75,6 +75,20 @@ t_fs_nrs()
 	seq 0 $((T_NR_MOUNTS - 1))
 }
 
+#
+# outputs "1" if the fs number has "1" in its quorum/is_leader file.
+# All other cases output 0, including the fs nr being a client which
+# won't have a quorum/ dir.
+#
+t_fs_is_leader()
+{
+	if [ "$(cat $(t_sysfs_path $i)/quorum/is_leader 2>/dev/null)" == "1" ]; then
+		echo "1"
+	else
+		echo "0"
+	fi
+}
+
 #
 # Output the mount nr of the current server.  This takes no steps to
 # ensure that the server doesn't shut down and have some other mount
@@ -83,7 +97,7 @@ t_fs_nrs()
 t_server_nr()
 {
 	for i in $(t_fs_nrs); do
-		if [ "$(cat $(t_sysfs_path $i)/quorum/is_leader)" == "1" ]; then
+		if [ "$(t_fs_is_leader $i)" == "1" ]; then
 			echo $i
 			return
 		fi
@@ -101,7 +115,7 @@ t_server_nr()
 t_first_client_nr()
 {
 	for i in $(t_fs_nrs); do
-		if [ "$(cat $(t_sysfs_path $i)/quorum/is_leader)" == "0" ]; then
+		if [ "$(t_fs_is_leader $i)" == "0" ]; then
 			echo $i
 			return
 		fi
@@ -362,3 +376,49 @@ t_wait_for_leader() {
 		done
 	done
 }
+
+t_set_sysfs_mount_option() {
+	local nr="$1"
+	local name="$2"
+	local val="$3"
+	local opt="$(t_sysfs_path $nr)/mount_options/$name"
+
+	echo "$val" > "$opt"
+}
+
+t_set_all_sysfs_mount_options() {
+	local name="$1"
+	local val="$2"
+	local i
+
+	for i in $(t_fs_nrs); do
+		t_set_sysfs_mount_option $i $name $val
+	done
+}
+
+declare -A _saved_opts
+t_save_all_sysfs_mount_options() {
+	local name="$1"
+	local ind
+	local opt
+	local i
+
+	for i in $(t_fs_nrs); do
+		opt="$(t_sysfs_path $i)/mount_options/$name"
+		ind="$name_$i"
+
+		_saved_opts[$ind]="$(cat $opt)"
+	done
+}
+
+t_restore_all_sysfs_mount_options() {
+	local name="$1"
+	local ind
+	local i
+
+	for i in $(t_fs_nrs); do
+		ind="$name_$i"
+
+		t_set_sysfs_mount_option $i $name "${_saved_opts[$ind]}"
+	done
+}

tests/golden/basic-bad-mounts

@@ -0,0 +1,6 @@
+== prepare devices, mount point, and logs
+== bad devices, bad options
+== swapped devices
+== both meta devices
+== both data devices
+== good volume, bad option and good options

tests/golden/block-stale-reads

@@ -1,52 +1,2 @@
-== create shared test file
-== set and get xattrs between mount pairs while retrying
-# file: /mnt/test/test/block-stale-reads/file
-user.xat="1"
-
-counter block_cache_remove_stale changed
-counter block_cache_remove_stale changed
-# file: /mnt/test/test/block-stale-reads/file
-user.xat="2"
-
-counter block_cache_remove_stale changed
-counter block_cache_remove_stale changed
-# file: /mnt/test/test/block-stale-reads/file
-user.xat="3"
-
-counter block_cache_remove_stale changed
-counter block_cache_remove_stale changed
-# file: /mnt/test/test/block-stale-reads/file
-user.xat="4"
-
-counter block_cache_remove_stale changed
-counter block_cache_remove_stale changed
-# file: /mnt/test/test/block-stale-reads/file
-user.xat="5"
-
-counter block_cache_remove_stale changed
-counter block_cache_remove_stale changed
-# file: /mnt/test/test/block-stale-reads/file
-user.xat="6"
-
-counter block_cache_remove_stale changed
-counter block_cache_remove_stale changed
-# file: /mnt/test/test/block-stale-reads/file
-user.xat="7"
-
-counter block_cache_remove_stale changed
-counter block_cache_remove_stale changed
-# file: /mnt/test/test/block-stale-reads/file
-user.xat="8"
-
-counter block_cache_remove_stale changed
-counter block_cache_remove_stale changed
-# file: /mnt/test/test/block-stale-reads/file
-user.xat="9"
-
-counter block_cache_remove_stale changed
-counter block_cache_remove_stale changed
-# file: /mnt/test/test/block-stale-reads/file
-user.xat="10"
-
-counter block_cache_remove_stale changed
+== Issue scoutfs df to force block reads to trigger stale invalidation/retry
 counter block_cache_remove_stale changed

tests/golden/client-unmount-recovery

@@ -0,0 +1 @@
+== 60s of unmounting non-quorum clients during recovery

tests/golden/fallocate

@@ -0,0 +1,3 @@
+== creating reasonably large per-mount files
+== 10s of racing cold reads and fallocate nop
+== cleaning up files

tests/golden/lock-recover-invalidate

@@ -0,0 +1,3 @@
+== starting background invalidating read/write load
+== 60s of lock recovery during invalidating load
+== stopping background load

tests/golden/orphan-inodes

@@ -2,3 +2,4 @@
 == unlinked and opened inodes still exist
 == orphan from failed evict deletion is picked up
 == orphaned inos in all mounts all deleted
+== 30s of racing evict deletion, orphan scanning, and open by handle

tests/golden/renameat2-noreplace

@@ -0,0 +1,2 @@
+=== renameat2 noreplace flag test
+=== run two asynchronous calls to renameat2 NOREPLACE

tests/golden/xfstests

@@ -9,6 +9,8 @@ generic/011
 generic/013
 generic/014
 generic/020
+generic/023
+generic/024
 generic/028
 generic/032
 generic/034
@@ -82,6 +84,7 @@ generic/016
 generic/018
 generic/021
 generic/022
+generic/025
 generic/026
 generic/031
 generic/033
@@ -93,6 +96,7 @@ generic/060
 generic/061
 generic/063
 generic/064
+generic/078
 generic/079
 generic/081
 generic/082
@@ -278,4 +282,4 @@ shared/004
 shared/032
 shared/051
 shared/289
-Passed all 73 tests
+Passed all 75 tests

tests/run-tests.sh

@@ -227,8 +227,9 @@ test "$T_QUORUM" -le "$T_NR_MOUNTS" || \
 	 die "-q quorum mmembers must not be greater than -n mounts"
 
 # top level paths
-T_KMOD=$(realpath "$(dirname $0)/../kmod")
-T_UTILS=$(realpath "$T_KMOD/../utils")
+T_TESTS=$(realpath "$(dirname $0)")
+T_KMOD=$(realpath "$T_TESTS/../kmod")
+T_UTILS=$(realpath "$T_TESTS/../utils")
 
 test -d "$T_KMOD" || die "kmod/ repo dir $T_KMOD not directory"
 test -d "$T_UTILS" || die "utils/ repo dir $T_UTILS not directory"
@@ -379,13 +380,14 @@ cmd grep .  /sys/kernel/debug/tracing/options/trace_printk \
 # Build a fenced config that runs scripts out of the repository rather
 # than the default system directory
 #
-conf="$T_RESULTS/scoutfs-fencd.conf"
+conf="$T_RESULTS/scoutfs-fenced.conf"
 cat > $conf << EOF
 SCOUTFS_FENCED_DELAY=1
-SCOUTFS_FENCED_RUN=$T_UTILS/fenced/local-force-unmount
-SCOUTFS_FENCED_RUN_ARGS=""
+SCOUTFS_FENCED_RUN=$T_TESTS/fenced-local-force-unmount.sh
+SCOUTFS_FENCED_RUN_ARGS="ignored run args"
 EOF
 export SCOUTFS_FENCED_CONFIG_FILE="$conf"
+T_FENCED_LOG="$T_RESULTS/fenced.log"
 
 #
 # Run the agent in the background, log its output, an kill it if we
@@ -393,7 +395,7 @@ export SCOUTFS_FENCED_CONFIG_FILE="$conf"
 #
 fenced_log()
 {
-	echo "[$(timestamp)] $*" >> "$T_RESULTS/fenced.stdout.log"
+	echo "[$(timestamp)] $*" >> "$T_FENCED_LOG"
 }
 fenced_pid=""
 kill_fenced()
@@ -404,7 +406,7 @@ kill_fenced()
 	fi
 }
 trap kill_fenced EXIT
-$T_UTILS/fenced/scoutfs-fenced > "$T_RESULTS/fenced.stdout.log" 2> "$T_RESULTS/fenced.stderr.log" &
+$T_UTILS/fenced/scoutfs-fenced > "$T_FENCED_LOG" 2>&1 &
 fenced_pid=$!
 fenced_log "started fenced pid $fenced_pid in the background"
 

tests/sequence

@@ -1,9 +1,11 @@
 export-get-name-parent.sh
 basic-block-counts.sh
+basic-bad-mounts.sh
 inode-items-updated.sh
 simple-inode-index.sh
 simple-staging.sh
 simple-release-extents.sh
+fallocate.sh
 setattr_more.sh
 offline-extent-waiting.sh
 move-blocks.sh
@@ -15,6 +17,7 @@ lock-refleak.sh
 lock-shrink-consistency.sh
 lock-pr-cw-conflict.sh
 lock-revoke-getcwd.sh
+lock-recover-invalidate.sh
 export-lookup-evict-race.sh
 createmany-parallel.sh
 createmany-large-names.sh
@@ -33,8 +36,10 @@ resize-devices.sh
 fence-and-reclaim.sh
 orphan-inodes.sh
 mount-unmount-race.sh
+client-unmount-recovery.sh
 createmany-parallel-mounts.sh
 archive-light-cycle.sh
 block-stale-reads.sh
 inode-deletion.sh
+renameat2-noreplace.sh
 xfstests.sh

tests/src/dumb_renameat2.c

@@ -0,0 +1,93 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <fcntl.h>
+
+#ifndef RENAMEAT2_EXIST
+#include <unistd.h>
+#include <sys/syscall.h>
+
+#if !defined(SYS_renameat2) && defined(__x86_64__)
+#define SYS_renameat2 316			/* from arch/x86/entry/syscalls/syscall_64.tbl */
+#endif
+
+static int renameat2(int olddfd, const char *old_dir,
+		     int newdfd, const char *new_dir,
+		     unsigned int flags)
+{
+#ifdef SYS_renameat2
+	return syscall(SYS_renameat2, olddfd, old_dir, newdfd, new_dir, flags);
+#else
+	errno = ENOSYS;
+	return -1;
+#endif
+}
+#endif
+
+#ifndef RENAME_NOREPLACE
+#define RENAME_NOREPLACE	(1 << 0)	/* Don't overwrite newpath of rename */
+#endif
+#ifndef RENAME_EXCHANGE
+#define RENAME_EXCHANGE		(1 << 1)	/* Exchange oldpath and newpath */
+#endif
+#ifndef RENAME_WHITEOUT
+#define RENAME_WHITEOUT		(1 << 2)	/* Whiteout oldpath */
+#endif
+
+static void exit_usage(char **argv)
+{
+	fprintf(stderr,
+		"usage: %s [-n|-x|-w] old_path new_path\n"
+		"  -n  noreplace\n"
+		"  -x  exchange\n"
+		"  -w  whiteout\n", argv[0]);
+
+		exit(1);
+}
+
+int main(int argc, char **argv)
+{
+	const char *old_path = NULL;
+	const char *new_path = NULL;
+	unsigned int flags = 0;
+	int ret;
+	int c;
+
+	for (c = 1; c < argc; c++) {
+		if (argv[c][0] == '-') {
+			switch (argv[c][1]) {
+				case 'n':
+					flags |= RENAME_NOREPLACE;
+					break;
+				case 'x':
+					flags |= RENAME_EXCHANGE;
+					break;
+				case 'w':
+					flags |= RENAME_WHITEOUT;
+					break;
+				default:
+					exit_usage(argv);
+			}
+		} else if (!old_path) {
+			old_path = argv[c];
+		} else if (!new_path) {
+			new_path = argv[c];
+		} else {
+			exit_usage(argv);
+		}
+	}
+
+	if (!old_path || !new_path) {
+		printf("specify the correct directory path\n");
+		errno = ENOENT;
+		return 1;
+	}
+
+	ret = renameat2(AT_FDCWD, old_path, AT_FDCWD, new_path, flags);
+	if (ret == -1) {
+		perror("Error");
+		return 1;
+	}
+
+	return 0;
+}

tests/src/handle_fsetxattr.c

@@ -0,0 +1,189 @@
+/*
+ * Copyright (C) 2022 Versity Software, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <errno.h>
+#include <string.h>
+#include <endian.h>
+#include <time.h>
+#include <linux/types.h>
+#include <sys/xattr.h>
+
+#define FILEID_SCOUTFS			0x81
+#define FILEID_SCOUTFS_WITH_PARENT	0x82
+
+struct our_handle {
+	struct file_handle handle;
+	/*
+	 * scoutfs file handle can be ino or ino/parent. The
+	 * handle_type field of struct file_handle denotes which
+	 * version is in use. We only use the ino variant here.
+	 */
+	__le64 scoutfs_ino;
+};
+
+#define DEFAULT_NAME "user.handle_fsetxattr"
+#define DEFAULT_VALUE "value"
+
+static void exit_usage(void)
+{
+	printf(" -h/-?         output this usage message and exit\n"
+	       " -e            keep trying on enoent, consider success an error\n"
+	       " -i <num>      64bit inode number for handle open, can be multiple\n"
+	       " -m <string>   scoutfs mount path string for ioctl fd\n"
+	       " -n <string>   optional xattr name string, defaults to \""DEFAULT_NAME"\"\n"
+	       " -s <num>      loop for num seconds, defaults to 0 for one iteration"
+	       " -v <string>   optional xattr value string, defaults to \""DEFAULT_VALUE"\"\n");
+	exit(1);
+}
+
+int main(int argc, char **argv)
+{
+	struct our_handle handle;
+	struct timespec ts;
+	bool enoent_success_err = false;
+	uint64_t seconds = 0;
+	char *value = NULL;
+	char *name = NULL;
+	char *mnt = NULL;
+	int nr_inos = 0;
+	uint64_t *inos;
+	uint64_t i;
+	int *fds;
+	int mntfd;
+	int fd;
+	int ret;
+	char c;
+	int j;
+
+	/* can't have more inos than args */
+	inos = calloc(argc, sizeof(inos[0]));
+	fds = calloc(argc, sizeof(fds[0]));
+	if (!inos || !fds) {
+		perror("calloc");
+		exit(1);
+	}
+	for (i = 0; i < argc; i++)
+		fds[i] = -1;
+
+	while ((c = getopt(argc, argv, "+ei:m:n:s:v:")) != -1) {
+		switch (c) {
+			case 'e':
+				enoent_success_err = true;
+				break;
+			case 'i':
+				inos[nr_inos] = strtoll(optarg, NULL, 0);
+				nr_inos++;
+				break;
+			case 'm':
+				mnt = strdup(optarg);
+				break;
+			case 'n':
+				name = strdup(optarg);
+				break;
+			case 's':
+				seconds = strtoll(optarg, NULL, 0);
+				break;
+			case 'v':
+				value = strdup(optarg);
+				break;
+			case '?':
+				printf("unknown argument: %c\n", optind);
+			case 'h':
+				exit_usage();
+		}
+	}
+
+	if (nr_inos == 0) {
+		printf("specify non-zero inode number with -i\n");
+		exit(1);
+	}
+
+	if (!mnt) {
+		printf("specify scoutfs mount path for ioctl with -p\n");
+		exit(1);
+	}
+
+	if (name == NULL)
+		name = DEFAULT_NAME;
+	if (value == NULL)
+		value = DEFAULT_VALUE;
+
+	mntfd = open(mnt, O_RDONLY);
+	if (mntfd == -1) {
+		perror("opening mountpoint");
+		return 1;
+	}
+
+	clock_gettime(CLOCK_REALTIME, &ts);
+	seconds += ts.tv_sec;
+
+	for (i = 0; ; i++) {
+		for (j = 0; j < nr_inos; j++) {
+			fd = fds[j];
+
+			if (fd < 0) {
+				handle.handle.handle_bytes = sizeof(struct our_handle);
+				handle.handle.handle_type = FILEID_SCOUTFS;
+				handle.scoutfs_ino = htole64(inos[j]);
+
+				fd = open_by_handle_at(mntfd, &handle.handle, O_RDWR);
+				if (fd == -1) {
+					if (!enoent_success_err || errno != ENOENT) {
+						perror("open_by_handle_at");
+						return 1;
+					}
+					continue;
+				}
+				fds[j] = fd;
+			}
+
+			ret = fsetxattr(fd, name, value, strlen(value), 0);
+			if (ret < 0) {
+				perror("fsetxattr");
+				return 1;
+			}
+		}
+
+		if ((i % 10) == 0) {
+			clock_gettime(CLOCK_REALTIME, &ts);
+			if (ts.tv_sec >= seconds)
+				break;
+		}
+	}
+
+	if (enoent_success_err) {
+		bool able = false;
+		for (i = 0; i < nr_inos; i++) {
+			if (fds[i] >= 0) {
+				printf("was able to open ino %"PRIu64"\n", inos[i]);
+				able = true;
+			}
+		}
+		if (able)
+			exit(1);
+	}
+
+	/* not bothering to close or free */
+	return 0;
+}

tests/tests/basic-bad-mounts.sh

@@ -0,0 +1,36 @@
+
+mount_fail()
+{
+	local mnt=${!#}
+
+	echo "mounting $@" >> $T_TMP.mount.out
+	mount -t scoutfs "$@" >> $T_TMP.mount.out 2>&1
+	if [ $? == 0 ]; then
+		umount "$mnt" || t_fail "couldn't unmount"
+		t_fail "bad mount succeeded"
+	fi
+}
+
+echo "== prepare devices, mount point, and logs"
+SCR="/mnt/scoutfs.extra"
+mkdir -p "$SCR"
+> $T_TMP.mount.out
+scoutfs mkfs -f -Q 0,127.0.0.1,53000 "$T_EX_META_DEV" "$T_EX_DATA_DEV" > $T_TMP.mkfs.out 2>&1 \
+	|| t_fail "mkfs failed"
+
+echo "== bad devices, bad options"
+mount_fail -o _bad /dev/null /dev/null "$SCR"
+
+echo "== swapped devices"
+mount_fail -o metadev_path=$T_EX_DATA_DEV,quorum_slot_nr=0 "$T_EX_META_DEV" "$SCR"
+
+echo "== both meta devices"
+mount_fail -o metadev_path=$T_EX_META_DEV,quorum_slot_nr=0 "$T_EX_META_DEV" "$SCR"
+
+echo "== both data devices"
+mount_fail -o metadev_path=$T_EX_DATA_DEV,quorum_slot_nr=0 "$T_EX_DATA_DEV" "$SCR"
+
+echo "== good volume, bad option and good options"
+mount_fail -o _bad,metadev_path=$T_EX_META_DEV,quorum_slot_nr=0 "$T_EX_DATA_DEV" "$SCR" 
+
+t_pass

tests/tests/block-stale-reads.sh

@@ -5,57 +5,18 @@
 # persistent blocks to create stable block reading scenarios.    Instead
 # we use triggers to exercise how readers encounter stale blocks.
 #
+# Trigger retries in the block cache by calling scoutfs df
+# which in turn will call scoutfs_ioctl_alloc_detail. This
+# is guaranteed to exist, which will force block cache reads.
 
-t_require_commands touch setfattr getfattr
+echo "== Issue scoutfs df to force block reads to trigger stale invalidation/retry"
+nr=0
 
-inc_wrap_fs_nr()
-{
-	local nr="$(($1 + 1))"
+old=$(t_counter block_cache_remove_stale $nr)
+t_trigger_arm_silent block_remove_stale $nr
 
-	if [ "$nr" == "$T_NR_MOUNTS" ]; then
-		nr=0
-	fi
+scoutfs df -p "$T_M0" > /dev/null
 
-	echo $nr
-}
-
-GETFATTR="getfattr --absolute-names"
-SETFATTR="setfattr"
-
-echo "== create shared test file"
-touch "$T_D0/file"
-$SETFATTR -n user.xat -v 0 "$T_D0/file"
-
-#
-# Trigger retries in the block cache as we bounce xattr values around
-# between sequential pairs of mounts.  This is a little silly because if
-# either of the mounts are the server then they'll almost certaily have
-# their trigger fired prematurely by message handling btree calls while
-# working with the t_ helpers long before we work with the xattrs.  But
-# the block cache stale retry path is still being exercised.
-#
-echo "== set and get xattrs between mount pairs while retrying"
-set_nr=0
-get_nr=$(inc_wrap_fs_nr $set_nr)
-
-for i in $(seq 1 10); do
-	eval set_file="\$T_D${set_nr}/file"
-	eval get_file="\$T_D${get_nr}/file"
-
-	old_set=$(t_counter block_cache_remove_stale $set_nr)
-	old_get=$(t_counter block_cache_remove_stale $get_nr)
-
-	t_trigger_arm_silent block_remove_stale $set_nr
-	t_trigger_arm_silent block_remove_stale $get_nr
-
-	$SETFATTR -n user.xat -v $i "$set_file"
-	$GETFATTR -n user.xat "$get_file" 2>&1 | t_filter_fs
-
-	t_counter_diff_changed block_cache_remove_stale $old_set $set_nr
-	t_counter_diff_changed block_cache_remove_stale $old_get $get_nr
-
-	set_nr="$get_nr"
-	get_nr=$(inc_wrap_fs_nr $set_nr)
-done
+t_counter_diff_changed block_cache_remove_stale $old $nr
 
 t_pass

tests/tests/client-unmount-recovery.sh

@@ -0,0 +1,61 @@
+#
+# Unmount Server and unmount a client as it's replaying to a remaining server
+#
+
+majority_nr=$(t_majority_count)
+quorum_nr=$T_QUORUM
+
+test "$quorum_nr" == "$majority_nr" && \
+        t_skip "all quorum members make up majority, need more mounts to unmount"
+
+test "$T_NR_MOUNTS" -lt "$T_QUORUM" && \
+        t_skip "Need enough non-quorum clients to unmount"
+
+for i in $(t_fs_nrs); do
+        mounted[$i]=1
+done
+
+LENGTH=60
+echo "== ${LENGTH}s of unmounting non-quorum clients during recovery"
+END=$((SECONDS + LENGTH))
+while [ "$SECONDS" -lt "$END" ]; do
+        sv=$(t_server_nr)
+        rid=$(t_mount_rid $sv)
+        echo "sv $sv rid $rid" >> "$T_TMP.log"
+        sync
+        t_umount $sv &
+
+        for i in $(t_fs_nrs); do
+                if [ "$i" -ge "$quorum_nr" ]; then
+                        t_umount $i &
+                        echo "umount $i pid $pid quo $quorum_nr" \
+                                >> $T_TMP.log
+                        mounted[$i]=0
+                fi
+        done
+
+        wait
+
+        t_mount $sv &
+        for i in $(t_fs_nrs); do
+                if [ "${mounted[$i]}" == 0 ]; then
+                        t_mount $i &
+                fi
+        done
+
+        wait
+
+        declare RID_LIST=$(cat /sys/fs/scoutfs/*/rid | sort -u)
+        read -a rid_arr <<< $RID_LIST
+
+        declare LOCK_LIST=$(cut -d' ' -f 5 /sys/kernel/debug/scoutfs/*/server_locks | sort -u)
+        read -a lock_arr <<< $LOCK_LIST
+
+        for i in "${lock_arr[@]}"; do
+                if [[ ! " ${rid_arr[*]} " =~ " $i " ]]; then
+                        t_fail "RID($i): exists when not mounted"
+                fi
+        done
+done
+
+t_pass

tests/tests/fallocate.sh

@@ -0,0 +1,38 @@
+
+t_require_commands fallocate cat
+
+echo "== creating reasonably large per-mount files"
+for n in $(t_fs_nrs); do
+	eval path="\$T_D${n}/file-$n"
+
+	LC_ALL=C fallocate -l 128MiB  "$path" || \
+		t_fail "initial creating fallocate failed"
+done
+
+#
+# we had lock inversions between read and fallocate, dropping
+# the cache each time forces waiting for IO during the calls
+# with the inverted locks held so we have a better chance
+# of the deadlock happening.
+#
+DURATION=10
+echo "== ${DURATION}s of racing cold reads and fallocate nop"
+END=$((SECONDS + DURATION))
+while [ $SECONDS -le $END ]; do
+
+	echo 3 > /proc/sys/vm/drop_caches
+
+	for n in $(t_fs_nrs); do
+		eval path="\$T_D${n}/file-$n"
+		
+		LC_ALL=C fallocate -o 0 -l 4KiB  "$path" &
+		cat "$path" > /dev/null &
+	done
+
+	wait || t_fail "fallocate or cat failed"
+done
+
+echo "== cleaning up files"
+rm -f "$T_D0"/file-*
+
+t_pass

tests/tests/fence-and-reclaim.sh

@@ -45,6 +45,18 @@ check_read_write()
 	fi
 }
 
+# verify that fenced ran our testing fence script
+verify_fenced_run()
+{
+	local rids="$@"
+	local rid
+
+	for rid in $rids; do
+		grep -q ".* running rid '$rid'.* args 'ignored run args'" "$T_FENCED_LOG" || \
+			t_fail "fenced didn't execute RUN script for rid $rid"
+	done
+}
+
 echo "== make sure all mounts can see each other"
 check_read_write
 
@@ -62,12 +74,14 @@ done
 while t_rid_is_fencing $rid; do
 	sleep .5
 done
+verify_fenced_run $rid
 t_mount $cl
 check_read_write
 
 echo "== force unmount all non-server, connection timeout, fence nop, mount"
 sv=$(t_server_nr)
 pattern="nonsense"
+rids=""
 sync
 for cl in $(t_fs_nrs); do
 	if [ $cl == $sv ]; then
@@ -75,6 +89,7 @@ for cl in $(t_fs_nrs); do
 	fi
 
 	rid=$(t_mount_rid $cl)
+	rids="$rids $rid"
 	pattern="$pattern|$rid"
 	echo "cl $cl sv $sv rid $rid" >> "$T_TMP.log"
 
@@ -89,6 +104,7 @@ done
 while test -d $(echo /sys/fs/scoutfs/*/fence/* | cut -d " " -f 1); do
 	sleep .5
 done
+verify_fenced_run $rids
 # remount all the clients
 for cl in $(t_fs_nrs); do
 	if [ $cl == $sv ]; then
@@ -109,11 +125,17 @@ t_wait_for_leader
 while t_rid_is_fencing $rid; do
 	sleep .5
 done
+verify_fenced_run $rid
 t_mount $sv
 check_read_write
 
 echo "== force unmount everything, new server fences all previous"
 sync
+rids=""
+# get rids before forced unmount breaks scoutfs statfs
+for nr in $(t_fs_nrs); do
+	rids="$rids $(t_mount_rid $nr)"
+done
 for nr in $(t_fs_nrs); do
 	t_force_umount $nr
 done
@@ -122,6 +144,7 @@ t_mount_all
 while test -d $(echo /sys/fs/scoutfs/*/fence/* | cut -d " " -f 1); do
 	sleep .5
 done
+verify_fenced_run $rids
 check_read_write
 
 t_pass

tests/tests/lock-recover-invalidate.sh

@@ -0,0 +1,43 @@
+#
+# trigger server failover and lock recovery during heavy invalidating
+# load on multiple mounts
+#
+
+majority_nr=$(t_majority_count)
+quorum_nr=$T_QUORUM
+
+test "$quorum_nr" == "$majority_nr" && \
+        t_skip "need remaining majority when leader unmounted"
+
+test "$T_NR_MOUNTS" -lt "$((quorum_nr + 2))" && \
+        t_skip "need at least 2 non-quorum load mounts"
+
+echo "== starting background invalidating read/write load"
+touch "$T_D0/file"
+load_pids=""
+for i in $(t_fs_nrs); do
+	if [ "$i" -ge "$quorum_nr" ]; then
+		eval path="\$T_D${i}/file"
+
+		(while true; do touch $path > /dev/null 2>&1; done) &
+		load_pids="$load_pids $!"
+		(while true; do stat $path > /dev/null 2>&1; done) &
+		load_pids="$load_pids $!"
+	fi
+done
+
+# had it reproduce in ~40s on wimpy debug kernel guests
+LENGTH=60
+echo "== ${LENGTH}s of lock recovery during invalidating load"
+END=$((SECONDS + LENGTH))
+while [ "$SECONDS" -lt "$END" ]; do
+        sv=$(t_server_nr)
+        t_umount $sv
+        t_mount $sv
+	# new server had to process greeting for mount to finish
+done
+
+echo "== stopping background load"
+kill $load_pids
+
+t_pass

tests/tests/orphan-inodes.sh

@@ -26,9 +26,17 @@ inode_exists()
 {
 	local ino="$1"
 
-	handle_cat "$T_M0" "$ino" > "$T_TMP.handle_cat.log" 2>&1
+	scoutfs get-allocated-inos -i "$ino" -s -p "$T_M0" > $T_TMP.inos.log 2>&1
+	test "$?" == 0 -a "$(head -1 $T_TMP.inos.log)" == "$ino"
 }
 
+t_save_all_sysfs_mount_options orphan_scan_delay_ms
+restore_delays()
+{
+	t_restore_all_sysfs_mount_options orphan_scan_delay_ms
+}
+trap restore_delays EXIT
+
 echo "== test our inode existance function"
 path="$T_D0/file"
 touch "$path"
@@ -37,6 +45,7 @@ inode_exists $ino || echo "$ino didn't exist"
 
 echo "== unlinked and opened inodes still exist"
 sleep 1000000 < "$path" &
+sleep .1 # wait for background sleep to run and open stdin
 pid="$!"
 rm -f "$path"
 inode_exists $ino || echo "$ino didn't exist"
@@ -44,7 +53,8 @@ inode_exists $ino || echo "$ino didn't exist"
 echo "== orphan from failed evict deletion is picked up"
 # pending kill signal stops evict from getting locks and deleting
 silent_kill $pid
-sleep 55
+t_set_sysfs_mount_option 0 orphan_scan_delay_ms 1000
+sleep 5
 inode_exists $ino && echo "$ino still exists"
 
 echo "== orphaned inos in all mounts all deleted"
@@ -55,6 +65,7 @@ for nr in $(t_fs_nrs); do
 	touch "$path"
 	inos="$inos $(stat -c %i $path)"
 	sleep 1000000 < "$path" &
+	sleep .1 # wait for background sleep to run and open stdin
 	pids="$pids $!"
 	rm -f "$path"
 done
@@ -69,9 +80,63 @@ while test -d $(echo /sys/fs/scoutfs/*/fence/* | cut -d " " -f 1); do
 	sleep .5
 done
 # wait for orphan scans to run
-sleep 55
+t_set_all_sysfs_mount_options orphan_scan_delay_ms 1000
+# also have to wait for delayed log merge work from mount
+sleep 15
 for ino in $inos; do
 	inode_exists $ino && echo "$ino still exists"
 done
 
+RUNTIME=30
+echo "== ${RUNTIME}s of racing evict deletion, orphan scanning, and open by handle"
+
+# exclude last client mount
+last=""
+for nr in $(t_fs_nrs); do
+	last=$nr
+done
+
+END=$((SECONDS + RUNTIME))
+while [ $SECONDS -lt $END ]; do
+	# hold open per-mount unlinked files
+	pids=""
+	ino_args=""
+	for nr in $(t_fs_nrs); do
+		test $nr == $last && continue
+
+		eval path="\$T_D${nr}/racing-$nr"
+		touch "$path"
+		ino_args="$ino_args -i $(stat -c %i $path)"
+
+		sleep 1000000 < "$path" &
+		sleep .1 # wait for sleep to start and open input :/
+		pids="$pids $!"
+		rm -f "$path"
+	done
+
+	# remount excluded last client to force log merging and make orphan visible
+	sync
+	t_umount $last
+	t_mount $last
+
+	# get all mounts scanning orphans at high frequency
+	t_set_all_sysfs_mount_options orphan_scan_delay_ms 100
+
+	# spin having tasks in each mount trying to open/fsetxattr all inos
+	for nr in $(t_fs_nrs); do
+		test $nr == $last && continue
+
+		eval path="\$T_M${nr}"
+		handle_fsetxattr -e $ino_args -m "$path" -s 2 &
+	done
+
+	# trigger eviction deletion of each file in each mount
+	silent_kill $pids
+
+	wait || t_fail "handle_fsetxattr failed"
+
+	# slow down orphan scanning for the next iteration
+	t_set_all_sysfs_mount_options orphan_scan_delay_ms $(((RUNTIME * 2) * 1000))
+done
+
 t_pass

tests/tests/renameat2-noreplace.sh

@@ -0,0 +1,37 @@
+#
+# simple renameat2 NOREPLACE unit test
+#
+
+t_require_commands dumb_renameat2
+t_require_mounts 2
+
+echo "=== renameat2 noreplace flag test"
+
+# give each mount their own dir (lock group) to minimize create contention
+mkdir $T_M0/dir0
+mkdir $T_M1/dir1
+
+echo "=== run two asynchronous calls to renameat2 NOREPLACE"
+for i in $(seq 0 100); do
+        # prepare inputs in isolation
+        touch "$T_M0/dir0/old0"
+        touch "$T_M1/dir1/old1"
+
+        # race doing noreplace renames, both can't succeed
+        dumb_renameat2 -n "$T_M0/dir0/old0" "$T_M0/dir0/sharednew" 2> /dev/null &
+        pid0=$!
+        dumb_renameat2 -n "$T_M1/dir1/old1" "$T_M1/dir0/sharednew" 2> /dev/null &
+        pid1=$!
+
+        wait $pid0
+        rc0=$?
+        wait $pid1
+        rc1=$?
+
+        test "$rc0" == 0 -a "$rc1" == 0 && t_fail "both renames succeeded"
+
+        # blow away possible files for either race outcome
+        rm -f "$T_M0/dir0/old0" "$T_M1/dir1/old1" "$T_M0/dir0/sharednew" "$T_M1/dir1/sharednew"
+done
+
+t_pass

tests/tests/xfstests.sh

@@ -60,13 +60,9 @@ EOF
 
 cat << EOF > local.exclude
 generic/003	# missing atime update in buffered read
-generic/023	# renameat2 not implemented
-generic/024	# renameat2 not implemented
-generic/025	# renameat2 not implemented
 generic/029	# mmap missing
 generic/030	# mmap missing
 generic/075	# file content mismatch failures (fds, etc)
-generic/078	# renameat2 not implemented
 generic/080	# mmap missing
 generic/103	# enospc causes trans commit failures
 generic/105	# needs trigage: something about acls

utils/fenced/local-force-unmount

@@ -1,35 +0,0 @@
-#!/usr/bin/bash
-
-echo_fail() {
-	echo "$@" > /dev/stderr
-	exit 1
-}
-
-rid="$SCOUTFS_FENCED_REQ_RID"
-
-#
-# Look for a local mount with the rid to fence.  Typically we'll at
-# least find the mount with the server that requested the fence that
-# we're processing.   But it's possible that mounts are unmounted
-# before, or while, we're running.
-#
-mnts=$(findmnt -l -n -t scoutfs -o TARGET) || \
-	echo_fail "findmnt -t scoutfs failed" > /dev/stderr
-
-for mnt in $mnts; do
-	mnt_rid=$(scoutfs statfs -p "$mnt" -s rid) || \
-		echo_fail "scoutfs statfs $mnt failed"
-
-	if [ "$mnt_rid" == "$rid" ]; then
-		umount -f "$mnt" || \
-			echo_fail "umout -f $mnt"
-
-		exit 0
-	fi
-done
-
-#
-# If the mount doesn't exist on this host then it can't access the
-# devices by definition and can be considered fenced.
-#
-exit 0

utils/fenced/scoutfs-fenced

@@ -55,9 +55,21 @@ test -x "$SCOUTFS_FENCED_RUN" || \
 	error_exit "SCOUTFS_FENCED_RUN '$SCOUTFS_FENCED_RUN' isn't executable"
 
 #
-# main loop watching for fence request across all filesystems 
+# Main loop watching for fence request across all filesystems.   The
+# server can shut down without waiting for pending fence requests to
+# finish.  All of the interaction with the fence directory and files can
+# fail at any moment.  We will generate log messages when the dir or
+# files disappear.
 #
 
+# generate failure messages to stderr while still echoing 0 for the caller
+careful_cat()
+{
+	local path="$@"
+
+	cat "$@" || echo 0
+}
+
 while sleep $SCOUTFS_FENCED_DELAY; do
 	for fence in /sys/fs/scoutfs/*/fence/*; do
 		# catches unmatched regex when no dirs
@@ -66,7 +78,8 @@ while sleep $SCOUTFS_FENCED_DELAY; do
 		fi
 
 		# skip requests that have been handled
-		if [ $(cat "$fence/fenced") == 1 -o $(cat "$fence/error") == 1 ]; then
+		if [ "$(careful_cat $fence/fenced)" == 1 -o \
+		     "$(careful_cat $fence/error)" == 1 ]; then
 			continue
 		fi
 
@@ -81,10 +94,10 @@ while sleep $SCOUTFS_FENCED_DELAY; do
 		export SCOUTFS_FENCED_REQ_RID="$rid"
 		export SCOUTFS_FENCED_REQ_IP="$ip"
 
-		$run $SCOUTFS_FENCED_RUN_ARGS
+		$SCOUTFS_FENCED_RUN $SCOUTFS_FENCED_RUN_ARGS
 		rc=$?
 		if [ "$rc" != 0 ]; then
-			log_message "server $srv fencing rid $rid saw error status $rc from $run"
+			log_message "server $srv fencing rid $rid saw error status $rc"
 			echo 1 > "$fence/error"
 			continue
 		fi

utils/fenced/scoutfs-fenced.conf.example

@@ -1,3 +1,6 @@
+# delay, in seconds, between each check for pending fence requests.
 SCOUTFS_FENCED_DELAY=1
-SCOUTFS_FENCED_RUN=/usr/libexec/scoutfs-fenced/run/local-force-unmount
+# path to executable to run to service fence request
+#SCOUTFS_FENCED_RUN=
+# arguments to pass to binary
 SCOUTFS_FENCED_RUN_ARGS=""

utils/man/scoutfs.5

@@ -21,6 +21,21 @@ contains the filesystem's metadata.
 .sp
 This option is required.
 .TP
+.B orphan_scan_delay_ms=<number>
+This option sets the average expected delay, in milliseconds, between
+each mount's scan of the global orphaned inode list.  Jitter is added to
+avoid contention so each individual delay between scans is a random
+value up to 20% less than or greater than this average expected delay.
+.sp
+The minimum value for this option is 100ms which is very short and is
+only reasonable for testing or experiments.   The default is 10000ms (10
+seconds) and the maximum is 60000ms (1 minute).
+.sp
+This option can be changed in an active mount by writing to its file in
+the options directory in the mount's sysfs directory.  Writing a new
+value will cause the next pending orphan scan to be rescheduled
+with the newly written delay time.
+.TP
 .B quorum_slot_nr=<number>
 The quorum_slot_nr option assigns a quorum member slot to the mount.
 The mount will use the slot assignment to claim exclusive ownership of

utils/man/scoutfs.8

@@ -15,7 +15,7 @@ environment variable. If that variable is also absent the current working
 directory will be used.
 
 .TP
-.BI "change-format-version [-V, --format-version VERS] [-F|--offline META-DEVICE DATA-DEVICE]"
+.BI "change-format-version [-V, --format-version VERS] [-F|--offline] META-DEVICE DATA-DEVICE"
 .sp
 Change the format version of an existing file system.  The maxmimum
 supported version is used by default.   A specific version in the range
@@ -25,7 +25,7 @@ output of --help.
 .PD 0
 .TP
 .sp
-.B "-F, --offline META-DEVICE DATA-DEVICE"
+.B "-F, --offline"
 Change the format version by writing directly to the metadata and data
 devices.   Like mkfs, this writes directly to the devices without
 protection and must only be used on completely unmounted devices.   The
@@ -42,6 +42,40 @@ the super blocks on both devices.
 .RE
 .PD
 
+.TP
+.BI "change-quorum-config {-Q|--quorum-slot NR,ADDR,PORT} [-F|--offline] META-DEVICE"
+.sp
+Change the quorum configuration for an existing file system.  The new
+configuration completely replaces the old configuration.   Any slots
+from the old configuration that should be retained must be described
+with arguments in the new configuration.
+.sp
+Currently the configuration may only be changed offline.
+.sp
+.RS 1.0i
+.PD 0
+.TP
+.B "-Q, --quorum-slot NR,ADDR,PORT"
+The quorum configuration is built by specifying configured slots with
+multiple arguments as described in the
+.B mkfs
+command.
+.TP
+.B "-F, --offline"
+Perform the change offline by updating the superblock in the metadata
+device.   The command will read the super block and refuse to make the
+change if it sees any evidence that the metadata device is currently in
+use.   The file system must be successfully unmounted after possibly
+recovering any previously unresolved mounts for the change to be
+successful.   After the change succeeds the newly configured slots can
+be used by mounts.
+.sp
+The offline change directly reads from and writes to the device and does
+not protect against concurrent use of the device.   It must be carefully
+run when the file system will not be mounted.
+.RE
+.PD
+
 .TP
 .BI "df [-h|--human-readable] [-p|--path PATH]"
 .sp
@@ -563,7 +597,7 @@ format.
 .PD
 
 .TP
-.BI "print META-DEVICE"
+.BI "print {-S|--skip-likely-huge} META-DEVICE"
 .sp
 Prints out all of the metadata in the file system.  This makes no effort
 to ensure that the structures are consistent as they're traversed and
@@ -573,6 +607,20 @@ output.
 .PD 0
 .TP
 .sp
+.B "-S, --skip-likely-huge"
+Skip printing structures that are likely to be very large.  The
+structures that are skipped tend to be global and whose size tends to be
+related to the size of the volume.   Examples of skipped structures include
+the global fs items, srch files, and metadata and data
+allocators.  Similar structures that are not skipped are related to the
+number of mounts and are maintained at a relatively reasonable size.
+These include per-mount log trees, srch files, allocators, and the
+metadata allocators used by server commits.
+.sp
+Skipping the larger structures limits the print output to a relatively
+constant size rather than being a large multiple of the used metadata
+space of the volume making the output much more useful for inspection.
+.TP
 .B "META-DEVICE"
 The path to the metadata device for the filesystem whose metadata will be
 printed.  Since this command reads via the host's buffer cache, it may not
@@ -583,6 +631,33 @@ command is used first.
 .RE
 .PD
 
+.TP
+.BI "get-allocated-inos [-i|--ino INO] [-s|--single] [-p|--path PATH]"
+.sp
+This debugging command prints allocated inode numbers.  It only prints
+inodes
+found in the group that contains the starting inode.  The printed inode
+numbers aren't necessarily reachable.  They could be anywhere in the
+process from being unlinked to finally deleted when their items
+were found.
+.RS 1.0i
+.PD 0
+.TP
+.sp
+.B "-i, --ino INO"
+The first 64bit inode number which could be printed.
+.TP
+.B "-s, --single"
+Only print the single starting inode when it is allocated, all other allocated
+inode numbers will be ignored.
+.TP
+.B "-p, --path PATH"
+A path within a ScoutFS filesystem.
+.RE
+.PD
+
+.TP
+
 .SH SEE ALSO
 .BR scoutfs (5),
 .BR xattr (7),

utils/scoutfs-utils.spec.in

@@ -55,7 +55,6 @@ install -m 755 -D src/scoutfs $RPM_BUILD_ROOT%{_sbindir}/scoutfs
 install -m 644 -D src/ioctl.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/ioctl.h
 install -m 644 -D src/format.h $RPM_BUILD_ROOT%{_includedir}/scoutfs/format.h
 install -m 755 -D fenced/scoutfs-fenced $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/scoutfs-fenced
-install -m 755 -D fenced/local-force-unmount $RPM_BUILD_ROOT%{_libexecdir}/scoutfs-fenced/run/local-force-unmount
 install -m 644 -D fenced/scoutfs-fenced.service $RPM_BUILD_ROOT%{_unitdir}/scoutfs-fenced.service
 install -m 644 -D fenced/scoutfs-fenced.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/scoutfs/scoutfs-fenced.conf.example
 

utils/src/change_format_version.c

@@ -45,16 +45,11 @@ static int do_change_fmt_vers(struct change_fmt_vers_args *args)
 {
 	struct scoutfs_super_block *meta_super = NULL;
 	struct scoutfs_super_block *data_super = NULL;
-	struct scoutfs_quorum_block *qblk = NULL;
-	struct scoutfs_quorum_block_event *beg;
-	struct scoutfs_quorum_block_event *end;
 	bool wrote_meta = false;
-	bool in_use = false;
 	char uuid_str[37];
 	int meta_fd = -1;
 	int data_fd = -1;
 	int ret;
-	int i;
 
 	meta_fd = open(args->meta_device, O_DIRECT | O_SYNC | O_RDWR | O_EXCL);
 	if (meta_fd < 0) {
@@ -117,44 +112,13 @@ static int do_change_fmt_vers(struct change_fmt_vers_args *args)
 		goto out;
 	}
 
-	if (meta_super->mounted_clients.ref.blkno != 0) {
-		fprintf(stderr, "meta superblock mounted clients btree is not empty.\n");
-		ret = -EBUSY;
-		in_use = true;
+	ret = meta_super_in_use(meta_fd, meta_super);
+	if (ret < 0) {
+		if (ret == -EBUSY)
+			fprintf(stderr, "The filesystem must be fully recovered and cleanly unmounted to change the format version\n");
 		goto out;
 	}
 
-	/* check for active quorum slots */
-	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
-		if (!quorum_slot_present(meta_super, i))
-			continue;
-		ret = read_block(meta_fd, SCOUTFS_QUORUM_BLKNO + i, SCOUTFS_BLOCK_SM_SHIFT,
-				 (void **)&qblk);
-		if (ret < 0) {
-			fprintf(stderr, "error reading quorum block for slot %u\n", i);
-			goto out;
-		}
-
-		beg = &qblk->events[SCOUTFS_QUORUM_EVENT_BEGIN];
-		end = &qblk->events[SCOUTFS_QUORUM_EVENT_END];
-
-		if (le64_to_cpu(beg->write_nr) > le64_to_cpu(end->write_nr)) {
-			fprintf(stderr, "mount in quorum slot %u could still be running.\n"
-					"  begin event: write_nr %llu timestamp %llu.%08u\n"
-					"    end event: write_nr %llu timestamp %llu.%08u\n",
-				i, le64_to_cpu(beg->write_nr), le64_to_cpu(beg->ts.sec),
-				le32_to_cpu(beg->ts.nsec),
-				le64_to_cpu(end->write_nr), le64_to_cpu(end->ts.sec),
-				le32_to_cpu(end->ts.nsec));
-			ret = -EBUSY;
-			in_use = true;
-			goto out;
-		}
-
-		free(qblk);
-		qblk = NULL;
-	}
-
 	if (le64_to_cpu(meta_super->fmt_vers) != args->fmt_vers) {
 		meta_super->fmt_vers = cpu_to_le64(args->fmt_vers);
 
@@ -195,11 +159,7 @@ static int do_change_fmt_vers(struct change_fmt_vers_args *args)
 		le64_to_cpu(meta_super->fmt_vers));
 
 out:
-	if (in_use)
-		fprintf(stderr, "The filesystem must be fully recovered and cleanly unmounted to change the format version\n");
 
-	if (qblk)
-		free(qblk);
 	if (meta_super)
 		free(meta_super);
 	if (data_super)
@@ -262,7 +222,7 @@ static struct argp_option options[] = {
 static struct argp argp = {
 	options,
 	parse_opt,
-	"",
+	"META-DEVICE DATA-DEVICE",
 	"Change format version of an existing ScoutFS filesystem"
 };
 

utils/src/change_quorum_config.c

@@ -0,0 +1,171 @@
+#define _GNU_SOURCE /* O_DIRECT */
+#include <unistd.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <uuid/uuid.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <assert.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <inttypes.h>
+#include <argp.h>
+
+#include "sparse.h"
+#include "cmd.h"
+#include "util.h"
+#include "format.h"
+#include "parse.h"
+#include "dev.h"
+#include "quorum.h"
+
+struct change_quorum_args {
+	char *meta_device;
+	bool offline;
+	int nr_slots;
+	struct scoutfs_quorum_slot slots[SCOUTFS_QUORUM_MAX_SLOTS];
+};
+
+static int do_change_quorum(struct change_quorum_args *args)
+{
+	struct scoutfs_super_block *meta_super = NULL;
+	char uuid_str[37];
+	int meta_fd = -1;
+	int ret;
+
+	meta_fd = open(args->meta_device, O_DIRECT | O_SYNC | O_RDWR | O_EXCL);
+	if (meta_fd < 0) {
+		ret = -errno;
+		fprintf(stderr, "failed to open meta device '%s': %s (%d)\n",
+			args->meta_device, strerror(errno), errno);
+		goto out;
+	}
+
+	ret = read_block_verify(meta_fd, SCOUTFS_BLOCK_MAGIC_SUPER, 0, SCOUTFS_SUPER_BLKNO,
+				SCOUTFS_BLOCK_SM_SHIFT, (void **)&meta_super);
+	if (ret) {
+		ret = -errno;
+		fprintf(stderr, "failed to read meta super block: %s (%d)\n",
+			strerror(errno), errno);
+		goto out;
+	}
+
+	ret = meta_super_in_use(meta_fd, meta_super);
+	if (ret < 0) {
+		if (ret == -EBUSY)
+			fprintf(stderr, "The filesystem must be fully recovered and cleanly unmounted to change the quorum config\n");
+		goto out;
+	}
+
+	assert(sizeof(meta_super->qconf.slots) == sizeof(args->slots));
+	memcpy(meta_super->qconf.slots, args->slots, sizeof(meta_super->qconf.slots));
+	le64_add_cpu(&meta_super->qconf.version, 1);
+
+	ret = write_block(meta_fd, SCOUTFS_BLOCK_MAGIC_SUPER, meta_super->hdr.fsid, 1,
+			  SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT, &meta_super->hdr);
+	if (ret)
+		goto out;
+
+	uuid_unparse(meta_super->uuid, uuid_str);
+
+	printf("Successfully changed quorum config for scoutfs filesystem:\n"
+	       "  meta device path:       %s\n"
+	       "  fsid:                   %llx\n"
+	       "  uuid:                   %s\n"
+	       "  quorum config version:  %llu\n"
+	       "  quorum slots:           ",
+		args->meta_device,
+		le64_to_cpu(meta_super->hdr.fsid),
+		uuid_str,
+		le64_to_cpu(meta_super->qconf.version));
+
+	print_quorum_slots(meta_super->qconf.slots, array_size(meta_super->qconf.slots),
+			   "                          ");
+
+out:
+
+	if (meta_super)
+		free(meta_super);
+	if (meta_fd != -1)
+		close(meta_fd);
+	return ret;
+}
+
+static int parse_opt(int key, char *arg, struct argp_state *state)
+{
+	struct change_quorum_args *args = state->input;
+	struct scoutfs_quorum_slot slot;
+	int ret;
+
+	switch (key) {
+	case 'F':
+		args->offline = true;
+		break;
+	case 'Q':
+		ret = parse_quorum_slot(&slot, arg);
+		if (ret < 0)
+			return ret;
+		if (args->slots[ret].addr.v4.family != cpu_to_le16(SCOUTFS_AF_NONE))
+			argp_error(state, "Quorum slot %u already specified before slot '%s'\n",
+				   ret, arg);
+		args->slots[ret] = slot;
+		args->nr_slots++;
+		break;
+	case ARGP_KEY_ARG:
+		if (!args->meta_device)
+			args->meta_device = strdup_or_error(state, arg);
+		else
+			argp_error(state, "more than one metadata device argument given");
+		break;
+	case ARGP_KEY_FINI:
+		if (!args->offline)
+			argp_error(state, "must specify --offline");
+		if (!args->meta_device)
+			argp_error(state, "no metadata device argument given");
+		if (!args->nr_slots)
+			argp_error(state, "must specify at least one quorum slot with --quorum-slot|-Q");
+		if (!valid_quorum_slots(args->slots))
+			argp_error(state, "invalid quorum slot configuration");
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static struct argp_option options[] = {
+	{ "quorum-slot", 'Q', "NR,ADDR,PORT", 0, "Specify quorum slot addresses [Required]"},
+	{ "offline", 'F', NULL, 0, "Write format version in offline device super blocks [Currently Required]"},
+	{ NULL }
+};
+
+static struct argp argp = {
+	options,
+	parse_opt,
+	"META-DEVICE",
+	"Change quorum slots and addresses of an existing ScoutFS filesystem"
+};
+
+static int change_quorum_cmd(int argc, char *argv[])
+{
+	struct change_quorum_args change_quorum_args = {
+		.offline = false,
+	};
+	int ret;
+
+	ret = argp_parse(&argp, argc, argv, 0, NULL, &change_quorum_args);
+	if (ret)
+		return ret;
+
+	return do_change_quorum(&change_quorum_args);
+}
+
+static void __attribute__((constructor)) change_quorum_ctor(void)
+{
+	cmd_register_argp("change-quorum-config", &argp, GROUP_CORE, change_quorum_cmd);
+}

utils/src/get_allocated_inos.c

@@ -0,0 +1,137 @@
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <string.h>
+#include <argp.h>
+
+#include "sparse.h"
+#include "parse.h"
+#include "util.h"
+#include "format.h"
+#include "ioctl.h"
+#include "cmd.h"
+
+struct get_allocated_inos_args {
+	char *path;
+	u64 ino;
+	bool have_ino;
+	bool single;
+};
+
+static int do_get_allocated_inos(struct get_allocated_inos_args *args)
+{
+	struct scoutfs_ioctl_get_allocated_inos gai;
+	u64 *inos = NULL;
+	int fd = -1;
+	u64 bytes;
+	int ret;
+	int i;
+
+	if (args->single)
+		bytes = sizeof(*inos);
+	else
+		bytes = SCOUTFS_LOCK_INODE_GROUP_NR * sizeof(*inos);
+
+	inos = malloc(bytes);
+	if (!inos) {
+		fprintf(stderr, "inode number array allocation failed\n");
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	fd = get_path(args->path, O_RDONLY);
+	if (fd < 0)
+		return fd;
+
+	memset(&gai, 0, sizeof(gai));
+	gai.start_ino = args->ino;
+	gai.inos_ptr = (unsigned long)inos;
+	gai.inos_bytes = bytes;
+
+	ret = ioctl(fd, SCOUTFS_IOC_GET_ALLOCATED_INOS, &gai);
+	if (ret < 0) {
+		ret = -errno;
+		fprintf(stderr, "get_allocated_inos ioctl failed: "
+			"%s (%d)\n", strerror(errno), errno);
+		goto out;
+	}
+
+	if (args->single && ret > 0 && inos[0] != args->ino)
+		ret = 0;
+
+	for (i = 0; i < ret; i++)
+		printf("%llu\n", inos[i]);
+
+	ret = 0;
+out:
+	if (fd >= 0)
+		close(fd);
+	free(inos);
+
+	return ret;
+};
+
+static int parse_opt(int key, char *arg, struct argp_state *state)
+{
+	struct get_allocated_inos_args *args = state->input;
+	int ret;
+
+	switch (key) {
+	case 'i':
+		ret = parse_u64(arg, &args->ino);
+		if (ret)
+			return ret;
+		args->have_ino = true;
+	case 'p':
+		args->path = strdup_or_error(state, arg);
+		break;
+	case 's':
+		args->single = true;
+		break;
+	case ARGP_KEY_FINI:
+		if (!args->have_ino)
+			argp_error(state, "must provide --ino starting inode number option");
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static struct argp_option options[] = {
+	{ "ino", 'i', "NUMBER", 0, "Start from 64bit inode number (required)"},
+	{ "path", 'p', "PATH", 0, "Path to ScoutFS filesystem"},
+	{ "single", 's', NULL, 0, "Only print single specific inode number argument"},
+	{ NULL }
+};
+
+static struct argp argp = {
+	options,
+	parse_opt,
+	NULL,
+	"Print allocated inode numbers from starting inode number"
+};
+
+static int get_allocated_inos_cmd(int argc, char **argv)
+{
+
+	struct get_allocated_inos_args get_allocated_inos_args = {NULL};
+	int ret;
+
+	ret = argp_parse(&argp, argc, argv, 0, NULL, &get_allocated_inos_args);
+	if (ret)
+		return ret;
+
+	return do_get_allocated_inos(&get_allocated_inos_args);
+}
+
+static void __attribute__((constructor)) get_allocated_inos_ctor(void)
+{
+	cmd_register_argp("get-allocated-inos", &argp, GROUP_DEBUG, get_allocated_inos_cmd);
+}

utils/src/mkfs.c

@@ -31,6 +31,7 @@
 #include "btree.h"
 #include "leaf_item_hash.h"
 #include "blkid.h"
+#include "quorum.h"
 
 
 /*
@@ -139,7 +140,6 @@ static int do_mkfs(struct mkfs_args *args)
 	int data_fd = -1;
 	char uuid_str[37];
 	void *zeros = NULL;
-	char *indent;
 	u64 blkno;
 	u64 meta_size;
 	u64 data_size;
@@ -224,6 +224,7 @@ static int do_mkfs(struct mkfs_args *args)
 	assert(sizeof(args->slots) ==
 		     member_sizeof(struct scoutfs_super_block, qconf.slots));
 	memcpy(super->qconf.slots, args->slots, sizeof(args->slots));
+	super->qconf.version = cpu_to_le64(1);
 
 	if (invalid_data_alloc_zone_blocks(le64_to_cpu(super->total_data_blocks),
 					   args->data_alloc_zone_blocks)) {
@@ -350,14 +351,15 @@ static int do_mkfs(struct mkfs_args *args)
 	uuid_unparse(super->uuid, uuid_str);
 
 	printf("Created scoutfs filesystem:\n"
-	       "  meta device path:     %s\n"
-	       "  data device path:     %s\n"
-	       "  fsid:                 %llx\n"
-	       "  uuid:                 %s\n"
-	       "  format version:       %llu\n"
-	       "  64KB metadata blocks: "SIZE_FMT"\n"
-	       "  4KB data blocks:      "SIZE_FMT"\n"
-	       "  quorum slots:         ",
+	       "  meta device path:       %s\n"
+	       "  data device path:       %s\n"
+	       "  fsid:                   %llx\n"
+	       "  uuid:                   %s\n"
+	       "  format version:         %llu\n"
+	       "  64KB metadata blocks:   "SIZE_FMT"\n"
+	       "  4KB data blocks:        "SIZE_FMT"\n"
+	       "  quorum config version:  %llu\n"
+	       "  quorum slots:           ",
 		args->meta_device,
 	        args->data_device,
 		le64_to_cpu(super->hdr.fsid),
@@ -366,22 +368,11 @@ static int do_mkfs(struct mkfs_args *args)
 		SIZE_ARGS(le64_to_cpu(super->total_meta_blocks),
 			  SCOUTFS_BLOCK_LG_SIZE),
 		SIZE_ARGS(le64_to_cpu(super->total_data_blocks),
-			  SCOUTFS_BLOCK_SM_SIZE));
+			  SCOUTFS_BLOCK_SM_SIZE),
+		le64_to_cpu(super->qconf.version));
 
-	indent = "";
-	for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
-		struct scoutfs_quorum_slot *sl = &super->qconf.slots[i];
-		struct in_addr in;
-
-		if (sl->addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4))
-			continue;
-
-		in.s_addr = htonl(le32_to_cpu(sl->addr.v4.addr));
-		printf("%s%u: %s:%u", indent,
-		       i, inet_ntoa(in), le16_to_cpu(sl->addr.v4.port));
-		indent = "\n                        ";
-	}
-	printf("\n");
+	print_quorum_slots(super->qconf.slots, array_size(super->qconf.slots),
+			   "                          ");
 
 	ret = 0;
 out:
@@ -398,45 +389,6 @@ out:
 	return ret;
 }
 
-static bool valid_quorum_slots(struct scoutfs_quorum_slot *slots)
-{
-	struct in_addr in;
-	bool valid = true;
-	char *addr;
-	int i;
-	int j;
-
-	for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
-		if (slots[i].addr.v4.family == cpu_to_le16(SCOUTFS_AF_NONE))
-			continue;
-
-		if (slots[i].addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4)) {
-			fprintf(stderr, "quorum slot nr %u has invalid family %u\n",
-				i, le16_to_cpu(slots[i].addr.v4.family));
-			valid = false;
-		}
-
-		for (j = i + 1; j < SCOUTFS_QUORUM_MAX_SLOTS; j++) {
-			if (slots[i].addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4))
-				continue;
-
-			if (slots[i].addr.v4.addr == slots[j].addr.v4.addr &&
-			    slots[i].addr.v4.port == slots[j].addr.v4.port) {
-
-				in.s_addr =
-					htonl(le32_to_cpu(slots[i].addr.v4.addr));
-				addr = inet_ntoa(in);
-				fprintf(stderr, "quorum slot nr %u and %u have the same address %s:%u\n",
-					i, j, addr,
-					le16_to_cpu(slots[i].addr.v4.port));
-				valid = false;
-			}
-		}
-	}
-
-	return valid;
-}
-
 static int parse_opt(int key, char *arg, struct argp_state *state)
 {
 	struct mkfs_args *args = state->input;
@@ -517,7 +469,7 @@ static int parse_opt(int key, char *arg, struct argp_state *state)
 		break;
 	case ARGP_KEY_FINI:
 		if (!args->nr_slots)
-			argp_error(state, "must specify at least one quorum slot with --quorum-count|-Q");
+			argp_error(state, "must specify at least one quorum slot with --quorum-slot|-Q");
 		if (!args->meta_device)
 			argp_error(state, "no metadata device argument given");
 		if (!args->data_device)

utils/src/print.c

@@ -8,6 +8,7 @@
 #include <errno.h>
 #include <string.h>
 #include <stdarg.h>
+#include <stdbool.h>
 #include <ctype.h>
 #include <uuid/uuid.h>
 #include <sys/socket.h>
@@ -278,6 +279,8 @@ static int print_log_trees_item(struct scoutfs_key *key, u64 seq, u8 flags, void
 		       "      data_freed: "ALCROOT_F"\n"
 		       "      srch_file: "SRF_FMT"\n"
 		       "      inode_count_delta: %lld\n"
+		       "      get_trans_seq: %lld\n"
+		       "      commit_trans_seq: %lld\n"
 		       "      max_item_seq: %llu\n"
 		       "      finalize_seq: %llu\n"
 		       "      rid: %016llx\n"
@@ -296,6 +299,8 @@ static int print_log_trees_item(struct scoutfs_key *key, u64 seq, u8 flags, void
 		       ALCROOT_A(&lt->data_freed),
 		       SRF_A(&lt->srch_file),
 		       le64_to_cpu(lt->inode_count_delta),
+		       le64_to_cpu(lt->get_trans_seq),
+		       le64_to_cpu(lt->commit_trans_seq),
 		       le64_to_cpu(lt->max_item_seq),
 		       le64_to_cpu(lt->finalize_seq),
 		       le64_to_cpu(lt->rid),
@@ -922,10 +927,6 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
 
 	uuid_unparse(super->uuid, uuid_str);
 
-	if (!(le64_to_cpu(super->flags) && SCOUTFS_FLAG_IS_META_BDEV))
-	    fprintf(stderr,
-		    "**** Printing metadata from a data device! Did you mean to do this? ****\n");
-
 	printf("super blkno %llu\n", blkno);
 	print_block_header(&super->hdr, SCOUTFS_BLOCK_SM_SIZE);
 	printf("  fmt_vers %llu uuid %s\n",
@@ -989,9 +990,10 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
 
 struct print_args {
 	char *meta_device;
+	bool skip_likely_huge;
 };
 
-static int print_volume(int fd)
+static int print_volume(int fd, struct print_args *args)
 {
 	struct scoutfs_super_block *super = NULL;
 	struct print_recursion_args pa;
@@ -1006,6 +1008,13 @@ static int print_volume(int fd)
 
 	print_super_block(super, SCOUTFS_SUPER_BLKNO);
 
+	if (!(le64_to_cpu(super->flags) & SCOUTFS_FLAG_IS_META_BDEV)) {
+		fprintf(stderr,
+			"**** Printing from data device is not allowed ****\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
 	ret = print_quorum_blocks(fd, super);
 
 	err = print_btree(fd, super, "mounted_clients", &super->mounted_clients,
@@ -1034,23 +1043,26 @@ static int print_volume(int fd)
 			ret = err;
 	}
 
-	for (i = 0; i < array_size(super->meta_alloc); i++) {
-		snprintf(str, sizeof(str), "meta_alloc[%u]", i);
-		err = print_btree(fd, super, str, &super->meta_alloc[i].root,
+	if (!args->skip_likely_huge) {
+		for (i = 0; i < array_size(super->meta_alloc); i++) {
+			snprintf(str, sizeof(str), "meta_alloc[%u]", i);
+			err = print_btree(fd, super, str, &super->meta_alloc[i].root,
+					  print_alloc_item, NULL);
+			if (err && !ret)
+				ret = err;
+		}
+
+		err = print_btree(fd, super, "data_alloc", &super->data_alloc.root,
 				  print_alloc_item, NULL);
 		if (err && !ret)
 			ret = err;
 	}
 
-	err = print_btree(fd, super, "data_alloc", &super->data_alloc.root,
-			  print_alloc_item, NULL);
-	if (err && !ret)
-		ret = err;
-
 	err = print_btree(fd, super, "srch_root", &super->srch_root,
 			  print_srch_root_item, NULL);
 	if (err && !ret)
 		ret = err;
+
 	err = print_btree(fd, super, "logs_root", &super->logs_root,
 			  print_log_trees_item, NULL);
 	if (err && !ret)
@@ -1058,20 +1070,25 @@ static int print_volume(int fd)
 
 	pa.super = super;
 	pa.fd = fd;
-	err = print_btree_leaf_items(fd, super, &super->srch_root.ref,
-				     print_srch_root_files, &pa);
-	if (err && !ret)
-		ret = err;
+	if (!args->skip_likely_huge) {
+		err = print_btree_leaf_items(fd, super, &super->srch_root.ref,
+					     print_srch_root_files, &pa);
+		if (err && !ret)
+			ret = err;
+	}
 	err = print_btree_leaf_items(fd, super, &super->logs_root.ref,
 				     print_log_trees_roots, &pa);
 	if (err && !ret)
 		ret = err;
 
-	err = print_btree(fd, super, "fs_root", &super->fs_root,
-			  print_fs_item, NULL);
-	if (err && !ret)
-		ret = err;
+	if (!args->skip_likely_huge) {
+		err = print_btree(fd, super, "fs_root", &super->fs_root,
+				  print_fs_item, NULL);
+		if (err && !ret)
+			ret = err;
+	}
 
+out:
 	free(super);
 
 	return ret;
@@ -1090,7 +1107,7 @@ static int do_print(struct print_args *args)
 		return ret;
 	}
 
-	ret = print_volume(fd);
+	ret = print_volume(fd, args);
 	close(fd);
 	return ret;
 };
@@ -1100,6 +1117,9 @@ static int parse_opt(int key, char *arg, struct argp_state *state)
 	struct print_args *args = state->input;
 
 	switch (key) {
+	case 'S':
+		args->skip_likely_huge = true;
+		break;
 	case ARGP_KEY_ARG:
 		if (!args->meta_device)
 			args->meta_device = strdup_or_error(state, arg);
@@ -1117,8 +1137,13 @@ static int parse_opt(int key, char *arg, struct argp_state *state)
 	return 0;
 }
 
+static struct argp_option options[] = {
+	{ "skip-likely-huge", 'S', NULL, 0, "Skip large structures to minimize output size"},
+	{ NULL }
+};
+
 static struct argp argp = {
-	NULL,
+	options,
 	parse_opt,
 	"META-DEV",
 	"Print metadata structures"

utils/src/quorum.c

@@ -1,3 +1,7 @@
+#include <stdio.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
 #include "sparse.h"
 #include "util.h"
 #include "format.h"
@@ -8,3 +12,68 @@ bool quorum_slot_present(struct scoutfs_super_block *super, int i)
 {
 	return super->qconf.slots[i].addr.v4.family == cpu_to_le16(SCOUTFS_AF_IPV4);
 }
+
+bool valid_quorum_slots(struct scoutfs_quorum_slot *slots)
+{
+	struct in_addr in;
+	bool valid = true;
+	char *addr;
+	int i;
+	int j;
+
+	for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
+		if (slots[i].addr.v4.family == cpu_to_le16(SCOUTFS_AF_NONE))
+			continue;
+
+		if (slots[i].addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4)) {
+			fprintf(stderr, "quorum slot nr %u has invalid family %u\n",
+				i, le16_to_cpu(slots[i].addr.v4.family));
+			valid = false;
+		}
+
+		for (j = i + 1; j < SCOUTFS_QUORUM_MAX_SLOTS; j++) {
+			if (slots[i].addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4))
+				continue;
+
+			if (slots[i].addr.v4.addr == slots[j].addr.v4.addr &&
+			    slots[i].addr.v4.port == slots[j].addr.v4.port) {
+
+				in.s_addr =
+					htonl(le32_to_cpu(slots[i].addr.v4.addr));
+				addr = inet_ntoa(in);
+				fprintf(stderr, "quorum slot nr %u and %u have the same address %s:%u\n",
+					i, j, addr,
+					le16_to_cpu(slots[i].addr.v4.port));
+				valid = false;
+			}
+		}
+	}
+
+	return valid;
+}
+
+/*
+ * Print quorum slots to stdout, a line at a time.   The first line is
+ * not indented and the rest of the lines use the indent string from the
+ * caller.
+ */
+void print_quorum_slots(struct scoutfs_quorum_slot *slots, int nr, char *indent)
+{
+	struct scoutfs_quorum_slot *sl;
+	struct in_addr in;
+	bool first = true;
+	int i;
+
+	for (i = 0, sl = slots; i < SCOUTFS_QUORUM_MAX_SLOTS; i++, sl++) {
+
+		if (sl->addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4))
+			continue;
+
+		in.s_addr = htonl(le32_to_cpu(sl->addr.v4.addr));
+		printf("%s%u: %s:%u\n", first ? "" : indent,
+		       i, inet_ntoa(in), le16_to_cpu(sl->addr.v4.port));
+
+		first = false;
+	}
+}
+

utils/src/quorum.h

@@ -4,5 +4,7 @@
 #include <stdbool.h>
 
 bool quorum_slot_present(struct scoutfs_super_block *super, int i);
+bool valid_quorum_slots(struct scoutfs_quorum_slot *slots);
+void print_quorum_slots(struct scoutfs_quorum_slot *slots, int nr, char *indent);
 
 #endif

utils/src/util.c

@@ -12,6 +12,7 @@
 #include "util.h"
 #include "format.h"
 #include "crc.h"
+#include "quorum.h"
 
 #define ENV_PATH "SCOUTFS_MOUNT_PATH"
 
@@ -201,3 +202,56 @@ int write_block_sync(int fd, u32 magic, __le64 fsid, u64 seq, u64 blkno,
 
 	return 0;
 }
+
+/*
+ * Check to see if the metadata super block indicates that there might
+ * be active mounts using the system.  Returns -errno, 0, or -EBUSY if
+ * we found evidence that the device might be in use.
+ */
+int meta_super_in_use(int meta_fd, struct scoutfs_super_block *meta_super)
+{
+	struct scoutfs_quorum_block *qblk = NULL;
+	struct scoutfs_quorum_block_event *beg;
+	struct scoutfs_quorum_block_event *end;
+	int ret = 0;
+	int i;
+
+	if (meta_super->mounted_clients.ref.blkno != 0) {
+		fprintf(stderr, "meta superblock mounted clients btree is not empty.\n");
+		ret = -EBUSY;
+		goto out;
+	}
+
+	/* check for active quorum slots */
+	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
+		if (!quorum_slot_present(meta_super, i))
+			continue;
+		ret = read_block(meta_fd, SCOUTFS_QUORUM_BLKNO + i, SCOUTFS_BLOCK_SM_SHIFT,
+				 (void **)&qblk);
+		if (ret < 0) {
+			fprintf(stderr, "error reading quorum block for slot %u\n", i);
+			goto out;
+		}
+
+		beg = &qblk->events[SCOUTFS_QUORUM_EVENT_BEGIN];
+		end = &qblk->events[SCOUTFS_QUORUM_EVENT_END];
+
+		if (le64_to_cpu(beg->write_nr) > le64_to_cpu(end->write_nr)) {
+			fprintf(stderr, "mount in quorum slot %u could still be running.\n"
+					"  begin event: write_nr %llu timestamp %llu.%08u\n"
+					"    end event: write_nr %llu timestamp %llu.%08u\n",
+				i, le64_to_cpu(beg->write_nr), le64_to_cpu(beg->ts.sec),
+				le32_to_cpu(beg->ts.nsec),
+				le64_to_cpu(end->write_nr), le64_to_cpu(end->ts.sec),
+				le32_to_cpu(end->ts.nsec));
+			ret = -EBUSY;
+			goto out;
+		}
+
+		free(qblk);
+		qblk = NULL;
+	}
+
+out:
+	return ret;
+}

utils/src/util.h

@@ -117,10 +117,12 @@ int read_block_crc(int fd, u64 blkno, int shift, void **ret_val);
 int read_block_verify(int fd, u32 magic, u64 fsid, u64 blkno, int shift, void **ret_val);
 
 struct scoutfs_block_header;
+struct scoutfs_super_block;
 int write_block(int fd, u32 magic, __le64 fsid, u64 seq, u64 blkno,
 		int shift, struct scoutfs_block_header *hdr);
 int write_block_sync(int fd, u32 magic, __le64 fsid, u64 seq, u64 blkno,
 		     int shift, struct scoutfs_block_header *hdr);
+int meta_super_in_use(int meta_fd, struct scoutfs_super_block *meta_super);
 
 #define __stringify_1(x) #x
 #define __stringify(x) __stringify_1(x)