(wip) add some quick lock server stats

Currently the first inode number that can be allocated directly follows
the root inode.  This means the first batch of allocated inodes are in
the same lock group as the root inode.

The root inode is a bit special.  It is always hot as absolute path
lookups and inode-to-path resolution always read directory entries from
the root.

Let's try aligning the first free inode number to the next inode lock
group boundary.  This will stop work in those inodes from necessarily
conflicting with work in the root inode.

Signed-off-by: Zach Brown <zab@versity.com>

README.md

@@ -65,8 +65,13 @@ The steps for getting scoutfs mounted and operational are:
  2. Make a new filesystem on the devices with the userspace utilities
  3. Mount the devices on all the nodes
 
-In this example we run all of these commands on three nodes.  The names
-of the block devices are the same on all the nodes.
+In this example we use three nodes.  The names of the block devices are
+the same on all the nodes.  Two of the nodes will be quorum members.  A
+majority of quorum members must be mounted to elect a leader to run a
+server that all the mounts connect to.  It should be noted that two
+quorum members results in a majority of one, each member itself, so
+split brain elections are possible but so unlikely that it's fine for a
+demonstration.
 
 1. Get the Kernel Module and Userspace Binaries
 
@@ -88,24 +93,30 @@ of the block devices are the same on all the nodes.
    alias scoutfs=$PWD/scoutfs/utils/src/scoutfs
    ```
 
-2. Make a New Filesystem (**destroys contents, no questions asked**)
+2. Make a New Filesystem (**destroys contents**)
 
-   We specify that two of our three nodes must be present to form a
-   quorum for the system to function.
+   We specify quorum slots with the addresses of each of the quorum
+   member nodes, the metadata device, and the data device.
 
    ```shell
-   scoutfs mkfs -Q 2 /dev/meta_dev /dev/data_dev
+   scoutfs mkfs -Q 0,$NODE0_ADDR,12345 -Q 1,$NODE1_ADDR,12345 /dev/meta_dev /dev/data_dev
    ```
 
 3. Mount the Filesystem
 
-   Each mounting node provides its local IP address on which it will run
-   an internal server for the other mounts if it is elected the leader by
-   the quorum.
+   First, mount each of the quorum nodes so that they can elect and
+   start a server for the remaining node to connect to.  The slot numbers
+   were specified with the leading "0,..."  and "1,..." in the mkfs options
+   above.
 
    ```shell
-   mkdir /mnt/scoutfs
-   mount -t scoutfs -o server_addr=$NODE_ADDR,metadev_path=/dev/meta_dev /dev/data_dev /mnt/scoutfs
+   mount -t scoutfs -o quorum_slot_nr=$SLOT_NR,metadev_path=/dev/meta_dev /dev/data_dev /mnt/scoutfs
+   ```
+
+   Then mount the remaining node which can now connect to the running server.
+
+   ```shell
+   mount -t scoutfs -o metadev_path=/dev/meta_dev /dev/data_dev /mnt/scoutfs
    ```
 
 4. For Kicks, Observe the Metadata Change Index

kmod/src/Makefile

@@ -27,9 +27,11 @@ scoutfs-y +=			\
 	lock_server.o		\
 	msg.o			\
 	net.o			\
+	omap.o			\
 	options.o		\
 	per_task.o		\
 	quorum.o		\
+	recov.o			\
 	scoutfs_trace.o		\
 	server.o		\
 	sort_priv.o		\

kmod/src/alloc.c

@@ -252,7 +252,7 @@ void scoutfs_alloc_init(struct scoutfs_alloc *alloc,
 {
 	memset(alloc, 0, sizeof(struct scoutfs_alloc));
 
-	spin_lock_init(&alloc->lock);
+	seqlock_init(&alloc->seqlock);
 	mutex_init(&alloc->mutex);
 	alloc->avail = *avail;
 	alloc->freed = *freed;
@@ -358,31 +358,24 @@ static void list_block_sort(struct scoutfs_alloc_list_block *lblk)
 
 /*
  * We're always reading blocks that we own, so we shouldn't see stale
- * references.  But the cached block can be stale and we can need to
- * invalidate it.
+ * references but we could retry reads after dropping stale cached
+ * blocks.  If we do see a stale error then we've hit persistent
+ * corruption.
  */
-static int read_list_block(struct super_block *sb,
-			   struct scoutfs_alloc_list_ref *ref,
+static int read_list_block(struct super_block *sb, struct scoutfs_block_ref *ref,
 			   struct scoutfs_block **bl_ret)
 {
-	struct scoutfs_block *bl = NULL;
+	int ret;
 
-	bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno));
-	if (!IS_ERR_OR_NULL(bl) &&
-	    !scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno,
-					  SCOUTFS_BLOCK_MAGIC_ALLOC_LIST)) {
-		scoutfs_inc_counter(sb, alloc_stale_cached_list_block);
-		scoutfs_block_invalidate(sb, bl);
-		scoutfs_block_put(sb, bl);
-		bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno));
-	}
-	if (IS_ERR(bl)) {
-		*bl_ret = NULL;
-		return PTR_ERR(bl);
-	}
+	ret = scoutfs_block_read_ref(sb, ref, SCOUTFS_BLOCK_MAGIC_ALLOC_LIST, bl_ret);
+	if (ret < 0) {
+		if (ret == -ESTALE) {
+			scoutfs_inc_counter(sb, alloc_stale_list_block);
+			ret = -EIO;
+		}
+	};
 
-	*bl_ret = bl;
-	return 0;
+	return ret;
 }
 
 /*
@@ -396,86 +389,12 @@ static int read_list_block(struct super_block *sb,
 static int dirty_list_block(struct super_block *sb,
 			    struct scoutfs_alloc *alloc,
 			    struct scoutfs_block_writer *wri,
-			    struct scoutfs_alloc_list_ref *ref,
+			    struct scoutfs_block_ref *ref,
 			    u64 dirty, u64 *old,
 			    struct scoutfs_block **bl_ret)
 {
-	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
-	struct scoutfs_block *cow_bl = NULL;
-	struct scoutfs_block *bl = NULL;
-	struct scoutfs_alloc_list_block *lblk;
-	bool undo_alloc = false;
-	u64 blkno;
-	int ret;
-	int err;
-
-	blkno = le64_to_cpu(ref->blkno);
-	if (blkno) {
-		ret = read_list_block(sb, ref, &bl);
-		if (ret < 0)
-			goto out;
-
-		if (scoutfs_block_writer_is_dirty(sb, bl)) {
-			ret = 0;
-			goto out;
-		}
-	}
-
-	if (dirty == 0) {
-		ret = scoutfs_alloc_meta(sb, alloc, wri, &dirty);
-		if (ret < 0)
-			goto out;
-		undo_alloc = true;
-	}
-
-	cow_bl = scoutfs_block_create(sb, dirty);
-	if (IS_ERR(cow_bl)) {
-		ret = PTR_ERR(cow_bl);
-		goto out;
-	}
-
-	if (old) {
-		*old = blkno;
-	} else if (blkno) {
-		ret = scoutfs_free_meta(sb, alloc, wri, blkno);
-		if (ret < 0)
-			goto out;
-	}
-
-	if (bl)
-		memcpy(cow_bl->data, bl->data, SCOUTFS_BLOCK_LG_SIZE);
-	else
-		memset(cow_bl->data, 0, SCOUTFS_BLOCK_LG_SIZE);
-	scoutfs_block_put(sb, bl);
-	bl = cow_bl;
-	cow_bl = NULL;
-
-	lblk = bl->data;
-	lblk->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_ALLOC_LIST);
-	lblk->hdr.fsid = super->hdr.fsid;
-	lblk->hdr.blkno = cpu_to_le64(bl->blkno);
-	prandom_bytes(&lblk->hdr.seq, sizeof(lblk->hdr.seq));
-
-	ref->blkno = lblk->hdr.blkno;
-	ref->seq = lblk->hdr.seq;
-
-	scoutfs_block_writer_mark_dirty(sb, wri, bl);
-	ret = 0;
-
-out:
-	scoutfs_block_put(sb, cow_bl);
-	if (ret < 0 && undo_alloc) {
-		err = scoutfs_free_meta(sb, alloc, wri, dirty);
-		BUG_ON(err); /* inconsistent */
-	}
-
-	if (ret < 0) {
-		scoutfs_block_put(sb, bl);
-		bl = NULL;
-	}
-	*bl_ret = bl;
-
-	return ret;
+	return scoutfs_block_dirty_ref(sb, alloc, wri, ref, SCOUTFS_BLOCK_MAGIC_ALLOC_LIST,
+				       bl_ret, dirty, old);
 }
 
 /* Allocate a new dirty list block if we fill up more than 3/4 of the block. */
@@ -497,7 +416,7 @@ static int dirty_alloc_blocks(struct super_block *sb,
 			      struct scoutfs_alloc *alloc,
 			      struct scoutfs_block_writer *wri)
 {
-	struct scoutfs_alloc_list_ref orig_freed;
+	struct scoutfs_block_ref orig_freed;
 	struct scoutfs_alloc_list_block *lblk;
 	struct scoutfs_block *av_bl = NULL;
 	struct scoutfs_block *fr_bl = NULL;
@@ -607,7 +526,8 @@ int scoutfs_alloc_meta(struct super_block *sb, struct scoutfs_alloc *alloc,
 	if (ret < 0)
 		goto out;
 
-	spin_lock(&alloc->lock);
+	write_seqlock(&alloc->seqlock);
+
 	lblk = alloc->dirty_avail_bl->data;
 	if (WARN_ON_ONCE(lblk->nr == 0)) {
 		/* shouldn't happen, transaction should commit first */
@@ -617,7 +537,8 @@ int scoutfs_alloc_meta(struct super_block *sb, struct scoutfs_alloc *alloc,
 		list_block_remove(&alloc->avail, lblk, 1);
 		ret = 0;
 	}
-	spin_unlock(&alloc->lock);
+
+	write_sequnlock(&alloc->seqlock);
 
 out:
 	if (ret < 0)
@@ -640,7 +561,8 @@ int scoutfs_free_meta(struct super_block *sb, struct scoutfs_alloc *alloc,
 	if (ret < 0)
 		goto out;
 
-	spin_lock(&alloc->lock);
+	write_seqlock(&alloc->seqlock);
+
 	lblk = alloc->dirty_freed_bl->data;
 	if (WARN_ON_ONCE(list_block_space(lblk->nr) == 0)) {
 		/* shouldn't happen, transaction should commit first */
@@ -649,7 +571,8 @@ int scoutfs_free_meta(struct super_block *sb, struct scoutfs_alloc *alloc,
 		list_block_add(&alloc->freed, lblk, blkno);
 		ret = 0;
 	}
-	spin_unlock(&alloc->lock);
+
+	write_sequnlock(&alloc->seqlock);
 
 out:
 	scoutfs_inc_counter(sb, alloc_free_meta);
@@ -1106,7 +1029,7 @@ int scoutfs_alloc_splice_list(struct super_block *sb,
 			      struct scoutfs_alloc_list_head *src)
 {
 	struct scoutfs_alloc_list_block *lblk;
-	struct scoutfs_alloc_list_ref *ref;
+	struct scoutfs_block_ref *ref;
 	struct scoutfs_block *prev = NULL;
 	struct scoutfs_block *bl = NULL;
 	int ret = 0;
@@ -1147,17 +1070,23 @@ out:
 
 /*
  * Returns true if meta avail and free don't have room for the given
- * number of alloctions or frees.
+ * number of allocations or frees.  This is called at a significantly
+ * higher frequency than allocations as writers try to enter
+ * transactions.  This is the only reader of the seqlock which gives
+ * read-mostly sampling instead of bouncing a spinlock around all the
+ * cores.
  */
 bool scoutfs_alloc_meta_low(struct super_block *sb,
 			    struct scoutfs_alloc *alloc, u32 nr)
 {
+	unsigned int seq;
 	bool lo;
 
-	spin_lock(&alloc->lock);
-	lo = le32_to_cpu(alloc->avail.first_nr) < nr ||
-	     list_block_space(alloc->freed.first_nr) < nr;
-	spin_unlock(&alloc->lock);
+	do {
+		seq = read_seqbegin(&alloc->seqlock);
+		lo = le32_to_cpu(alloc->avail.first_nr) < nr ||
+		     list_block_space(alloc->freed.first_nr) < nr;
+	} while (read_seqretry(&alloc->seqlock, seq));
 
 	return lo;
 }
@@ -1169,8 +1098,8 @@ bool scoutfs_alloc_meta_low(struct super_block *sb,
 int scoutfs_alloc_foreach(struct super_block *sb,
 			  scoutfs_alloc_foreach_cb_t cb, void *arg)
 {
-	struct scoutfs_btree_ref stale_refs[2] = {{0,}};
-	struct scoutfs_btree_ref refs[2] = {{0,}};
+	struct scoutfs_block_ref stale_refs[2] = {{0,}};
+	struct scoutfs_block_ref refs[2] = {{0,}};
 	struct scoutfs_super_block *super = NULL;
 	struct scoutfs_srch_compact *sc;
 	struct scoutfs_log_trees lt;

kmod/src/alloc.h

@@ -72,7 +72,8 @@
  * transaction.
  */
 struct scoutfs_alloc {
-	spinlock_t lock;
+	/* writers rarely modify list_head avail/freed.  readers often check for _meta_alloc_low */
+	seqlock_t seqlock;
 	struct mutex mutex;
 	struct scoutfs_block *dirty_avail_bl;
 	struct scoutfs_block *dirty_freed_bl;

kmod/src/block.h

@@ -13,27 +13,16 @@ struct scoutfs_block {
 	void *priv;
 };
 
-__le32 scoutfs_block_calc_crc(struct scoutfs_block_header *hdr, u32 size);
-bool scoutfs_block_valid_crc(struct scoutfs_block_header *hdr, u32 size);
-bool scoutfs_block_valid_ref(struct super_block *sb,
-			     struct scoutfs_block_header *hdr,
-			     __le64 seq, __le64 blkno);
-
-struct scoutfs_block *scoutfs_block_create(struct super_block *sb, u64 blkno);
-struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno);
-void scoutfs_block_invalidate(struct super_block *sb, struct scoutfs_block *bl);
-bool scoutfs_block_consistent_ref(struct super_block *sb,
-				  struct scoutfs_block *bl,
-				  __le64 seq, __le64 blkno, u32 magic);
+int scoutfs_block_read_ref(struct super_block *sb, struct scoutfs_block_ref *ref, u32 magic,
+			   struct scoutfs_block **bl_ret);
 void scoutfs_block_put(struct super_block *sb, struct scoutfs_block *bl);
 
 void scoutfs_block_writer_init(struct super_block *sb,
 			       struct scoutfs_block_writer *wri);
-void scoutfs_block_writer_mark_dirty(struct super_block *sb,
-				     struct scoutfs_block_writer *wri,
-				     struct scoutfs_block *bl);
-bool scoutfs_block_writer_is_dirty(struct super_block *sb,
-				   struct scoutfs_block *bl);
+int scoutfs_block_dirty_ref(struct super_block *sb, struct scoutfs_alloc *alloc,
+			    struct scoutfs_block_writer *wri, struct scoutfs_block_ref *ref,
+			    u32 magic, struct scoutfs_block **bl_ret,
+			    u64 dirty_blkno, u64 *ref_blkno);
 int scoutfs_block_writer_write(struct super_block *sb,
 			       struct scoutfs_block_writer *wri);
 void scoutfs_block_writer_forget_all(struct super_block *sb,

kmod/src/btree.c

@@ -80,7 +80,7 @@ enum btree_walk_flags {
 	 BTW_NEXT	= (1 <<  0), /* return >= key */
 	 BTW_PREV	= (1 <<  1), /* return <= key */
 	 BTW_DIRTY	= (1 <<  2), /* cow stable blocks */
-	 BTW_ALLOC	= (1 <<  3), /* allocate a new block for 0 ref */
+	 BTW_ALLOC	= (1 <<  3), /* allocate a new block for 0 ref, requires dirty */
 	 BTW_INSERT	= (1 <<  4), /* walking to insert, try splitting */
 	 BTW_DELETE	= (1 <<  5), /* walking to delete, try joining */
 };
@@ -619,140 +619,36 @@ static void move_items(struct scoutfs_btree_block *dst,
  * This is used to lookup cached blocks, read blocks, cow blocks for
  * dirtying, and allocate new blocks.
  *
- * Btree blocks don't have rigid cache consistency.  We can be following
- * block references into cached blocks that are now stale or can be
- * following a stale root into blocks that have been overwritten.  If we
- * hit a block that looks stale we first invalidate the cache and retry,
- * returning -ESTALE if it still looks wrong.  The caller can retry the
- * read from a more current root or decide that this is a persistent
- * error.
+ * If we read a stale block we return stale so the caller can retry with
+ * a newer root or return an error.
  */
 static int get_ref_block(struct super_block *sb,
 			 struct scoutfs_alloc *alloc,
 			 struct scoutfs_block_writer *wri, int flags,
-			 struct scoutfs_btree_ref *ref,
+			 struct scoutfs_block_ref *ref,
 			 struct scoutfs_block **bl_ret)
 {
-	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
-	struct scoutfs_btree_block *bt = NULL;
-	struct scoutfs_btree_block *new;
-	struct scoutfs_block *new_bl = NULL;
-	struct scoutfs_block *bl = NULL;
-	bool retried = false;
-	u64 blkno;
-	u64 seq;
 	int ret;
 
-	/* always get the current block, either to return or cow from */
-	if (ref && ref->blkno) {
-retry:
+	if (WARN_ON_ONCE((flags & BTW_ALLOC) && !(flags & BTW_DIRTY)))
+		return -EINVAL;
 
-		bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno));
-		if (IS_ERR(bl)) {
-			trace_scoutfs_btree_read_error(sb, ref);
-			scoutfs_inc_counter(sb, btree_read_error);
-			ret = PTR_ERR(bl);
-			goto out;
-		}
-		bt = (void *)bl->data;
-
-		if (!scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno,
-						  SCOUTFS_BLOCK_MAGIC_BTREE) ||
-		    scoutfs_trigger(sb, BTREE_STALE_READ)) {
-
-			scoutfs_inc_counter(sb, btree_stale_read);
-
-			scoutfs_block_invalidate(sb, bl);
-			scoutfs_block_put(sb, bl);
-			bl = NULL;
-
-			if (!retried) {
-				retried = true;
-				goto retry;
-			}
-
-			ret = -ESTALE;
-			goto out;
-		}
-
-		/*
-		 * We need to create a new dirty copy of the block if
-		 * the caller asked for it.  If the block is already
-		 * dirty then we can return it.
-		 */
-		if (!(flags & BTW_DIRTY) ||
-		    scoutfs_block_writer_is_dirty(sb, bl)) {
-			ret = 0;
-			goto out;
-		}
-
-	} else if (!(flags & BTW_ALLOC)) {
+	if (ref->blkno == 0 && !(flags & BTW_ALLOC)) {
 		ret = -ENOENT;
 		goto out;
 	}
 
-	ret = scoutfs_alloc_meta(sb, alloc, wri, &blkno);
-	if (ret < 0)
-		goto out;
-
-	prandom_bytes(&seq, sizeof(seq));
-
-	new_bl = scoutfs_block_create(sb, blkno);
-	if (IS_ERR(new_bl)) {
-		ret = scoutfs_free_meta(sb, alloc, wri, blkno);
-		BUG_ON(ret);
-		ret = PTR_ERR(new_bl);
-		goto out;
-	}
-	new = (void *)new_bl->data;
-
-	/* free old stable blkno we're about to overwrite */
-	if (ref && ref->blkno) {
-		ret = scoutfs_free_meta(sb, alloc, wri,
-					le64_to_cpu(ref->blkno));
-		if (ret) {
-			ret = scoutfs_free_meta(sb, alloc, wri, blkno);
-			BUG_ON(ret);
-			scoutfs_block_put(sb, new_bl);
-			new_bl = NULL;
-			goto out;
-		}
-	}
-
-	scoutfs_block_writer_mark_dirty(sb, wri, new_bl);
-
-	trace_scoutfs_btree_dirty_block(sb, blkno, seq,
-					bt ? le64_to_cpu(bt->hdr.blkno) : 0,
-					bt ? le64_to_cpu(bt->hdr.seq) : 0);
-
-	if (bt) {
-		/* returning a cow of an existing block */
-		memcpy(new, bt, SCOUTFS_BLOCK_LG_SIZE);
-		scoutfs_block_put(sb, bl);
-	} else {
-		/* returning a newly allocated block */
-		memset(new, 0, SCOUTFS_BLOCK_LG_SIZE);
-		new->hdr.fsid = super->hdr.fsid;
-	}
-	bl = new_bl;
-	bt = new;
-
-	bt->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_BTREE);
-	bt->hdr.blkno = cpu_to_le64(blkno);
-	bt->hdr.seq = cpu_to_le64(seq);
-	if (ref) {
-		ref->blkno = bt->hdr.blkno;
-		ref->seq = bt->hdr.seq;
-	}
-	ret = 0;
-
+	if (flags & BTW_DIRTY)
+		ret = scoutfs_block_dirty_ref(sb, alloc, wri, ref, SCOUTFS_BLOCK_MAGIC_BTREE,
+					      bl_ret, 0, NULL);
+	else
+		ret = scoutfs_block_read_ref(sb, ref, SCOUTFS_BLOCK_MAGIC_BTREE, bl_ret);
 out:
-	if (ret) {
-		scoutfs_block_put(sb, bl);
-		bl = NULL;
+	if (ret < 0) {
+		if (ret == -ESTALE)
+			scoutfs_inc_counter(sb, btree_stale_read);
 	}
 
-	*bl_ret = bl;
 	return ret;
 }
 
@@ -766,7 +662,7 @@ static void create_parent_item(struct scoutfs_btree_block *parent,
 {
 	struct scoutfs_avl_node *par;
 	int cmp;
-	struct scoutfs_btree_ref ref = {
+	struct scoutfs_block_ref ref = {
 		.blkno = child->hdr.blkno,
 		.seq = child->hdr.seq,
 	};
@@ -784,7 +680,7 @@ static void update_parent_item(struct scoutfs_btree_block *parent,
 			       struct scoutfs_btree_item *par_item,
 			       struct scoutfs_btree_block *child)
 {
-	struct scoutfs_btree_ref *ref = item_val(parent, par_item);
+	struct scoutfs_block_ref *ref = item_val(parent, par_item);
 
 	par_item->key = *item_key(last_item(child));
 	ref->blkno = child->hdr.blkno;
@@ -832,12 +728,13 @@ static int try_split(struct super_block *sb,
 	struct scoutfs_block *par_bl = NULL;
 	struct scoutfs_btree_block *left;
 	struct scoutfs_key max_key;
+	struct scoutfs_block_ref zeros;
 	int ret;
 	int err;
 
 	/* parents need to leave room for child references */
 	if (right->level)
-		val_len = sizeof(struct scoutfs_btree_ref);
+		val_len = sizeof(struct scoutfs_block_ref);
 
 	/* don't need to split if there's enough space for the item */
 	if (mid_free_item_room(right, val_len))
@@ -849,7 +746,8 @@ static int try_split(struct super_block *sb,
 	scoutfs_inc_counter(sb, btree_split);
 
 	/* alloc split neighbour first to avoid unwinding tree growth */
-	ret = get_ref_block(sb, alloc, wri, BTW_ALLOC, NULL, &left_bl);
+	memset(&zeros, 0, sizeof(zeros));
+	ret = get_ref_block(sb, alloc, wri, BTW_ALLOC | BTW_DIRTY, &zeros, &left_bl);
 	if (ret)
 		return ret;
 	left = left_bl->data;
@@ -857,7 +755,8 @@ static int try_split(struct super_block *sb,
 	init_btree_block(left, right->level);
 
 	if (!parent) {
-		ret = get_ref_block(sb, alloc, wri, BTW_ALLOC, NULL, &par_bl);
+		memset(&zeros, 0, sizeof(zeros));
+		ret = get_ref_block(sb, alloc, wri, BTW_ALLOC | BTW_DIRTY, &zeros, &par_bl);
 		if (ret) {
 			err = scoutfs_free_meta(sb, alloc, wri,
 						le64_to_cpu(left->hdr.blkno));
@@ -905,7 +804,7 @@ static int try_join(struct super_block *sb,
 	struct scoutfs_btree_item *sib_par_item;
 	struct scoutfs_btree_block *sib;
 	struct scoutfs_block *sib_bl;
-	struct scoutfs_btree_ref *ref;
+	struct scoutfs_block_ref *ref;
 	unsigned int sib_tot;
 	bool move_right;
 	int to_move;
@@ -1194,7 +1093,7 @@ static int btree_walk(struct super_block *sb,
 	struct scoutfs_btree_item *prev;
 	struct scoutfs_avl_node *next_node;
 	struct scoutfs_avl_node *node;
-	struct scoutfs_btree_ref *ref;
+	struct scoutfs_block_ref *ref;
 	unsigned int level;
 	unsigned int nr;
 	int ret;
@@ -1225,8 +1124,7 @@ restart:
 		if (!(flags & BTW_INSERT)) {
 			ret = -ENOENT;
 		} else {
-			ret = get_ref_block(sb, alloc, wri, BTW_ALLOC,
-					    &root->ref, &bl);
+			ret = get_ref_block(sb, alloc, wri, BTW_ALLOC | BTW_DIRTY, &root->ref, &bl);
 			if (ret == 0) {
 				bt = bl->data;
 				init_btree_block(bt, 0);

kmod/src/client.c

@@ -31,16 +31,14 @@
 #include "net.h"
 #include "endian_swap.h"
 #include "quorum.h"
+#include "omap.h"
 
 /*
  * The client is responsible for maintaining a connection to the server.
- * This includes managing quorum elections that determine which client
- * should run the server that all the clients connect to.
  */
 
 #define CLIENT_CONNECT_DELAY_MS		(MSEC_PER_SEC / 10)
 #define CLIENT_CONNECT_TIMEOUT_MS	(1 * MSEC_PER_SEC)
-#define CLIENT_QUORUM_TIMEOUT_MS	(5 * MSEC_PER_SEC)
 
 struct client_info {
 	struct super_block *sb;
@@ -52,7 +50,6 @@ struct client_info {
 	struct delayed_work connect_dwork;
 
 	u64 server_term;
-	u64 greeting_umb;
 
 	bool sending_farewell;
 	int farewell_error;
@@ -154,7 +151,7 @@ static int client_lock_response(struct super_block *sb,
 				void *resp, unsigned int resp_len,
 				int error, void *data)
 {
-	if (resp_len != sizeof(struct scoutfs_net_lock_grant_response))
+	if (resp_len != sizeof(struct scoutfs_net_lock))
 		return -EINVAL;
 
 	/* XXX error? */
@@ -219,6 +216,39 @@ int scoutfs_client_srch_commit_compact(struct super_block *sb,
 					res, sizeof(*res), NULL, 0);
 }
 
+int scoutfs_client_send_omap_response(struct super_block *sb, u64 id,
+				      struct scoutfs_open_ino_map *map)
+{
+	struct client_info *client = SCOUTFS_SB(sb)->client_info;
+
+	return scoutfs_net_response(sb, client->conn, SCOUTFS_NET_CMD_OPEN_INO_MAP,
+				    id, 0, map, sizeof(*map));
+}
+
+/* The client is receiving an omap request from the server */
+static int client_open_ino_map(struct super_block *sb, struct scoutfs_net_connection *conn,
+			       u8 cmd, u64 id, void *arg, u16 arg_len)
+{
+	if (arg_len != sizeof(struct scoutfs_open_ino_map_args))
+		return -EINVAL;
+
+	return scoutfs_omap_client_handle_request(sb, id, arg);
+}
+
+/* The client is sending an omap request to the server */
+int scoutfs_client_open_ino_map(struct super_block *sb, u64 group_nr,
+				struct scoutfs_open_ino_map *map)
+{
+	struct client_info *client = SCOUTFS_SB(sb)->client_info;
+	struct scoutfs_open_ino_map_args args = {
+		.group_nr = cpu_to_le64(group_nr),
+		.req_id = 0,
+	};
+
+	return scoutfs_net_sync_request(sb, client->conn, SCOUTFS_NET_CMD_OPEN_INO_MAP,
+					&args, sizeof(args), map, sizeof(*map));
+}
+
 /* The client is receiving a invalidation request from the server */
 static int client_lock(struct super_block *sb,
 		       struct scoutfs_net_connection *conn, u8 cmd, u64 id,
@@ -292,52 +322,30 @@ static int client_greeting(struct super_block *sb,
 	scoutfs_net_client_greeting(sb, conn, new_server);
 
 	client->server_term = le64_to_cpu(gr->server_term);
-	client->greeting_umb = le64_to_cpu(gr->unmount_barrier);
 	ret = 0;
 out:
 	return ret;
 }
 
 /*
- * This work is responsible for maintaining a connection from the client
- * to the server.  It's queued on mount and disconnect and we requeue
- * the work if the work fails and we're not shutting down.
+ * The client is deciding if it needs to keep trying to reconnect to
+ * have its farewell request processed.  The server removes our mounted
+ * client item last so that if we don't see it we know the server has
+ * processed our farewell and we don't need to reconnect, we can unmount
+ * safely.
  *
- * In the typical case a mount reads the super blocks and finds the
- * address of the currently running server and connects to it.
- * Non-voting clients who can't connect will keep trying alternating
- * reading the address and getting connect timeouts.
- *
- * Voting mounts will try to elect a leader if they can't connect to the
- * server.  When a quorum can't connect and are able to elect a leader
- * then a new server is started.  The new server will write its address
- * in the super and everyone will be able to connect.
- *
- * There's a tricky bit of coordination required to safely unmount.
- * Clients need to tell the server that they won't be coming back with a
- * farewell request.  Once a client receives its farewell response it
- * can exit.  But a majority of clients need to stick around to elect a
- * server to process all their farewell requests.  This is coordinated
- * by having the greeting tell the server that a client is a voter.  The
- * server then holds on to farewell requests from voters until only
- * requests from the final quorum remain.  These farewell responses are
- * only sent after updating an unmount barrier in the super to indicate
- * to the final quorum that they can safely exit without having received
- * a farewell response over the network.
+ * This is peeking at btree blocks that the server could be actively
+ * freeing with cow updates so it can see stale blocks, we just return
+ * the error and we'll retry eventually as the connection times out.
  */
-static void scoutfs_client_connect_worker(struct work_struct *work)
+static int lookup_mounted_client_item(struct super_block *sb, u64 rid)
 {
-	struct client_info *client = container_of(work, struct client_info,
-						  connect_dwork.work);
-	struct super_block *sb = client->sb;
-	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
-	struct scoutfs_super_block *super = NULL;
-	struct mount_options *opts = &sbi->opts;
-	const bool am_voter = opts->server_addr.sin_addr.s_addr != 0;
-	struct scoutfs_net_greeting greet;
-	struct sockaddr_in sin;
-	ktime_t timeout_abs;
-	u64 elected_term;
+	struct scoutfs_key key = {
+		.sk_zone = SCOUTFS_MOUNTED_CLIENT_ZONE,
+		.skmc_rid = cpu_to_le64(rid),
+	};
+	struct scoutfs_super_block *super;
+	SCOUTFS_BTREE_ITEM_REF(iref);
 	int ret;
 
 	super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS);
@@ -350,57 +358,77 @@ static void scoutfs_client_connect_worker(struct work_struct *work)
 	if (ret)
 		goto out;
 
-	/* can safely unmount if we see that server processed our farewell */
-	if (am_voter && client->sending_farewell &&
-	    (le64_to_cpu(super->unmount_barrier) > client->greeting_umb)) {
+	ret = scoutfs_btree_lookup(sb, &super->mounted_clients, &key, &iref);
+	if (ret == 0) {
+		scoutfs_btree_put_iref(&iref);
+		ret = 1;
+	}
+	if (ret == -ENOENT)
+		ret = 0;
+
+	kfree(super);
+out:
+	return ret;
+}
+
+/*
+ * This work is responsible for maintaining a connection from the client
+ * to the server.  It's queued on mount and disconnect and we requeue
+ * the work if the work fails and we're not shutting down.
+ *
+ * We ask quorum for an address to try and connect to.  If there isn't
+ * one, or it fails, we back off a bit before trying again.
+ *
+ * There's a tricky bit of coordination required to safely unmount.
+ * Clients need to tell the server that they won't be coming back with a
+ * farewell request.  Once the server processes a farewell request from
+ * the client it can forget the client.  If the connection is broken
+ * before the client gets the farewell response it doesn't want to
+ * reconnect to send it again.. instead the client can read the metadata
+ * device to check for the lack of an item which indicates that the
+ * server has processed its farewell.
+ */
+static void scoutfs_client_connect_worker(struct work_struct *work)
+{
+	struct client_info *client = container_of(work, struct client_info,
+						  connect_dwork.work);
+	struct super_block *sb = client->sb;
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	struct scoutfs_super_block *super = &sbi->super;
+	struct mount_options *opts = &sbi->opts;
+	const bool am_quorum = opts->quorum_slot_nr >= 0;
+	struct scoutfs_net_greeting greet;
+	struct sockaddr_in sin;
+	int ret;
+
+	/* can unmount once server farewell handling removes our item */
+	if (client->sending_farewell &&
+	    lookup_mounted_client_item(sb, sbi->rid) == 0) {
 		client->farewell_error = 0;
 		complete(&client->farewell_comp);
 		ret = 0;
 		goto out;
 	}
 
-	/* try to connect to the super's server address */
-	scoutfs_addr_to_sin(&sin, &super->server_addr);
-	if (sin.sin_addr.s_addr != 0 && sin.sin_port != 0)
-		ret = scoutfs_net_connect(sb, client->conn, &sin,
-					  CLIENT_CONNECT_TIMEOUT_MS);
-	else
-		ret = -ENOTCONN;
-
-	/* voters try to elect a leader if they couldn't connect */
-	if (ret < 0) {
-		/* non-voters will keep retrying */
-		if (!am_voter)
-			goto out;
-
-		/* make sure local server isn't writing super during votes */
-		scoutfs_server_stop(sb);
-
-		timeout_abs = ktime_add_ms(ktime_get(),
-					   CLIENT_QUORUM_TIMEOUT_MS);
-
-		ret = scoutfs_quorum_election(sb, timeout_abs,
-					le64_to_cpu(super->quorum_server_term),
-					&elected_term);
-		/* start the server if we were asked to */
-		if (elected_term > 0)
-			ret = scoutfs_server_start(sb, &opts->server_addr,
-						   elected_term);
-		ret = -ENOTCONN;
+	ret = scoutfs_quorum_server_sin(sb, &sin);
+	if (ret < 0)
+		goto out;
+
+	ret = scoutfs_net_connect(sb, client->conn, &sin,
+				  CLIENT_CONNECT_TIMEOUT_MS);
+	if (ret < 0)
 		goto out;
-	}
 
 	/* send a greeting to verify endpoints of each connection */
 	greet.fsid = super->hdr.fsid;
 	greet.version = super->version;
 	greet.server_term = cpu_to_le64(client->server_term);
-	greet.unmount_barrier = cpu_to_le64(client->greeting_umb);
 	greet.rid = cpu_to_le64(sbi->rid);
 	greet.flags = 0;
 	if (client->sending_farewell)
 		greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL);
-	if (am_voter)
-		greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_VOTER);
+	if (am_quorum)
+		greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_QUORUM);
 
 	ret = scoutfs_net_submit_request(sb, client->conn,
 					 SCOUTFS_NET_CMD_GREETING,
@@ -409,7 +437,6 @@ static void scoutfs_client_connect_worker(struct work_struct *work)
 	if (ret)
 		scoutfs_net_shutdown(sb, client->conn);
 out:
-	kfree(super);
 
 	/* always have a small delay before retrying to avoid storms */
 	if (ret && !atomic_read(&client->shutting_down))
@@ -420,6 +447,7 @@ out:
 static scoutfs_net_request_t client_req_funcs[] = {
 	[SCOUTFS_NET_CMD_LOCK]			= client_lock,
 	[SCOUTFS_NET_CMD_LOCK_RECOVER]		= client_lock_recover,
+	[SCOUTFS_NET_CMD_OPEN_INO_MAP]		= client_open_ino_map,
 };
 
 /*

kmod/src/client.h

@@ -22,6 +22,10 @@ int scoutfs_client_srch_get_compact(struct super_block *sb,
 				    struct scoutfs_srch_compact *sc);
 int scoutfs_client_srch_commit_compact(struct super_block *sb,
 				       struct scoutfs_srch_compact *res);
+int scoutfs_client_send_omap_response(struct super_block *sb, u64 id,
+				      struct scoutfs_open_ino_map *map);
+int scoutfs_client_open_ino_map(struct super_block *sb, u64 group_nr,
+				struct scoutfs_open_ino_map *map);
 
 int scoutfs_client_setup(struct super_block *sb);
 void scoutfs_client_destroy(struct super_block *sb);

kmod/src/counters.h

@@ -20,17 +20,21 @@
 	EXPAND_COUNTER(alloc_list_freed_hi)			\
 	EXPAND_COUNTER(alloc_move)				\
 	EXPAND_COUNTER(alloc_moved_extent)			\
-	EXPAND_COUNTER(alloc_stale_cached_list_block)		\
-	EXPAND_COUNTER(block_cache_access)			\
+	EXPAND_COUNTER(alloc_stale_list_block)			\
+	EXPAND_COUNTER(block_cache_access_update)		\
 	EXPAND_COUNTER(block_cache_alloc_failure)		\
 	EXPAND_COUNTER(block_cache_alloc_page_order)		\
 	EXPAND_COUNTER(block_cache_alloc_virt)			\
 	EXPAND_COUNTER(block_cache_end_io_error)		\
 	EXPAND_COUNTER(block_cache_forget)			\
 	EXPAND_COUNTER(block_cache_free)			\
-	EXPAND_COUNTER(block_cache_invalidate)			\
-	EXPAND_COUNTER(block_cache_lru_move)			\
+	EXPAND_COUNTER(block_cache_free_work)			\
+	EXPAND_COUNTER(block_cache_remove_stale)		\
 	EXPAND_COUNTER(block_cache_shrink)			\
+	EXPAND_COUNTER(block_cache_shrink_next)			\
+	EXPAND_COUNTER(block_cache_shrink_recent)		\
+	EXPAND_COUNTER(block_cache_shrink_remove)		\
+	EXPAND_COUNTER(block_cache_shrink_restart)		\
 	EXPAND_COUNTER(btree_compact_values)			\
 	EXPAND_COUNTER(btree_compact_values_enomem)		\
 	EXPAND_COUNTER(btree_delete)				\
@@ -42,7 +46,6 @@
 	EXPAND_COUNTER(btree_lookup)				\
 	EXPAND_COUNTER(btree_next)				\
 	EXPAND_COUNTER(btree_prev)				\
-	EXPAND_COUNTER(btree_read_error)			\
 	EXPAND_COUNTER(btree_split)				\
 	EXPAND_COUNTER(btree_stale_read)			\
 	EXPAND_COUNTER(btree_update)				\
@@ -73,6 +76,7 @@
 	EXPAND_COUNTER(ext_op_remove)				\
 	EXPAND_COUNTER(forest_bloom_fail)			\
 	EXPAND_COUNTER(forest_bloom_pass)			\
+	EXPAND_COUNTER(forest_bloom_stale)			\
 	EXPAND_COUNTER(forest_read_items)			\
 	EXPAND_COUNTER(forest_roots_next_hint)			\
 	EXPAND_COUNTER(forest_set_bloom_bits)			\
@@ -108,9 +112,6 @@
 	EXPAND_COUNTER(item_write_dirty)			\
 	EXPAND_COUNTER(lock_alloc)				\
 	EXPAND_COUNTER(lock_free)				\
-	EXPAND_COUNTER(lock_grace_extended)			\
-	EXPAND_COUNTER(lock_grace_set)				\
-	EXPAND_COUNTER(lock_grace_wait)				\
 	EXPAND_COUNTER(lock_grant_request)			\
 	EXPAND_COUNTER(lock_grant_response)			\
 	EXPAND_COUNTER(lock_grant_work)				\
@@ -139,18 +140,21 @@
 	EXPAND_COUNTER(net_recv_invalid_message)		\
 	EXPAND_COUNTER(net_recv_messages)			\
 	EXPAND_COUNTER(net_unknown_request)			\
-	EXPAND_COUNTER(quorum_cycle)				\
-	EXPAND_COUNTER(quorum_elected_leader)			\
-	EXPAND_COUNTER(quorum_election_timeout)			\
-	EXPAND_COUNTER(quorum_failure)				\
-	EXPAND_COUNTER(quorum_read_block)			\
-	EXPAND_COUNTER(quorum_read_block_error)			\
+	EXPAND_COUNTER(quorum_elected)				\
+	EXPAND_COUNTER(quorum_fence_error)			\
+	EXPAND_COUNTER(quorum_fence_leader)			\
 	EXPAND_COUNTER(quorum_read_invalid_block)		\
-	EXPAND_COUNTER(quorum_saw_super_leader)			\
-	EXPAND_COUNTER(quorum_timedout)				\
-	EXPAND_COUNTER(quorum_write_block)			\
-	EXPAND_COUNTER(quorum_write_block_error)		\
-	EXPAND_COUNTER(quorum_fenced)				\
+	EXPAND_COUNTER(quorum_recv_error)			\
+	EXPAND_COUNTER(quorum_recv_heartbeat)			\
+	EXPAND_COUNTER(quorum_recv_invalid)			\
+	EXPAND_COUNTER(quorum_recv_resignation)			\
+	EXPAND_COUNTER(quorum_recv_vote)			\
+	EXPAND_COUNTER(quorum_send_heartbeat)			\
+	EXPAND_COUNTER(quorum_send_resignation)			\
+	EXPAND_COUNTER(quorum_send_request)			\
+	EXPAND_COUNTER(quorum_send_vote)			\
+	EXPAND_COUNTER(quorum_server_shutdown)			\
+	EXPAND_COUNTER(quorum_term_follower)			\
 	EXPAND_COUNTER(server_commit_hold)			\
 	EXPAND_COUNTER(server_commit_queue)			\
 	EXPAND_COUNTER(server_commit_worker)			\
@@ -160,7 +164,6 @@
 	EXPAND_COUNTER(srch_compact_flush)			\
 	EXPAND_COUNTER(srch_compact_log_page)			\
 	EXPAND_COUNTER(srch_compact_removed_entry)		\
-	EXPAND_COUNTER(srch_inconsistent_ref)			\
 	EXPAND_COUNTER(srch_rotate_log)				\
 	EXPAND_COUNTER(srch_search_log)				\
 	EXPAND_COUNTER(srch_search_log_block)			\

kmod/src/data.c

@@ -1135,7 +1135,8 @@ static void truncate_inode_pages_extent(struct inode *inode, u64 start, u64 len)
  */
 #define MOVE_DATA_EXTENTS_PER_HOLD 16
 int scoutfs_data_move_blocks(struct inode *from, u64 from_off,
-			     u64 byte_len, struct inode *to, u64 to_off)
+			     u64 byte_len, struct inode *to, u64 to_off, bool is_stage,
+			     u64 data_version)
 {
 	struct scoutfs_inode_info *from_si = SCOUTFS_I(from);
 	struct scoutfs_inode_info *to_si = SCOUTFS_I(to);
@@ -1145,6 +1146,7 @@ int scoutfs_data_move_blocks(struct inode *from, u64 from_off,
 	struct data_ext_args from_args;
 	struct data_ext_args to_args;
 	struct scoutfs_extent ext;
+	struct timespec cur_time;
 	LIST_HEAD(locks);
 	bool done = false;
 	loff_t from_size;
@@ -1180,6 +1182,11 @@ int scoutfs_data_move_blocks(struct inode *from, u64 from_off,
 		goto out;
 	}
 
+	if (is_stage && (data_version != SCOUTFS_I(to)->data_version)) {
+		ret = -ESTALE;
+		goto out;
+	}
+
 	from_iblock = from_off >> SCOUTFS_BLOCK_SM_SHIFT;
 	count = (byte_len + SCOUTFS_BLOCK_SM_MASK) >> SCOUTFS_BLOCK_SM_SHIFT;
 	to_iblock = to_off >> SCOUTFS_BLOCK_SM_SHIFT;
@@ -1202,7 +1209,7 @@ int scoutfs_data_move_blocks(struct inode *from, u64 from_off,
 	/* can't stage once data_version changes */
 	scoutfs_inode_get_onoff(from, &junk, &from_offline);
 	scoutfs_inode_get_onoff(to, &junk, &to_offline);
-	if (from_offline || to_offline) {
+	if (from_offline || (to_offline && !is_stage)) {
 		ret = -ENODATA;
 		goto out;
 	}
@@ -1246,6 +1253,8 @@ int scoutfs_data_move_blocks(struct inode *from, u64 from_off,
 
 		/* arbitrarily limit the number of extents per trans hold */
 		for (i = 0; i < MOVE_DATA_EXTENTS_PER_HOLD; i++) {
+			struct scoutfs_extent off_ext;
+
 			/* find the next extent to move */
 			ret = scoutfs_ext_next(sb, &data_ext_ops, &from_args,
 					       from_iblock, 1, &ext);
@@ -1274,10 +1283,27 @@ int scoutfs_data_move_blocks(struct inode *from, u64 from_off,
 
 			to_start = to_iblock + (from_start - from_iblock);
 
-			/* insert the new, fails if it overlaps */
-			ret = scoutfs_ext_insert(sb, &data_ext_ops, &to_args,
-						 to_start, len,
-						 map, ext.flags);
+			if (is_stage) {
+				ret = scoutfs_ext_next(sb, &data_ext_ops, &to_args,
+						       to_start, 1, &off_ext);
+				if (ret)
+					break;
+
+				if (!scoutfs_ext_inside(to_start, len, &off_ext) ||
+				    !(off_ext.flags & SEF_OFFLINE)) {
+					ret = -EINVAL;
+					break;
+				}
+
+				ret = scoutfs_ext_set(sb, &data_ext_ops, &to_args,
+							 to_start, len,
+							 map, ext.flags);
+			} else {
+				/* insert the new, fails if it overlaps */
+				ret = scoutfs_ext_insert(sb, &data_ext_ops, &to_args,
+							 to_start, len,
+							 map, ext.flags);
+			}
 			if (ret < 0)
 				break;
 
@@ -1285,10 +1311,18 @@ int scoutfs_data_move_blocks(struct inode *from, u64 from_off,
 			ret = scoutfs_ext_set(sb, &data_ext_ops, &from_args,
 					      from_start, len, 0, 0);
 			if (ret < 0) {
-				/* remove inserted new on err */
-				err = scoutfs_ext_remove(sb, &data_ext_ops,
-							 &to_args, to_start,
-							 len);
+				if (is_stage) {
+					/* re-mark dest range as offline */
+					WARN_ON_ONCE(!(off_ext.flags & SEF_OFFLINE));
+					err = scoutfs_ext_set(sb, &data_ext_ops, &to_args,
+							      to_start, len,
+							      0, off_ext.flags);
+				} else {
+					/* remove inserted new on err */
+					err = scoutfs_ext_remove(sb, &data_ext_ops,
+								 &to_args, to_start,
+								 len);
+				}
 				BUG_ON(err); /* XXX inconsistent */
 				break;
 			}
@@ -1316,12 +1350,15 @@ int scoutfs_data_move_blocks(struct inode *from, u64 from_off,
 		up_write(&from_si->extent_sem);
 		up_write(&to_si->extent_sem);
 
-		from->i_ctime = from->i_mtime =
-			to->i_ctime = to->i_mtime = CURRENT_TIME;
+		cur_time = CURRENT_TIME;
+		if (!is_stage) {
+			to->i_ctime = to->i_mtime = cur_time;
+			scoutfs_inode_inc_data_version(to);
+			scoutfs_inode_set_data_seq(to);
+		}
+		from->i_ctime = from->i_mtime = cur_time;
 		scoutfs_inode_inc_data_version(from);
-		scoutfs_inode_inc_data_version(to);
 		scoutfs_inode_set_data_seq(from);
-		scoutfs_inode_set_data_seq(to);
 
 		scoutfs_update_inode_item(from, from_lock, &locks);
 		scoutfs_update_inode_item(to, to_lock, &locks);

kmod/src/data.h

@@ -59,7 +59,8 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len);
 int scoutfs_data_init_offline_extent(struct inode *inode, u64 size,
 				     struct scoutfs_lock *lock);
 int scoutfs_data_move_blocks(struct inode *from, u64 from_off,
-			     u64 byte_len, struct inode *to, u64 to_off);
+			     u64 byte_len, struct inode *to, u64 to_off, bool to_stage,
+			     u64 data_version);
 
 int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len,
 			    u8 sef, u8 op, struct scoutfs_data_wait *ow,

kmod/src/dir.c

@@ -30,6 +30,7 @@
 #include "item.h"
 #include "lock.h"
 #include "hash.h"
+#include "omap.h"
 #include "counters.h"
 #include "scoutfs_trace.h"
 
@@ -813,6 +814,7 @@ static int scoutfs_link(struct dentry *old_dentry,
 	struct scoutfs_lock *dir_lock;
 	struct scoutfs_lock *inode_lock = NULL;
 	LIST_HEAD(ind_locks);
+	bool del_orphan;
 	u64 dir_size;
 	u64 ind_seq;
 	u64 hash;
@@ -841,6 +843,8 @@ static int scoutfs_link(struct dentry *old_dentry,
 		goto out_unlock;
 
 	dir_size = i_size_read(dir) + dentry->d_name.len;
+	del_orphan = (inode->i_nlink == 0);
+
 retry:
 	ret = scoutfs_inode_index_start(sb, &ind_seq) ?:
 	      scoutfs_inode_index_prepare(sb, &ind_locks, dir, false) ?:
@@ -855,6 +859,12 @@ retry:
 	if (ret)
 		goto out;
 
+	if (del_orphan) {
+		ret = scoutfs_orphan_dirty(sb, scoutfs_ino(inode));
+		if (ret)
+			goto out;
+	}
+
 	pos = SCOUTFS_I(dir)->next_readdir_pos++;
 
 	ret = add_entry_items(sb, scoutfs_ino(dir), hash, pos,
@@ -870,6 +880,11 @@ retry:
 	inode->i_ctime = dir->i_mtime;
 	inc_nlink(inode);
 
+	if (del_orphan) {
+		ret = scoutfs_orphan_delete(sb, scoutfs_ino(inode));
+		WARN_ON_ONCE(ret);
+	}
+
 	scoutfs_update_inode_item(inode, inode_lock, &ind_locks);
 	scoutfs_update_inode_item(dir, dir_lock, &ind_locks);
 
@@ -1760,6 +1775,42 @@ static int scoutfs_dir_open(struct inode *inode, struct file *file)
 }
 #endif
 
+static int scoutfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
+{
+	struct super_block *sb = dir->i_sb;
+	struct inode *inode = NULL;
+	struct scoutfs_lock *dir_lock = NULL;
+	struct scoutfs_lock *inode_lock = NULL;
+	LIST_HEAD(ind_locks);
+	int ret;
+
+	if (dentry->d_name.len > SCOUTFS_NAME_LEN)
+		return -ENAMETOOLONG;
+
+	inode = lock_hold_create(dir, dentry, mode, 0,
+				 &dir_lock, &inode_lock, &ind_locks);
+	if (IS_ERR(inode))
+		return PTR_ERR(inode);
+
+	inode->i_mtime = inode->i_atime = inode->i_ctime = CURRENT_TIME;
+	insert_inode_hash(inode);
+	d_tmpfile(dentry, inode);
+
+	scoutfs_update_inode_item(inode, inode_lock, &ind_locks);
+	scoutfs_update_inode_item(dir, dir_lock, &ind_locks);
+	scoutfs_inode_index_unlock(sb, &ind_locks);
+
+	ret = scoutfs_orphan_inode(inode);
+	WARN_ON_ONCE(ret); /* XXX returning error but items deleted */
+
+	scoutfs_release_trans(sb);
+	scoutfs_inode_index_unlock(sb, &ind_locks);
+	scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_WRITE);
+	scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE);
+
+	return ret;
+}
+
 const struct file_operations scoutfs_dir_fops = {
 	.KC_FOP_READDIR	= scoutfs_readdir,
 #ifdef KC_FMODE_KABI_ITERATE
@@ -1770,7 +1821,10 @@ const struct file_operations scoutfs_dir_fops = {
 	.llseek		= generic_file_llseek,
 };
 
-const struct inode_operations scoutfs_dir_iops = {
+
+
+const struct inode_operations_wrapper scoutfs_dir_iops = {
+	.ops = {
 	.lookup		= scoutfs_lookup,
 	.mknod		= scoutfs_mknod,
 	.create		= scoutfs_create,
@@ -1787,6 +1841,8 @@ const struct inode_operations scoutfs_dir_iops = {
 	.removexattr	= scoutfs_removexattr,
 	.symlink	= scoutfs_symlink,
 	.permission	= scoutfs_permission,
+	},
+	.tmpfile	= scoutfs_tmpfile,
 };
 
 void scoutfs_dir_exit(void)

kmod/src/dir.h

@@ -5,7 +5,7 @@
 #include "lock.h"
 
 extern const struct file_operations scoutfs_dir_fops;
-extern const struct inode_operations scoutfs_dir_iops;
+extern const struct inode_operations_wrapper scoutfs_dir_iops;
 extern const struct inode_operations scoutfs_symlink_iops;
 
 struct scoutfs_link_backref_entry {
@@ -14,7 +14,7 @@ struct scoutfs_link_backref_entry {
 	u64 dir_pos;
 	u16 name_len;
 	struct scoutfs_dirent dent;
-	/* the full name is allocated and stored in dent.name[0] */
+	/* the full name is allocated and stored in dent.name[] */
 };
 
 int scoutfs_dir_get_backref_path(struct super_block *sb, u64 ino, u64 dir_ino,

kmod/src/ext.c

@@ -38,7 +38,7 @@ static bool ext_overlap(struct scoutfs_extent *ext, u64 start, u64 len)
 	return !(e_end < start || ext->start > end);
 }
 
-static bool ext_inside(u64 start, u64 len, struct scoutfs_extent *out)
+bool scoutfs_ext_inside(u64 start, u64 len, struct scoutfs_extent *out)
 {
 	u64 in_end = start + len - 1;
 	u64 out_end = out->start + out->len - 1;
@@ -241,7 +241,7 @@ int scoutfs_ext_remove(struct super_block *sb, struct scoutfs_ext_ops *ops,
 		goto out;
 
 	/* removed extent must be entirely within found */
-	if (!ext_inside(start, len, &found)) {
+	if (!scoutfs_ext_inside(start, len, &found)) {
 		ret = -EINVAL;
 		goto out;
 	}
@@ -341,7 +341,7 @@ int scoutfs_ext_set(struct super_block *sb, struct scoutfs_ext_ops *ops,
 
 	if (ret == 0 && ext_overlap(&found, start, len)) {
 		/* set extent must be entirely within found */
-		if (!ext_inside(start, len, &found)) {
+		if (!scoutfs_ext_inside(start, len, &found)) {
 			ret = -EINVAL;
 			goto out;
 		}

kmod/src/ext.h

@@ -31,5 +31,6 @@ int scoutfs_ext_alloc(struct super_block *sb, struct scoutfs_ext_ops *ops,
 		      struct scoutfs_extent *ext);
 int scoutfs_ext_set(struct super_block *sb, struct scoutfs_ext_ops *ops,
 		    void *arg, u64 start, u64 len, u64 map, u8 flags);
+bool scoutfs_ext_inside(u64 start, u64 len, struct scoutfs_extent *out);
 
 #endif

kmod/src/file.c

@@ -27,8 +27,14 @@
 #include "file.h"
 #include "inode.h"
 #include "per_task.h"
+#include "omap.h"
 
-/* TODO: Direct I/O, AIO */
+/*
+ * Start a high level file read.  We check for offline extents in the
+ * read region here so that we only check the extents once.  We use the
+ * dio count to prevent releasing while we're reading after we've
+ * checked the extents.
+ */
 ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
 			      unsigned long nr_segs, loff_t pos)
 {
@@ -42,30 +48,32 @@ ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
 	int ret;
 
 retry:
+	/* protect checked extents from release */
+	mutex_lock(&inode->i_mutex);
+	atomic_inc(&inode->i_dio_count);
+	mutex_unlock(&inode->i_mutex);
+
 	ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ,
 				 SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock);
 	if (ret)
 		goto out;
 
 	if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, inode_lock)) {
-		/* protect checked extents from stage/release */
-		mutex_lock(&inode->i_mutex);
-		atomic_inc(&inode->i_dio_count);
-		mutex_unlock(&inode->i_mutex);
-
 		ret = scoutfs_data_wait_check_iov(inode, iov, nr_segs, pos,
 						  SEF_OFFLINE,
 						  SCOUTFS_IOC_DWO_READ,
 						  &dw, inode_lock);
 		if (ret != 0)
 			goto out;
+	} else {
+		WARN_ON_ONCE(true);
 	}
 
 	ret = generic_file_aio_read(iocb, iov, nr_segs, pos);
 
 out:
-	if (scoutfs_per_task_del(&si->pt_data_lock, &pt_ent))
-		inode_dio_done(inode);
+	inode_dio_done(inode);
+	scoutfs_per_task_del(&si->pt_data_lock, &pt_ent);
 	scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ);
 
 	if (scoutfs_data_wait_found(&dw)) {

kmod/src/forest.c

@@ -66,8 +66,8 @@ struct forest_info {
 	struct forest_info *name = SCOUTFS_SB(sb)->forest_info
 
 struct forest_refs {
-	struct scoutfs_btree_ref fs_ref;
-	struct scoutfs_btree_ref logs_ref;
+	struct scoutfs_block_ref fs_ref;
+	struct scoutfs_block_ref logs_ref;
 };
 
 /* initialize some refs that initially aren't equal */
@@ -96,20 +96,16 @@ static void calc_bloom_nrs(struct forest_bloom_nrs *bloom,
 	}
 }
 
-static struct scoutfs_block *read_bloom_ref(struct super_block *sb,
-					    struct scoutfs_btree_ref *ref)
+static struct scoutfs_block *read_bloom_ref(struct super_block *sb, struct scoutfs_block_ref *ref)
 {
 	struct scoutfs_block *bl;
+	int ret;
 
-	bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno));
-	if (IS_ERR(bl))
-		return bl;
-
-	if (!scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno,
-					  SCOUTFS_BLOCK_MAGIC_BLOOM)) {
-		scoutfs_block_invalidate(sb, bl);
-		scoutfs_block_put(sb, bl);
-		return ERR_PTR(-ESTALE);
+	ret = scoutfs_block_read_ref(sb, ref, SCOUTFS_BLOCK_MAGIC_BLOOM, &bl);
+	if (ret < 0) {
+		if (ret == -ESTALE)
+			scoutfs_inc_counter(sb, forest_bloom_stale);
+		bl = ERR_PTR(ret);
 	}
 
 	return bl;
@@ -280,7 +276,6 @@ int scoutfs_forest_read_items(struct super_block *sb,
 	scoutfs_inc_counter(sb, forest_read_items);
 	calc_bloom_nrs(&bloom, &lock->start);
 
-	roots = lock->roots;
 retry:
 	ret = scoutfs_client_get_roots(sb, &roots);
 	if (ret)
@@ -353,15 +348,9 @@ retry:
 	ret = 0;
 out:
 	if (ret == -ESTALE) {
-		if (memcmp(&prev_refs, &refs, sizeof(refs)) == 0) {
-			ret = -EIO;
-			goto out;
-		}
+		if (memcmp(&prev_refs, &refs, sizeof(refs)) == 0)
+			return -EIO;
 		prev_refs = refs;
-
-		ret = scoutfs_client_get_roots(sb, &roots);
-		if (ret)
-			goto out;
 		goto retry;
 	}
 
@@ -381,18 +370,14 @@ out:
 int scoutfs_forest_set_bloom_bits(struct super_block *sb,
 				  struct scoutfs_lock *lock)
 {
-	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
 	DECLARE_FOREST_INFO(sb, finf);
-	struct scoutfs_block *new_bl = NULL;
 	struct scoutfs_block *bl = NULL;
 	struct scoutfs_bloom_block *bb;
-	struct scoutfs_btree_ref *ref;
+	struct scoutfs_block_ref *ref;
 	struct forest_bloom_nrs bloom;
 	int nr_set = 0;
-	u64 blkno;
 	u64 nr;
 	int ret;
-	int err;
 	int i;
 
 	nr = le64_to_cpu(finf->our_log.nr);
@@ -410,53 +395,11 @@ int scoutfs_forest_set_bloom_bits(struct super_block *sb,
 
 	ref = &finf->our_log.bloom_ref;
 
-	if (ref->blkno) {
-		bl = read_bloom_ref(sb, ref);
-		if (IS_ERR(bl)) {
-			ret = PTR_ERR(bl);
-			goto unlock;
-		}
-		bb = bl->data;
-	}
-
-	if (!ref->blkno || !scoutfs_block_writer_is_dirty(sb, bl)) {
-
-		ret = scoutfs_alloc_meta(sb, finf->alloc, finf->wri, &blkno);
-		if (ret < 0)
-			goto unlock;
-
-		new_bl = scoutfs_block_create(sb, blkno);
-		if (IS_ERR(new_bl)) {
-			err = scoutfs_free_meta(sb, finf->alloc, finf->wri,
-						blkno);
-			BUG_ON(err); /* could have dirtied */
-			ret = PTR_ERR(new_bl);
-			goto unlock;
-		}
-
-		if (bl) {
-			err = scoutfs_free_meta(sb, finf->alloc, finf->wri,
-						le64_to_cpu(ref->blkno));
-			BUG_ON(err); /* could have dirtied */
-			memcpy(new_bl->data, bl->data, SCOUTFS_BLOCK_LG_SIZE);
-		} else {
-			memset(new_bl->data, 0, SCOUTFS_BLOCK_LG_SIZE);
-		}
-
-		scoutfs_block_writer_mark_dirty(sb, finf->wri, new_bl);
-
-		scoutfs_block_put(sb, bl);
-		bl = new_bl;
-		bb = bl->data;
-		new_bl = NULL;
-
-		bb->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_BLOOM);
-		bb->hdr.fsid = super->hdr.fsid;
-		bb->hdr.blkno = cpu_to_le64(blkno);
-		prandom_bytes(&bb->hdr.seq, sizeof(bb->hdr.seq));
-		ref->blkno = bb->hdr.blkno;
-		ref->seq = bb->hdr.seq;
-	}
+	ret = scoutfs_block_dirty_ref(sb, finf->alloc, finf->wri, ref, SCOUTFS_BLOCK_MAGIC_BLOOM,
+				      &bl, 0, NULL);
+	if (ret < 0)
+		goto unlock;
+	bb = bl->data;
 
 	for (i = 0; i < ARRAY_SIZE(bloom.nrs); i++) {
 		if (!test_and_set_bit_le(bloom.nrs[i], bb->bits)) {

kmod/src/format.h

@@ -14,6 +14,7 @@
 #define SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK	0x897e4a7d
 #define SCOUTFS_BLOCK_MAGIC_SRCH_PARENT	0xb23a2a05
 #define SCOUTFS_BLOCK_MAGIC_ALLOC_LIST	0x8a93ac83
+#define SCOUTFS_BLOCK_MAGIC_QUORUM	0xbc310868
 
 /*
  * The super block, quorum block, and file data allocation granularity
@@ -54,15 +55,19 @@
 #define SCOUTFS_SUPER_BLKNO ((64ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT)
 
 /*
- * A reasonably large region of aligned quorum blocks follow the super
- * block.  Each voting cycle reads the entire region so we don't want it
- * to be too enormous.  256K seems like a reasonably chunky single IO.
- * The number of blocks in the region also determines the number of
- * mounts that have a reasonable probability of not overwriting each
- * other's random block locations.
+ * A small number of quorum blocks follow the super block, enough of
+ * them to match the starting offset of the super block so the region is
+ * aligned to the power of two that contains it.
  */
-#define SCOUTFS_QUORUM_BLKNO	((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT)
-#define SCOUTFS_QUORUM_BLOCKS	((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT)
+#define SCOUTFS_QUORUM_BLKNO	(SCOUTFS_SUPER_BLKNO + 1)
+#define SCOUTFS_QUORUM_BLOCKS	(SCOUTFS_SUPER_BLKNO - 1)
+
+/*
+ * Free metadata blocks start after the quorum blocks
+ */
+#define SCOUTFS_META_DEV_START_BLKNO				\
+	((SCOUTFS_QUORUM_BLKNO + SCOUTFS_QUORUM_BLOCKS) >>	\
+	 SCOUTFS_BLOCK_SM_LG_SHIFT)
 
 /*
  * Start data on the data device aligned as well.
@@ -81,11 +86,33 @@ struct scoutfs_timespec {
 	__u8 __pad[4];
 };
 
-/* XXX ipv6 */
-struct scoutfs_inet_addr {
-	__le32 addr;
+enum scoutfs_inet_family {
+	SCOUTFS_AF_NONE = 0,
+	SCOUTFS_AF_IPV4 = 1,
+	SCOUTFS_AF_IPV6 = 2,
+};
+
+struct scoutfs_inet_addr4 {
+	__le16 family;
 	__le16 port;
-	__u8 __pad[2];
+	__le32 addr;
+};
+
+/*
+ * Not yet supported by code.
+ */
+struct scoutfs_inet_addr6 {
+	__le16 family;
+	__le16 port;
+	__u8 addr[16];
+	__le32 flow_info;
+	__le32 scope_id;
+	__u8 __pad[4];
+};
+
+union scoutfs_inet_addr {
+	struct scoutfs_inet_addr4 v4;
+	struct scoutfs_inet_addr6 v6;
 };
 
 /*
@@ -101,6 +128,15 @@ struct scoutfs_block_header {
 	__le64 blkno;
 };
 
+/*
+ * A reference to a block.  The corresponding fields in the block_header
+ * must match after having read the block contents.
+ */
+struct scoutfs_block_ref {
+	__le64 blkno;
+	__le64 seq;
+};
+
 /*
  * scoutfs identifies all file system metadata items by a small key
  * struct.
@@ -159,9 +195,6 @@ struct scoutfs_key {
 #define sklt_rid	_sk_first
 #define sklt_nr		_sk_second
 
-/* lock clients */
-#define sklc_rid	_sk_first
-
 /* seqs */
 #define skts_trans_seq	_sk_first
 #define skts_rid	_sk_second
@@ -197,17 +230,12 @@ struct scoutfs_avl_node {
  */
 #define SCOUTFS_BTREE_MAX_HEIGHT 20
 
-struct scoutfs_btree_ref {
-	__le64 blkno;
-	__le64 seq;
-};
-
 /*
  * A height of X means that the first block read will have level X-1 and
  * the leaves will have level 0.
  */
 struct scoutfs_btree_root {
-	struct scoutfs_btree_ref ref;
+	struct scoutfs_block_ref ref;
 	__u8 height;
 	__u8 __pad[7];
 };
@@ -228,7 +256,7 @@ struct scoutfs_btree_block {
 	__le16 mid_free_len;
 	__u8 level;
 	__u8 __pad[7];
-	struct scoutfs_btree_item items[0];
+	struct scoutfs_btree_item items[];
 	/* leaf blocks have a fixed size item offset hash table at the end */
 };
 
@@ -248,18 +276,13 @@ struct scoutfs_btree_block {
 #define SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES \
 	(SCOUTFS_BTREE_LEAF_ITEM_HASH_NR * sizeof(__le16))
 
-struct scoutfs_alloc_list_ref {
-	__le64 blkno;
-	__le64 seq;
-};
-
 /*
  * first_nr tracks the nr of the first block in the list and is used for
  * allocation sizing. total_nr is the sum of the nr of all the blocks in
  * the list and is used for calculating total free block counts.
  */
 struct scoutfs_alloc_list_head {
-	struct scoutfs_alloc_list_ref ref;
+	struct scoutfs_block_ref ref;
 	__le64 total_nr;
 	__le32 first_nr;
 	__u8 __pad[4];
@@ -278,10 +301,10 @@ struct scoutfs_alloc_list_head {
  */
 struct scoutfs_alloc_list_block {
 	struct scoutfs_block_header hdr;
-	struct scoutfs_alloc_list_ref next;
+	struct scoutfs_block_ref next;
 	__le32 start;
 	__le32 nr;
-	__le64 blknos[0]; /* naturally aligned for sorting */
+	__le64 blknos[]; /* naturally aligned for sorting */
 };
 
 #define SCOUTFS_ALLOC_LIST_MAX_BLOCKS					      \
@@ -306,7 +329,7 @@ struct scoutfs_mounted_client_btree_val {
 	__u8 flags;
 };
 
-#define SCOUTFS_MOUNTED_CLIENT_VOTER	(1 << 0)
+#define SCOUTFS_MOUNTED_CLIENT_QUORUM	(1 << 0)
 
 /*
  * srch files are a contiguous run of blocks with compressed entries
@@ -324,15 +347,10 @@ struct scoutfs_srch_entry {
 
 #define SCOUTFS_SRCH_ENTRY_MAX_BYTES	(2 + (sizeof(__u64) * 3))
 
-struct scoutfs_srch_ref {
-	__le64 blkno;
-	__le64 seq;
-};
-
 struct scoutfs_srch_file {
 	struct scoutfs_srch_entry first;
 	struct scoutfs_srch_entry last;
-	struct scoutfs_srch_ref ref;
+	struct scoutfs_block_ref ref;
 	__le64 blocks;
 	__le64 entries;
 	__u8 height;
@@ -341,13 +359,13 @@ struct scoutfs_srch_file {
 
 struct scoutfs_srch_parent {
 	struct scoutfs_block_header hdr;
-	struct scoutfs_srch_ref refs[0];
+	struct scoutfs_block_ref refs[];
 };
 
 #define SCOUTFS_SRCH_PARENT_REFS				\
 	((SCOUTFS_BLOCK_LG_SIZE -				\
 	  offsetof(struct scoutfs_srch_parent, refs)) /		\
-	 sizeof(struct scoutfs_srch_ref))
+	 sizeof(struct scoutfs_block_ref))
 
 struct scoutfs_srch_block {
 	struct scoutfs_block_header hdr;
@@ -356,7 +374,7 @@ struct scoutfs_srch_block {
 	struct scoutfs_srch_entry tail;
 	__le32 entry_nr;
 	__le32 entry_bytes;
-	__u8 entries[0];
+	__u8 entries[];
 };
 
 /*
@@ -418,7 +436,7 @@ struct scoutfs_log_trees {
 	struct scoutfs_alloc_list_head meta_avail;
 	struct scoutfs_alloc_list_head meta_freed;
 	struct scoutfs_btree_root item_root;
-	struct scoutfs_btree_ref bloom_ref;
+	struct scoutfs_block_ref bloom_ref;
 	struct scoutfs_alloc_root data_avail;
 	struct scoutfs_alloc_root data_freed;
 	struct scoutfs_srch_file srch_file;
@@ -431,7 +449,7 @@ struct scoutfs_log_item_value {
 	__le64 vers;
 	__u8 flags;
 	__u8 __pad[7];
-	__u8 data[0];
+	__u8 data[];
 };
 
 /*
@@ -446,7 +464,7 @@ struct scoutfs_log_item_value {
 struct scoutfs_bloom_block {
 	struct scoutfs_block_header hdr;
 	__le64 total_set;
-	__le64 bits[0];
+	__le64 bits[];
 };
 
 /*
@@ -472,11 +490,10 @@ struct scoutfs_bloom_block {
 #define SCOUTFS_LOCK_ZONE			4
 /* Items only stored in server btrees */
 #define SCOUTFS_LOG_TREES_ZONE			6
-#define SCOUTFS_LOCK_CLIENTS_ZONE		7
-#define SCOUTFS_TRANS_SEQ_ZONE			8
-#define SCOUTFS_MOUNTED_CLIENT_ZONE		9
-#define SCOUTFS_SRCH_ZONE			10
-#define SCOUTFS_FREE_EXTENT_ZONE		11
+#define SCOUTFS_TRANS_SEQ_ZONE			7
+#define SCOUTFS_MOUNTED_CLIENT_ZONE		8
+#define SCOUTFS_SRCH_ZONE			9
+#define SCOUTFS_FREE_EXTENT_ZONE		10
 
 /* inode index zone */
 #define SCOUTFS_INODE_INDEX_META_SEQ_TYPE	1
@@ -528,7 +545,7 @@ struct scoutfs_xattr {
 	__le16 val_len;
 	__u8 name_len;
 	__u8 __pad[5];
-	__u8 name[0];
+	__u8 name[];
 };
 
 
@@ -537,49 +554,77 @@ struct scoutfs_xattr {
 
 #define SCOUTFS_UUID_BYTES 16
 
-/*
- * Mounts read all the quorum blocks and write to one random quorum
- * block during a cycle.  The min cycle time limits the per-mount iop
- * load during elections.  The random cycle delay makes it less likely
- * that mounts will read and write at the same time and miss each
- * other's writes.  An election only completes if a quorum of mounts
- * vote for a leader before any of their elections timeout.  This is
- * made less likely by the probability that mounts will overwrite each
- * others random block locations.  The max quorum count limits that
- * probability.  9 mounts only have a 55% chance of writing to unique 4k
- * blocks in a 256k region.  The election timeout is set to include
- * enough cycles to usually complete the election.  Once a leader is
- * elected it spends a number of cycles writing out blocks with itself
- * logged as a leader.  This reduces the possibility that servers
- * will have their log entries overwritten and not be fenced.
- */
-#define SCOUTFS_QUORUM_MAX_COUNT		9
-#define SCOUTFS_QUORUM_CYCLE_LO_MS		10
-#define SCOUTFS_QUORUM_CYCLE_HI_MS		20
-#define SCOUTFS_QUORUM_TERM_LO_MS		250
-#define SCOUTFS_QUORUM_TERM_HI_MS		500
-#define SCOUTFS_QUORUM_ELECTED_LOG_CYCLES	10
+#define SCOUTFS_QUORUM_MAX_SLOTS	15
 
-struct scoutfs_quorum_block {
+/*
+ * To elect a leader, members race to have their variable election
+ * timeouts expire.  If they're first to send a vote request with a
+ * greater term to a majority of waiting members they'll be elected with
+ * a majority.  If the timeouts are too close, the vote may be split and
+ * everyone will wait for another cycle of variable timeouts to expire.
+ *
+ * These determine how long it will take to elect a leader once there's
+ * no evidence of a server (no leader quorum blocks on mount; heartbeat
+ * timeout expired.)
+ */
+#define SCOUTFS_QUORUM_ELECT_MIN_MS	250
+#define SCOUTFS_QUORUM_ELECT_VAR_MS	100
+
+/*
+ * Once a leader is elected they send out heartbeats at regular
+ * intervals to force members to wait the much longer heartbeat timeout.
+ * Once heartbeat timeout expires without receiving a heartbeat they'll
+ * switch over the performing elections.
+ *
+ * These determine how long it could take members to notice that a
+ * leader has gone silent and start to elect a new leader.
+ */
+#define SCOUTFS_QUORUM_HB_IVAL_MS	100
+#define SCOUTFS_QUORUM_HB_TIMEO_MS	(5 * MSEC_PER_SEC)
+
+struct scoutfs_quorum_message {
 	__le64 fsid;
-	__le64 blkno;
+	__le64 version;
 	__le64 term;
-	__le64 write_nr;
-	__le64 voter_rid;
-	__le64 vote_for_rid;
+	__u8 type;
+	__u8 from;
+	__u8 __pad[2];
 	__le32 crc;
-	__u8 log_nr;
-	__u8 __pad[3];
-	struct scoutfs_quorum_log {
-		__le64 term;
-		__le64 rid;
-		struct scoutfs_inet_addr addr;
-	} log[0];
 };
 
-#define SCOUTFS_QUORUM_LOG_MAX						  \
-	((SCOUTFS_BLOCK_SM_SIZE - sizeof(struct scoutfs_quorum_block)) /  \
-		sizeof(struct scoutfs_quorum_log))
+/* a candidate requests a vote */
+#define SCOUTFS_QUORUM_MSG_REQUEST_VOTE	0
+/* followers send votes to candidates */
+#define SCOUTFS_QUORUM_MSG_VOTE		1
+/* elected leaders broadcast heartbeats to delay elections */
+#define SCOUTFS_QUORUM_MSG_HEARTBEAT	2
+/* leaders broadcast as they leave to break heartbeat timeout */
+#define SCOUTFS_QUORUM_MSG_RESIGNATION	3
+#define SCOUTFS_QUORUM_MSG_INVALID	4
+
+/*
+ * The version is currently always 0, but will be used by mounts to
+ * discover that membership has changed.
+ */
+struct scoutfs_quorum_config {
+	__le64 version;
+	struct scoutfs_quorum_slot {
+		union scoutfs_inet_addr addr;
+	} slots[SCOUTFS_QUORUM_MAX_SLOTS];
+};
+
+struct scoutfs_quorum_block {
+	struct scoutfs_block_header hdr;
+	__le64 term;
+	__le64 random_write_mark;
+	__le64 flags;
+	struct scoutfs_quorum_block_event {
+		__le64 rid;
+		struct scoutfs_timespec ts;
+	} write, update_term, set_leader, clear_leader, fenced;
+};
+
+#define SCOUTFS_QUORUM_BLOCK_LEADER (1 << 0)
 
 #define SCOUTFS_FLAG_IS_META_BDEV 0x01
 
@@ -597,19 +642,13 @@ struct scoutfs_super_block {
 	__le64 total_data_blocks;
 	__le64 first_data_blkno;
 	__le64 last_data_blkno;
-	__le64 quorum_fenced_term;
-	__le64 quorum_server_term;
-	__le64 unmount_barrier;
-	__u8 quorum_count;
-	__u8 __pad[7];
-	struct scoutfs_inet_addr server_addr;
+	struct scoutfs_quorum_config qconf;
 	struct scoutfs_alloc_root meta_alloc[2];
 	struct scoutfs_alloc_root data_alloc;
 	struct scoutfs_alloc_list_head server_meta_avail[2];
 	struct scoutfs_alloc_list_head server_meta_freed[2];
 	struct scoutfs_btree_root fs_root;
 	struct scoutfs_btree_root logs_root;
-	struct scoutfs_btree_root lock_clients;
 	struct scoutfs_btree_root trans_seqs;
 	struct scoutfs_btree_root mounted_clients;
 	struct scoutfs_btree_root srch_root;
@@ -685,7 +724,7 @@ struct scoutfs_dirent {
 	__le64 pos;
 	__u8 type;
 	__u8 __pad[7];
-	__u8 name[0];
+	__u8 name[];
 };
 
 #define SCOUTFS_NAME_LEN 255
@@ -718,9 +757,9 @@ enum scoutfs_dentry_type {
 	DIV_ROUND_UP(sizeof(struct scoutfs_xattr) + name_len + val_len, \
 		     (unsigned int)SCOUTFS_XATTR_MAX_PART_SIZE)
 
-#define SCOUTFS_LOCK_INODE_GROUP_NR	1024
+#define SCOUTFS_LOCK_INODE_GROUP_NR	128
 #define SCOUTFS_LOCK_INODE_GROUP_MASK	(SCOUTFS_LOCK_INODE_GROUP_NR - 1)
-#define SCOUTFS_LOCK_SEQ_GROUP_MASK	((1ULL << 10) - 1)
+#define SCOUTFS_LOCK_SEQ_GROUP_MASK	((1ULL << 7) - 1)
 
 /*
  * messages over the wire.
@@ -736,12 +775,6 @@ enum scoutfs_dentry_type {
  * the same serer after receiving a greeting response and to a new
  * server after failover.
  *
- * @unmount_barrier: Incremented every time the remaining majority of
- * quorum members all agree to leave.  The server tells a quorum member
- * the value that it's connecting under so that if the client sees the
- * value increase in the super block then it knows that the server has
- * processed its farewell and can safely unmount.
- *
  * @rid: The client's random id that was generated once as the mount
  * started up.  This identifies a specific remote mount across
  * connections and servers.  It's set to the client's rid in both the
@@ -751,13 +784,12 @@ struct scoutfs_net_greeting {
 	__le64 fsid;
 	__le64 version;
 	__le64 server_term;
-	__le64 unmount_barrier;
 	__le64 rid;
 	__le64 flags;
 };
 
 #define SCOUTFS_NET_GREETING_FLAG_FAREWELL	(1 << 0)
-#define SCOUTFS_NET_GREETING_FLAG_VOTER		(1 << 1)
+#define SCOUTFS_NET_GREETING_FLAG_QUORUM	(1 << 1)
 #define SCOUTFS_NET_GREETING_FLAG_INVALID	(~(__u64)0 << 2)
 
 /*
@@ -790,7 +822,7 @@ struct scoutfs_net_header {
 	__u8 flags;
 	__u8 error;
 	__u8 __pad[3];
-	__u8 data[0];
+	__u8 data[];
 };
 
 #define SCOUTFS_NET_FLAG_RESPONSE	(1 << 0)
@@ -808,6 +840,7 @@ enum scoutfs_net_cmd {
 	SCOUTFS_NET_CMD_LOCK_RECOVER,
 	SCOUTFS_NET_CMD_SRCH_GET_COMPACT,
 	SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT,
+	SCOUTFS_NET_CMD_OPEN_INO_MAP,
 	SCOUTFS_NET_CMD_FAREWELL,
 	SCOUTFS_NET_CMD_UNKNOWN,
 };
@@ -858,15 +891,10 @@ struct scoutfs_net_lock {
 	__u8 __pad[6];
 };
 
-struct scoutfs_net_lock_grant_response {
-	struct scoutfs_net_lock nl;
-	struct scoutfs_net_roots roots;
-};
-
 struct scoutfs_net_lock_recover {
 	__le16 nr;
 	__u8 __pad[6];
-	struct scoutfs_net_lock locks[0];
+	struct scoutfs_net_lock locks[];
 };
 
 #define SCOUTFS_NET_LOCK_MAX_RECOVER_NR					       \
@@ -881,6 +909,7 @@ enum scoutfs_lock_trace {
 	SLT_INVALIDATE,
 	SLT_REQUEST,
 	SLT_RESPONSE,
+	SLT_NR,
 };
 
 /*
@@ -933,4 +962,42 @@ enum scoutfs_corruption_sources {
 
 #define SC_NR_LONGS DIV_ROUND_UP(SC_NR_SOURCES, BITS_PER_LONG)
 
+#define SCOUTFS_OPEN_INO_MAP_SHIFT	7
+#define SCOUTFS_OPEN_INO_MAP_BITS	(1 << SCOUTFS_OPEN_INO_MAP_SHIFT)
+#define SCOUTFS_OPEN_INO_MAP_MASK	(SCOUTFS_OPEN_INO_MAP_BITS - 1)
+#define SCOUTFS_OPEN_INO_MAP_LE64S	(SCOUTFS_OPEN_INO_MAP_BITS / 64)
+
+/*
+ * The request and response conversation is as follows:
+ *
+ * client[init] -> server:
+ *	group_nr = G
+ *	req_id = 0	(I)
+ * server -> client[*]
+ *	group_nr = G
+ *	req_id = R
+ * client[*] -> server
+ *	group_nr = G	(I)
+ *	req_id = R
+ *	bits
+ * server -> client[init]
+ *	group_nr = G	(I)
+ *	req_id = R	(I)
+ *	bits
+ *
+ * Many of the fields in individual messages are ignored ("I") because
+ * the net id or the omap req_id can be used to identify the
+ * conversation.  We always include them on the wire to make inspected
+ * messages easier to follow.
+ */
+struct scoutfs_open_ino_map_args {
+	__le64 group_nr;
+	__le64 req_id;
+};
+
+struct scoutfs_open_ino_map {
+	struct scoutfs_open_ino_map_args args;
+	__le64 bits[SCOUTFS_OPEN_INO_MAP_LE64S];
+};
+
 #endif

kmod/src/inode.c

@@ -33,6 +33,7 @@
 #include "item.h"
 #include "client.h"
 #include "cmp.h"
+#include "omap.h"
 
 /*
  * XXX
@@ -82,6 +83,8 @@ static void scoutfs_inode_ctor(void *obj)
 	init_waitqueue_head(&si->data_waitq.waitq);
 	init_rwsem(&si->xattr_rwsem);
 	RB_CLEAR_NODE(&si->writeback_node);
+	scoutfs_lock_init_coverage(&si->ino_lock_cov);
+	atomic_set(&si->inv_iput_count, 0);
 
 	inode_init_once(&si->inode);
 }
@@ -141,12 +144,15 @@ static void remove_writeback_inode(struct inode_sb_info *inf,
 
 void scoutfs_destroy_inode(struct inode *inode)
 {
+	struct scoutfs_inode_info *si = SCOUTFS_I(inode);
 	DECLARE_INODE_SB_INFO(inode->i_sb, inf);
 
 	spin_lock(&inf->writeback_lock);
 	remove_writeback_inode(inf, SCOUTFS_I(inode));
 	spin_unlock(&inf->writeback_lock);
 
+	scoutfs_lock_del_coverage(inode->i_sb, &si->ino_lock_cov);
+
 	call_rcu(&inode->i_rcu, scoutfs_i_callback);
 }
 
@@ -182,7 +188,8 @@ static void set_inode_ops(struct inode *inode)
 		inode->i_fop = &scoutfs_file_fops;
 		break;
 	case S_IFDIR:
-		inode->i_op = &scoutfs_dir_iops;
+		inode->i_op = &scoutfs_dir_iops.ops;
+		inode->i_flags |= S_IOPS_WRAPPER;
 		inode->i_fop = &scoutfs_dir_fops;
 		break;
 	case S_IFLNK:
@@ -306,6 +313,8 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock,
 		if (ret == 0) {
 			load_inode(inode, &sinode);
 			atomic64_set(&si->last_refreshed, refresh_gen);
+			scoutfs_lock_add_coverage(sb, lock, &si->ino_lock_cov);
+			si->drop_invalidated = false;
 		}
 	} else {
 		ret = 0;
@@ -639,12 +648,22 @@ void scoutfs_inode_get_onoff(struct inode *inode, s64 *on, s64 *off)
 	} while (read_seqcount_retry(&si->seqcount, seq));
 }
 
+/*
+ * We have inversions between getting cluster locks while performing
+ * final deletion on a freeing inode and waiting on a freeing inode
+ * while holding a cluster lock.
+ *
+ * We can avoid these deadlocks by hiding freeing inodes in our hash
+ * lookup function.  We're fine with either returning null or populating
+ * a new inode overlapping with eviction freeing a previous instance of
+ * the inode.
+ */
 static int scoutfs_iget_test(struct inode *inode, void *arg)
 {
 	struct scoutfs_inode_info *si = SCOUTFS_I(inode);
 	u64 *ino = arg;
 
-	return si->ino == *ino;
+	return (si->ino == *ino) && !(inode->i_state & I_FREEING);
 }
 
 static int scoutfs_iget_set(struct inode *inode, void *arg)
@@ -687,6 +706,8 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino)
 		atomic64_set(&si->last_refreshed, 0);
 
 		ret = scoutfs_inode_refresh(inode, lock, 0);
+		if (ret == 0)
+			ret = scoutfs_omap_inc(sb, ino);
 		if (ret) {
 			iget_failed(inode);
 			inode = ERR_PTR(ret);
@@ -1383,6 +1404,8 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir,
 	si->next_xattr_id = 0;
 	si->have_item = false;
 	atomic64_set(&si->last_refreshed, lock->refresh_gen);
+	scoutfs_lock_add_coverage(sb, lock, &si->ino_lock_cov);
+	si->drop_invalidated = false;
 	si->flags = 0;
 
 	scoutfs_inode_set_meta_seq(inode);
@@ -1398,10 +1421,17 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir,
 	store_inode(&sinode, inode);
 	init_inode_key(&key, scoutfs_ino(inode));
 
+	ret = scoutfs_omap_inc(sb, ino);
+	if (ret < 0)
+		goto out;
+
 	ret = scoutfs_item_create(sb, &key, &sinode, sizeof(sinode), lock);
+	if (ret < 0)
+		scoutfs_omap_dec(sb, ino);
+out:
 	if (ret) {
 		iput(inode);
-		return ERR_PTR(ret);
+		inode = ERR_PTR(ret);
 	}
 
 	return inode;
@@ -1417,7 +1447,18 @@ static void init_orphan_key(struct scoutfs_key *key, u64 rid, u64 ino)
 	};
 }
 
-static int remove_orphan_item(struct super_block *sb, u64 ino)
+int scoutfs_orphan_dirty(struct super_block *sb, u64 ino)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	struct scoutfs_lock *lock = sbi->rid_lock;
+	struct scoutfs_key key;
+
+	init_orphan_key(&key, sbi->rid, ino);
+
+	return scoutfs_item_dirty(sb, &key, lock);
+}
+
+int scoutfs_orphan_delete(struct super_block *sb, u64 ino)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
 	struct scoutfs_lock *lock = sbi->rid_lock;
@@ -1435,15 +1476,15 @@ static int remove_orphan_item(struct super_block *sb, u64 ino)
 
 /*
  * Remove all the items associated with a given inode.  This is only
- * called once nlink has dropped to zero so we don't have to worry about
- * dirents referencing the inode or link backrefs.  Dropping nlink to 0
- * also created an orphan item.  That orphan item will continue
- * triggering attempts to finish previous partial deletion until all
- * deletion is complete and the orphan item is removed.
+ * called once nlink has dropped to zero and nothing has the inode open
+ * so we don't have to worry about dirents referencing the inode or link
+ * backrefs.  Dropping nlink to 0 also created an orphan item.  That
+ * orphan item will continue triggering attempts to finish previous
+ * partial deletion until all deletion is complete and the orphan item
+ * is removed.
  */
-static int delete_inode_items(struct super_block *sb, u64 ino)
+static int delete_inode_items(struct super_block *sb, u64 ino, struct scoutfs_lock *lock)
 {
-	struct scoutfs_lock *lock = NULL;
 	struct scoutfs_inode sinode;
 	struct scoutfs_key key;
 	LIST_HEAD(ind_locks);
@@ -1453,10 +1494,6 @@ static int delete_inode_items(struct super_block *sb, u64 ino)
 	u64 size;
 	int ret;
 
-	ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_WRITE, 0, ino, &lock);
-	if (ret)
-		return ret;
-
 	init_inode_key(&key, ino);
 
 	ret = scoutfs_item_lookup_exact(sb, &key, &sinode, sizeof(sinode),
@@ -1516,23 +1553,29 @@ retry:
 	if (ret)
 		goto out;
 
-	ret = remove_orphan_item(sb, ino);
+	ret = scoutfs_orphan_delete(sb, ino);
 out:
 	if (release)
 		scoutfs_release_trans(sb);
 	scoutfs_inode_index_unlock(sb, &ind_locks);
-	scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE);
+
 	return ret;
 }
 
 /*
  * iput_final has already written out the dirty pages to the inode
  * before we get here.  We're left with a clean inode that we have to
- * tear down.  If there are no more links to the inode then we also
- * remove all its persistent structures.
+ * tear down.  We use locking and open inode number bitmaps to decide if
+ * we should finally destroy an inode that is no longer open nor
+ * reachable through directory entries.
  */
 void scoutfs_evict_inode(struct inode *inode)
 {
+	struct super_block *sb = inode->i_sb;
+	const u64 ino = scoutfs_ino(inode);
+	struct scoutfs_lock *lock;
+	int ret;
+
 	trace_scoutfs_evict_inode(inode->i_sb, scoutfs_ino(inode),
 				  inode->i_nlink, is_bad_inode(inode));
 
@@ -1541,19 +1584,45 @@ void scoutfs_evict_inode(struct inode *inode)
 
 	truncate_inode_pages_final(&inode->i_data);
 
-	if (inode->i_nlink == 0)
-		delete_inode_items(inode->i_sb, scoutfs_ino(inode));
+	ret = scoutfs_omap_should_delete(sb, inode, &lock);
+	if (ret > 0) {
+		ret = delete_inode_items(inode->i_sb, scoutfs_ino(inode), lock);
+		scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE);
+	}
+	if (ret < 0)
+		scoutfs_err(sb, "error %d while checking to delete inode nr %llu, it might linger.",
+			    ret, ino);
+
+	scoutfs_omap_dec(sb, ino);
+
 clear:
 	clear_inode(inode);
 }
 
+/*
+ * We want to remove inodes from the cache as their count goes to 0 if
+ * they're no longer covered by a cluster lock or if while locked they
+ * were unlinked.
+ *
+ * We don't want unused cached inodes to linger outside of cluster
+ * locking so that they don't prevent final inode deletion on other
+ * nodes.  We don't have specific per-inode or per-dentry locks which
+ * would otherwise remove the stale caches as they're invalidated.
+ * Stale cached inodes provide little value because they're going to be
+ * refreshed the next time they're locked.  Populating the item cache
+ * and loading the inode item is a lot more expensive than initializing
+ * and inserting a newly allocated vfs inode.
+ */
 int scoutfs_drop_inode(struct inode *inode)
 {
-	int ret = generic_drop_inode(inode);
+	struct scoutfs_inode_info *si = SCOUTFS_I(inode);
+	struct super_block *sb = inode->i_sb;
 
-	trace_scoutfs_drop_inode(inode->i_sb, scoutfs_ino(inode),
-				 inode->i_nlink, inode_unhashed(inode));
-	return ret;
+	trace_scoutfs_drop_inode(sb, scoutfs_ino(inode), inode->i_nlink, inode_unhashed(inode),
+				 si->drop_invalidated);
+
+	return si->drop_invalidated || !scoutfs_lock_is_covered(sb, &si->ino_lock_cov) ||
+	       generic_drop_inode(inode);
 }
 
 /*
@@ -1570,8 +1639,10 @@ int scoutfs_scan_orphans(struct super_block *sb)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
 	struct scoutfs_lock *lock = sbi->rid_lock;
+	struct scoutfs_lock *inode_lock = NULL;
 	struct scoutfs_key key;
 	struct scoutfs_key last;
+	u64 ino;
 	int err = 0;
 	int ret;
 
@@ -1587,7 +1658,13 @@ int scoutfs_scan_orphans(struct super_block *sb)
 		if (ret < 0)
 			goto out;
 
-		ret = delete_inode_items(sb, le64_to_cpu(key.sko_ino));
+		ino = le64_to_cpu(key.sko_ino);
+
+		ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_WRITE, 0, ino, &inode_lock);
+		if (ret == 0) {
+			ret = delete_inode_items(sb, le64_to_cpu(key.sko_ino), inode_lock);
+			scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE);
+		}
 		if (ret && ret != -ENOENT && !err)
 			err = ret;
 
@@ -1621,19 +1698,28 @@ int scoutfs_orphan_inode(struct inode *inode)
 }
 
 /*
- * Track an inode that could have dirty pages.  Used to kick off writeback
- * on all dirty pages during transaction commit without tying ourselves in
- * knots trying to call through the high level vfs sync methods.
+ * Track an inode that could have dirty pages.  Used to kick off
+ * writeback on all dirty pages during transaction commit without tying
+ * ourselves in knots trying to call through the high level vfs sync
+ * methods.
+ *
+ * This is called by writers who hold the inode and transaction.  The
+ * inode's presence in the rbtree is removed by destroy_inode, prevented
+ * by the inode hold, and by committing the transaction, which is
+ * prevented by holding the transaction.  The inode can only go from
+ * empty to on the rbtree while we're here.
  */
 void scoutfs_inode_queue_writeback(struct inode *inode)
 {
 	DECLARE_INODE_SB_INFO(inode->i_sb, inf);
 	struct scoutfs_inode_info *si = SCOUTFS_I(inode);
 
-	spin_lock(&inf->writeback_lock);
-	if (RB_EMPTY_NODE(&si->writeback_node))
-		insert_writeback_inode(inf, si);
-	spin_unlock(&inf->writeback_lock);
+	if (RB_EMPTY_NODE(&si->writeback_node)) {
+		spin_lock(&inf->writeback_lock);
+		if (RB_EMPTY_NODE(&si->writeback_node))
+			insert_writeback_inode(inf, si);
+		spin_unlock(&inf->writeback_lock);
+	}
 }
 
 /*

kmod/src/inode.h

@@ -51,6 +51,13 @@ struct scoutfs_inode_info {
 	struct rw_semaphore xattr_rwsem;
 	struct rb_node writeback_node;
 
+	struct scoutfs_lock_coverage ino_lock_cov;
+
+	/* drop if i_count hits 0, allows drop while invalidate holds coverage */
+	bool drop_invalidated;
+	struct llist_node inv_iput_llnode;
+	atomic_t inv_iput_count;
+
 	struct inode inode;
 };
 
@@ -114,6 +121,8 @@ int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry,
 int scoutfs_setattr(struct dentry *dentry, struct iattr *attr);
 
 int scoutfs_scan_orphans(struct super_block *sb);
+int scoutfs_orphan_dirty(struct super_block *sb, u64 ino);
+int scoutfs_orphan_delete(struct super_block *sb, u64 ino);
 
 void scoutfs_inode_queue_writeback(struct inode *inode);
 int scoutfs_inode_walk_writeback(struct super_block *sb, bool write);

kmod/src/ioctl.c

@@ -972,12 +972,18 @@ static long scoutfs_ioc_move_blocks(struct file *file, unsigned long arg)
 		goto out;
 	}
 
+	if (mb.flags & SCOUTFS_IOC_MB_UNKNOWN) {
+		ret = -EINVAL;
+		goto out;
+	}
+
 	ret = mnt_want_write_file(file);
 	if (ret < 0)
 		goto out;
 
 	ret = scoutfs_data_move_blocks(from, mb.from_off, mb.len,
-				       to, mb.to_off);
+				       to, mb.to_off, !!(mb.flags & SCOUTFS_IOC_MB_STAGE),
+				       mb.data_version);
 	mnt_drop_write_file(file);
 out:
 	fput(from_file);

kmod/src/ioctl.h

@@ -163,7 +163,7 @@ struct scoutfs_ioctl_ino_path_result {
 	__u64 dir_pos;
 	__u16 path_bytes;
 	__u8  _pad[6];
-	__u8  path[0];
+	__u8  path[];
 };
 
 /* Get a single path from the root to the given inode number */
@@ -259,7 +259,7 @@ struct scoutfs_ioctl_data_waiting {
 	__u8 _pad[6];
 };
 
-#define SCOUTFS_IOC_DATA_WAITING_FLAGS_UNKNOWN		(U8_MAX << 0)
+#define SCOUTFS_IOC_DATA_WAITING_FLAGS_UNKNOWN		(U64_MAX << 0)
 
 #define SCOUTFS_IOC_DATA_WAITING _IOR(SCOUTFS_IOCTL_MAGIC, 6, \
 				      struct scoutfs_ioctl_data_waiting)
@@ -279,7 +279,7 @@ struct scoutfs_ioctl_setattr_more {
 };
 
 #define SCOUTFS_IOC_SETATTR_MORE_OFFLINE		(1 << 0)
-#define SCOUTFS_IOC_SETATTR_MORE_UNKNOWN		(U8_MAX << 1)
+#define SCOUTFS_IOC_SETATTR_MORE_UNKNOWN		(U64_MAX << 1)
 
 #define SCOUTFS_IOC_SETATTR_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 7, \
 				      struct scoutfs_ioctl_setattr_more)
@@ -418,12 +418,13 @@ struct scoutfs_ioctl_alloc_detail_entry {
  * on the same file system.
  *
  * from_fd specifies the source file and the ioctl is called on the
- * destination file.  Both files must have write access.  from_off
- * specifies the byte offset in the source, to_off is the byte offset in
- * the destination, and len is the number of bytes in the region to
- * move.   All of the offsets and lengths must be in multiples of 4KB,
- * except in the case where the from_off + len ends at the i_size of the
- * source file.
+ * destination file.  Both files must have write access.  from_off specifies
+ * the byte offset in the source, to_off is the byte offset in the
+ * destination, and len is the number of bytes in the region to move.  All of
+ * the offsets and lengths must be in multiples of 4KB, except in the case
+ * where the from_off + len ends at the i_size of the source
+ * file. data_version is only used when STAGE flag is set (see below).  flags
+ * field is currently only used to optionally specify STAGE behavior.
  *
  * This interface only moves extents which are block granular, it does
  * not perform RMW of sub-block byte extents and it does not overwrite
@@ -435,30 +436,41 @@ struct scoutfs_ioctl_alloc_detail_entry {
  * i_size.  The i_size update will maintain final partial blocks in the
  * source.
  *
- * It will return an error if either of the files have offline extents.
- * It will return 0 when all of the extents in the source region have
- * been moved to the destination.  Moving extents updates the ctime,
- * mtime, meta_seq, data_seq, and data_version fields of both the source
- * and destination inodes.  If an error is returned then partial
+ * If STAGE flag is not set, it will return an error if either of the files
+ * have offline extents.  It will return 0 when all of the extents in the
+ * source region have been moved to the destination.  Moving extents updates
+ * the ctime, mtime, meta_seq, data_seq, and data_version fields of both the
+ * source and destination inodes.  If an error is returned then partial
  * progress may have been made and inode fields may have been updated.
  *
+ * If STAGE flag is set, as above except destination range must be in an
+ * offline extent. Fields are updated only for source inode.
+ *
  * Errors specific to this interface include:
  *
  * EINVAL: from_off, len, or to_off aren't a multiple of 4KB; the source
  *	   and destination files are the same inode; either the source or
  *	   destination is not a regular file; the destination file has
- *	   an existing overlapping extent.
+ *	   an existing overlapping extent (if STAGE flag not set); the
+ *	   destination range is not in an offline extent (if STAGE set).
  * EOVERFLOW: either from_off + len or to_off + len exceeded 64bits.
  * EBADF: from_fd isn't a valid open file descriptor.
  * EXDEV: the source and destination files are in different filesystems.
  * EISDIR: either the source or destination is a directory.
- * ENODATA: either the source or destination file have offline extents.
+ * ENODATA: either the source or destination file have offline extents and
+ *	    STAGE flag is not set.
+ * ESTALE: data_version does not match destination data_version.
  */
+#define SCOUTFS_IOC_MB_STAGE		(1 << 0)
+#define SCOUTFS_IOC_MB_UNKNOWN		(U64_MAX << 1)
+
 struct scoutfs_ioctl_move_blocks {
 	__u64 from_fd;
 	__u64 from_off;
 	__u64 len;
 	__u64 to_off;
+	__u64 data_version;
+	__u64 flags;
 };
 
 #define SCOUTFS_IOC_MOVE_BLOCKS _IOR(SCOUTFS_IOCTL_MAGIC, 13, \

kmod/src/lock.c

@@ -34,6 +34,7 @@
 #include "data.h"
 #include "xattr.h"
 #include "item.h"
+#include "omap.h"
 
 /*
  * scoutfs uses a lock service to manage item cache consistency between
@@ -65,8 +66,6 @@
  * relative to that lock state we resend.
  */
 
-#define GRACE_PERIOD_KT	ms_to_ktime(10)
-
 /*
  * allocated per-super, freed on unmount.
  */
@@ -74,6 +73,7 @@ struct lock_info {
 	struct super_block *sb;
 	spinlock_t lock;
 	bool shutdown;
+	bool unmounting;
 	struct rb_root lock_tree;
 	struct rb_root lock_range_tree;
 	struct shrinker shrinker;
@@ -82,11 +82,14 @@ struct lock_info {
 	struct workqueue_struct *workq;
 	struct work_struct grant_work;
 	struct list_head grant_list;
-	struct delayed_work inv_dwork;
+	struct work_struct inv_work;
 	struct list_head inv_list;
 	struct work_struct shrink_work;
 	struct list_head shrink_list;
 	atomic64_t next_refresh_gen;
+	struct work_struct inv_iput_work;
+	struct llist_head inv_iput_llist;
+
 	struct dentry *tseq_dentry;
 	struct scoutfs_tseq_tree tseq_tree;
 };
@@ -122,21 +125,79 @@ static bool lock_modes_match(int granted, int requested)
 }
 
 /*
- * invalidate cached data associated with an inode whose lock is going
+ * Final iput can get into evict and perform final inode deletion which
+ * can delete a lot of items under locks and transactions.  We really
+ * don't want to be doing all that in an iput during invalidation.  When
+ * invalidation sees that iput might perform final deletion it puts them
+ * on a list and queues this work.
+ *
+ * Nothing stops multiple puts for multiple invalidations of an inode
+ * before the work runs so we can track multiple puts in flight.
+ */
+static void lock_inv_iput_worker(struct work_struct *work)
+{
+	struct lock_info *linfo = container_of(work, struct lock_info, inv_iput_work);
+	struct scoutfs_inode_info *si;
+	struct scoutfs_inode_info *tmp;
+	struct llist_node *inodes;
+	bool more;
+
+	inodes = llist_del_all(&linfo->inv_iput_llist);
+
+	llist_for_each_entry_safe(si, tmp, inodes, inv_iput_llnode) {
+		do {
+			more = atomic_dec_return(&si->inv_iput_count) > 0;
+			iput(&si->inode);
+		} while (more);
+	}
+}
+
+/*
+ * Invalidate cached data associated with an inode whose lock is going
  * away.
+ *
+ * We try to drop cached dentries and inodes covered by the lock if they
+ * aren't referenced.  This removes them from the mount's open map and
+ * allows deletions to be performed by unlink without having to wait for
+ * remote cached inodes to be dropped.
+ *
+ * If the cached inode was already deferring final inode deletion then
+ * we can't perform that inline in invalidation.  The locking alone
+ * deadlock, and it might also take multiple transactions to fully
+ * delete an inode with significant metadata.  We only perform the iput
+ * inline if we know that possible eviction can't perform the final
+ * deletion, otherwise we kick it off to async work.
  */
 static void invalidate_inode(struct super_block *sb, u64 ino)
 {
+	DECLARE_LOCK_INFO(sb, linfo);
+	struct scoutfs_inode_info *si;
 	struct inode *inode;
 
 	inode = scoutfs_ilookup(sb, ino);
 	if (inode) {
+		si = SCOUTFS_I(inode);
+
 		scoutfs_inc_counter(sb, lock_invalidate_inode);
 		if (S_ISREG(inode->i_mode)) {
 			truncate_inode_pages(inode->i_mapping, 0);
 			scoutfs_data_wait_changed(inode);
 		}
-		iput(inode);
+
+		/* can't touch during unmount, dcache destroys w/o locks */
+		if (!linfo->unmounting)
+			d_prune_aliases(inode);
+
+		si->drop_invalidated = true;
+		if (scoutfs_lock_is_covered(sb, &si->ino_lock_cov) && inode->i_nlink > 0) {
+			iput(inode);
+		} else {
+			/* defer iput to work context so we don't evict inodes from invalidation */ 
+			if (atomic_inc_return(&si->inv_iput_count) == 1)
+				llist_add(&si->inv_iput_llnode, &linfo->inv_iput_llist);
+			smp_wmb(); /* count and list visible before work executes */
+			queue_work(linfo->workq, &linfo->inv_iput_work);
+		}
 	}
 }
 
@@ -172,6 +233,16 @@ static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock,
 	/* have to invalidate if we're not in the only usable case */
 	if (!(prev == SCOUTFS_LOCK_WRITE && mode == SCOUTFS_LOCK_READ)) {
 retry:
+		/* invalidate inodes before removing coverage */
+		if (lock->start.sk_zone == SCOUTFS_FS_ZONE) {
+			ino = le64_to_cpu(lock->start.ski_ino);
+			last = le64_to_cpu(lock->end.ski_ino);
+			while (ino <= last) {
+				invalidate_inode(sb, ino);
+				ino++;
+			}
+		}
+
 		/* remove cov items to tell users that their cache is stale */
 		spin_lock(&lock->cov_list_lock);
 		list_for_each_entry_safe(cov, tmp, &lock->cov_list, head) {
@@ -187,15 +258,6 @@ retry:
 		}
 		spin_unlock(&lock->cov_list_lock);
 
-		if (lock->start.sk_zone == SCOUTFS_FS_ZONE) {
-			ino = le64_to_cpu(lock->start.ski_ino);
-			last = le64_to_cpu(lock->end.ski_ino);
-			while (ino <= last) {
-				invalidate_inode(sb, ino);
-				ino++;
-			}
-		}
-
 		scoutfs_item_invalidate(sb, &lock->start, &lock->end);
 	}
 
@@ -229,6 +291,7 @@ static void lock_free(struct lock_info *linfo, struct scoutfs_lock *lock)
 	BUG_ON(!list_empty(&lock->shrink_head));
 	BUG_ON(!list_empty(&lock->cov_list));
 
+	scoutfs_omap_free_lock_data(lock->omap_data);
 	kfree(lock);
 }
 
@@ -264,6 +327,7 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb,
 	lock->mode = SCOUTFS_LOCK_NULL;
 
 	atomic64_set(&lock->forest_bloom_nr, 0);
+	spin_lock_init(&lock->omap_spinlock);
 
 	trace_scoutfs_lock_alloc(sb, lock);
 
@@ -298,23 +362,6 @@ static bool lock_counts_match(int granted, unsigned int *counts)
 	return true;
 }
 
-/*
- * Returns true if there are any mode counts that match with the desired
- * mode.  There can be other non-matching counts as well but we're only
- * testing for the existence of any matching counts.
- */
-static bool lock_count_match_exists(int desired, unsigned int *counts)
-{
-	enum scoutfs_lock_mode mode;
-
-	for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) {
-		if (counts[mode] && lock_modes_match(desired, mode))
-			return true;
-	}
-
-	return false;
-}
-
 /*
  * An idle lock has nothing going on.  It can be present in the lru and
  * can be freed by the final put when it has a null mode.
@@ -531,46 +578,24 @@ static void put_lock(struct lock_info *linfo,struct scoutfs_lock *lock)
 	}
 }
 
-/*
- * Locks have a grace period that extends after activity and prevents
- * invalidation.  It's intended to let nodes do reasonable batches of
- * work as locks ping pong between nodes that are doing conflicting
- * work.
- */
-static void extend_grace(struct super_block *sb, struct scoutfs_lock *lock)
-{
-	ktime_t now = ktime_get();
-
-	if (ktime_after(now, lock->grace_deadline))
-		scoutfs_inc_counter(sb, lock_grace_set);
-	else
-		scoutfs_inc_counter(sb, lock_grace_extended);
-
-	lock->grace_deadline = ktime_add(now, GRACE_PERIOD_KT);
-}
-
 static void queue_grant_work(struct lock_info *linfo)
 {
 	assert_spin_locked(&linfo->lock);
 
-	if (!list_empty(&linfo->grant_list) && !linfo->shutdown)
+	if (!list_empty(&linfo->grant_list))
 		queue_work(linfo->workq, &linfo->grant_work);
 }
 
 /*
- * We immediately queue work on the assumption that the caller might
- * have made a change (set a lock mode) which can let one of the
- * invalidating locks make forward progress, even if other locks are
- * waiting for their grace period to elapse.  It's a trade-off between
- * invalidation latency and burning cpu repeatedly finding that locks
- * are still in their grace period.
+ * The caller has made a change (set a lock mode) which can let one of the
+ * invalidating locks make forward progress.
  */
 static void queue_inv_work(struct lock_info *linfo)
 {
 	assert_spin_locked(&linfo->lock);
 
-	if (!list_empty(&linfo->inv_list) && !linfo->shutdown)
-		mod_delayed_work(linfo->workq, &linfo->inv_dwork, 0);
+	if (!list_empty(&linfo->inv_list))
+		queue_work(linfo->workq, &linfo->inv_work);
 }
 
 /*
@@ -623,22 +648,12 @@ static void bug_on_inconsistent_grant_cache(struct super_block *sb,
  * Grant responses can be reordered with incoming invalidation requests
  * from the server so we have to be careful to only set the new mode
  * once the old mode matches.
- *
- * We extend the grace period as we grant the lock if there is a waiting
- * locker who can use the lock.  This stops invalidation from pulling
- * the granted lock out from under the requester, resulting in a lot of
- * churn with no forward progress.  Using the grace period avoids having
- * to identify a specific waiter and give it an acquired lock.  It's
- * also very similar to waking up the locker and having it win the race
- * against the invalidation.  In that case they'd extend the grace
- * period anyway as they unlock.
  */
 static void lock_grant_worker(struct work_struct *work)
 {
 	struct lock_info *linfo = container_of(work, struct lock_info,
 					       grant_work);
 	struct super_block *sb = linfo->sb;
-	struct scoutfs_net_lock_grant_response *gr;
 	struct scoutfs_net_lock *nl;
 	struct scoutfs_lock *lock;
 	struct scoutfs_lock *tmp;
@@ -648,8 +663,7 @@ static void lock_grant_worker(struct work_struct *work)
 	spin_lock(&linfo->lock);
 
 	list_for_each_entry_safe(lock, tmp, &linfo->grant_list, grant_head) {
-		gr = &lock->grant_resp;
-		nl = &lock->grant_resp.nl;
+		nl = &lock->grant_nl;
 
 		/* wait for reordered invalidation to finish */
 		if (lock->mode != nl->old_mode)
@@ -667,10 +681,6 @@ static void lock_grant_worker(struct work_struct *work)
 		lock->request_pending = 0;
 		lock->mode = nl->new_mode;
 		lock->write_version = le64_to_cpu(nl->write_version);
-		lock->roots = gr->roots;
-
-		if (lock_count_match_exists(nl->new_mode, lock->waiters))
-			extend_grace(sb, lock);
 
 		trace_scoutfs_lock_granted(sb, lock);
 		list_del_init(&lock->grant_head);
@@ -689,9 +699,8 @@ static void lock_grant_worker(struct work_struct *work)
  * work to process.
  */
 int scoutfs_lock_grant_response(struct super_block *sb,
-				struct scoutfs_net_lock_grant_response *gr)
+				struct scoutfs_net_lock *nl)
 {
-	struct scoutfs_net_lock *nl = &gr->nl;
 	DECLARE_LOCK_INFO(sb, linfo);
 	struct scoutfs_lock *lock;
 
@@ -705,7 +714,7 @@ int scoutfs_lock_grant_response(struct super_block *sb,
 	trace_scoutfs_lock_grant_response(sb, lock);
 	BUG_ON(!lock->request_pending);
 
-	lock->grant_resp = *gr;
+	lock->grant_nl = *nl;
 	list_add_tail(&lock->grant_head, &linfo->grant_list);
 	queue_grant_work(linfo);
 
@@ -717,7 +726,9 @@ int scoutfs_lock_grant_response(struct super_block *sb,
 /*
  * Each lock has received a lock invalidation request from the server
  * which specifies a new mode for the lock.  The server will only send
- * one invalidation request at a time for each lock.
+ * one invalidation request at a time for each lock.  The server can
+ * send another invalidate request after we send the response but before
+ * we reacquire the lock and finish invalidation.
  *
  * This is an unsolicited request from the server so it can arrive at
  * any time after we make the server aware of the lock by initially
@@ -734,11 +745,6 @@ int scoutfs_lock_grant_response(struct super_block *sb,
  * invalidate once the lock mode matches what the server told us to
  * invalidate.
  *
- * We delay invalidation processing until a grace period has elapsed
- * since the last unlock.  The intent is to let users do a reasonable
- * batch of work before dropping the lock.  Continuous unlocking can
- * continuously extend the deadline.
- *
  * Before we start invalidating the lock we set the lock to the new
  * mode, preventing further incompatible users of the old mode from
  * using the lock while we're invalidating.
@@ -750,15 +756,11 @@ int scoutfs_lock_grant_response(struct super_block *sb,
  */
 static void lock_invalidate_worker(struct work_struct *work)
 {
-	struct lock_info *linfo = container_of(work, struct lock_info,
-					       inv_dwork.work);
+	struct lock_info *linfo = container_of(work, struct lock_info, inv_work);
 	struct super_block *sb = linfo->sb;
 	struct scoutfs_net_lock *nl;
 	struct scoutfs_lock *lock;
 	struct scoutfs_lock *tmp;
-	unsigned long delay = MAX_JIFFY_OFFSET;
-	ktime_t now = ktime_get();
-	ktime_t deadline;
 	LIST_HEAD(ready);
 	u64 net_id;
 	int ret;
@@ -778,15 +780,6 @@ static void lock_invalidate_worker(struct work_struct *work)
 		if (!lock_counts_match(nl->new_mode, lock->users))
 			continue;
 
-		/* skip if grace hasn't elapsed, record earliest */
-		deadline = lock->grace_deadline;
-		if (!linfo->shutdown && ktime_before(now, deadline)) {
-			delay = min(delay,
-				    nsecs_to_jiffies(ktime_to_ns(
-						ktime_sub(deadline, now))));
-			scoutfs_inc_counter(linfo->sb, lock_grace_wait);
-			continue;
-		}
 		/* set the new mode, no incompatible users during inval */
 		lock->mode = nl->new_mode;
 
@@ -797,15 +790,21 @@ static void lock_invalidate_worker(struct work_struct *work)
 	spin_unlock(&linfo->lock);
 
 	if (list_empty(&ready))
-		goto out;
+		return;
 
 	/* invalidate once the lock is read */
 	list_for_each_entry(lock, &ready, inv_head) {
 		nl = &lock->inv_nl;
 		net_id = lock->inv_net_id;
 
-		ret = lock_invalidate(sb, lock, nl->old_mode, nl->new_mode);
-		BUG_ON(ret);
+		/* only lock protocol, inv can't call subsystems after shutdown */
+		if (!linfo->shutdown) {
+			ret = lock_invalidate(sb, lock, nl->old_mode, nl->new_mode);
+			BUG_ON(ret);
+		}
+
+		/* allow another request after we respond but before we finish */
+		lock->inv_net_id = 0;
 
 		/* respond with the key and modes from the request */
 		ret = scoutfs_client_lock_response(sb, net_id, nl);
@@ -818,53 +817,66 @@ static void lock_invalidate_worker(struct work_struct *work)
 	spin_lock(&linfo->lock);
 
 	list_for_each_entry_safe(lock, tmp, &ready, inv_head) {
-		list_del_init(&lock->inv_head);
-
-		lock->invalidate_pending = 0;
 		trace_scoutfs_lock_invalidated(sb, lock);
-		wake_up(&lock->waitq);
+		if (lock->inv_net_id == 0) {
+			/* finish if another request didn't arrive */
+			list_del_init(&lock->inv_head);
+			lock->invalidate_pending = 0;
+			wake_up(&lock->waitq);
+		} else {
+			/* another request filled nl/net_id, put it back on the list */
+			list_move_tail(&lock->inv_head, &linfo->inv_list);
+		}
 		put_lock(linfo, lock);
 	}
 
 	/* grant might have been waiting for invalidate request */
 	queue_grant_work(linfo);
 	spin_unlock(&linfo->lock);
-
-out:
-	/* queue delayed work if invalidations waiting on grace deadline */
-	if (delay != MAX_JIFFY_OFFSET)
-		queue_delayed_work(linfo->workq, &linfo->inv_dwork, delay);
 }
 
 /*
- * Record an incoming invalidate request from the server and add its lock
- * to the list for processing.
+ * Record an incoming invalidate request from the server and add its
+ * lock to the list for processing.  This request can be from a new
+ * server and racing with invalidation that frees from an old server.
+ * It's fine to not find the requested lock and send an immediate
+ * response.
  *
- * This is trusting the server and will crash if it's sent bad requests :/
+ * The invalidation process drops the linfo lock to send responses.  The
+ * moment it does so we can receive another invalidation request (the
+ * server can ask us to go from write->read then read->null).  We allow
+ * for one chain like this but it's a bug if we receive more concurrent
+ * invalidation requests than that.  The server should be only sending
+ * one at a time.
  */
 int scoutfs_lock_invalidate_request(struct super_block *sb, u64 net_id,
 				    struct scoutfs_net_lock *nl)
 {
 	DECLARE_LOCK_INFO(sb, linfo);
 	struct scoutfs_lock *lock;
+	int ret = 0;
 
 	scoutfs_inc_counter(sb, lock_invalidate_request);
 
 	spin_lock(&linfo->lock);
 	lock = get_lock(sb, &nl->key);
-	BUG_ON(!lock);
 	if (lock) {
-		BUG_ON(lock->invalidate_pending);
-		lock->invalidate_pending = 1;
-		lock->inv_nl = *nl;
+		BUG_ON(lock->inv_net_id != 0);
 		lock->inv_net_id = net_id;
-		list_add_tail(&lock->inv_head, &linfo->inv_list);
+		lock->inv_nl = *nl;
+		if (list_empty(&lock->inv_head)) {
+			list_add_tail(&lock->inv_head, &linfo->inv_list);
+			lock->invalidate_pending = 1;
+		}
 		trace_scoutfs_lock_invalidate_request(sb, lock);
 		queue_inv_work(linfo);
 	}
 	spin_unlock(&linfo->lock);
 
-	return 0;
+	if (!lock)
+		ret = scoutfs_client_lock_response(sb, net_id, nl);
+
+	return ret;
 }
 
 /*
@@ -995,7 +1007,7 @@ static int lock_key_range(struct super_block *sb, enum scoutfs_lock_mode mode, i
 	lock_inc_count(lock->waiters, mode);
 
 	for (;;) {
-		if (linfo->shutdown) {
+		if (WARN_ON_ONCE(linfo->shutdown)) {
 			ret = -ESHUTDOWN;
 			break;
 		}
@@ -1286,10 +1298,6 @@ int scoutfs_lock_rid(struct super_block *sb, enum scoutfs_lock_mode mode, int fl
 	return lock_key_range(sb, mode, flags, &start, &end, lock);
 }
 
-/*
- * As we unlock we always extend the grace period to give the caller
- * another pass at the lock before its invalidated.
- */
 void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, enum scoutfs_lock_mode mode)
 {
 	DECLARE_LOCK_INFO(sb, linfo);
@@ -1302,7 +1310,6 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, enum scou
 	spin_lock(&linfo->lock);
 
 	lock_dec_count(lock->users, mode);
-	extend_grace(sb, lock);
 	if (lock_mode_can_write(mode))
 		lock->dirty_trans_seq = scoutfs_trans_sample_seq(sb);
 
@@ -1477,7 +1484,7 @@ restart:
 		BUG_ON(lock->mode == SCOUTFS_LOCK_NULL);
 		BUG_ON(!list_empty(&lock->shrink_head));
 
-		if (linfo->shutdown || nr-- == 0)
+		if (nr-- == 0)
 			break;
 
 		__lock_del_lru(linfo, lock);
@@ -1504,7 +1511,7 @@ out:
 	return ret;
 }
 
-void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr)
+void scoutfs_free_unused_locks(struct super_block *sb)
 {
 	struct lock_info *linfo = SCOUTFS_SB(sb)->lock_info;
 	struct shrink_control sc = {
@@ -1532,15 +1539,40 @@ static void lock_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent)
 }
 
 /*
- * The caller is going to be calling _destroy soon and, critically, is
- * about to shutdown networking before calling us so that we don't get
- * any callbacks while we're destroying.  We have to ensure that we
- * won't call networking after this returns.
+ * shrink_dcache_for_umount() tears down dentries with no locking.  We
+ * need to make sure that our invalidation won't touch dentries before
+ * we return and the caller calls the generic vfs unmount path.
+ */
+void scoutfs_lock_unmount_begin(struct super_block *sb)
+{
+	DECLARE_LOCK_INFO(sb, linfo);
+
+	if (linfo) {
+		linfo->unmounting = true;
+		flush_work(&linfo->inv_work);
+	}
+}
+
+/*
+ * The caller is going to be shutting down transactions and the client.
+ * We need to make sure that locking won't call either after we return.
  *
- * Internal fs threads can be using locking, and locking can have async
- * work pending.  We use ->shutdown to force callers to return
- * -ESHUTDOWN and to prevent the future queueing of work that could call
- * networking.  Locks whose work is stopped will be torn down by _destroy.
+ * At this point all fs callers and internal services that use locks
+ * should have stopped.  We won't have any callers initiating lock
+ * transitions and sending requests.   We set the shutdown flag to catch
+ * anyone who breaks this rule.
+ *
+ * We unregister the shrinker so that we won't try and send null
+ * requests in response to memory pressure.  The locks will all be
+ * unceremoniously dropped once we get a farewell response from the
+ * server which indicates that they destroyed our locking state.
+ *
+ * We will still respond to invalidation requests that have to be
+ * processed to let unmount in other mounts acquire locks and make
+ * progress.  However, we don't fully process the invalidation because
+ * we're shutting down.  We only update the lock state and send the
+ * response.  We shouldn't have any users of locking that require
+ * invalidation correctness at this point.
  */
 void scoutfs_lock_shutdown(struct super_block *sb)
 {
@@ -1553,19 +1585,18 @@ void scoutfs_lock_shutdown(struct super_block *sb)
 
 	trace_scoutfs_lock_shutdown(sb, linfo);
 
-	spin_lock(&linfo->lock);
+	/* stop the shrinker from queueing work */
+	unregister_shrinker(&linfo->shrinker);
+	flush_work(&linfo->shrink_work);
 
+	/* cause current and future lock calls to return errors */
+	spin_lock(&linfo->lock);
 	linfo->shutdown = true;
 	for (node = rb_first(&linfo->lock_tree); node; node = rb_next(node)) {
 		lock = rb_entry(node, struct scoutfs_lock, node);
 		wake_up(&lock->waitq);
 	}
-
 	spin_unlock(&linfo->lock);
-
-	flush_work(&linfo->grant_work);
-	flush_delayed_work(&linfo->inv_dwork);
-	flush_work(&linfo->shrink_work);
 }
 
 /*
@@ -1593,8 +1624,6 @@ void scoutfs_lock_destroy(struct super_block *sb)
 
 	trace_scoutfs_lock_destroy(sb, linfo);
 
-	/* stop the shrinker from queueing work */
-	unregister_shrinker(&linfo->shrinker);
 
 	/* make sure that no one's actively using locks */
 	spin_lock(&linfo->lock);
@@ -1613,8 +1642,6 @@ void scoutfs_lock_destroy(struct super_block *sb)
 	spin_unlock(&linfo->lock);
 
 	if (linfo->workq) {
-		/* pending grace work queues normal work */
-		flush_workqueue(linfo->workq);
 		/* now all work won't queue itself */
 		destroy_workqueue(linfo->workq);
 	}
@@ -1640,8 +1667,10 @@ void scoutfs_lock_destroy(struct super_block *sb)
 			__lock_del_lru(linfo, lock);
 		if (!list_empty(&lock->grant_head))
 			list_del_init(&lock->grant_head);
-		if (!list_empty(&lock->inv_head))
+		if (!list_empty(&lock->inv_head)) {
 			list_del_init(&lock->inv_head);
+			lock->invalidate_pending = 0;
+		}
 		if (!list_empty(&lock->shrink_head))
 			list_del_init(&lock->shrink_head);
 		lock_remove(linfo, lock);
@@ -1673,11 +1702,13 @@ int scoutfs_lock_setup(struct super_block *sb)
 	INIT_LIST_HEAD(&linfo->lru_list);
 	INIT_WORK(&linfo->grant_work, lock_grant_worker);
 	INIT_LIST_HEAD(&linfo->grant_list);
-	INIT_DELAYED_WORK(&linfo->inv_dwork, lock_invalidate_worker);
+	INIT_WORK(&linfo->inv_work, lock_invalidate_worker);
 	INIT_LIST_HEAD(&linfo->inv_list);
 	INIT_WORK(&linfo->shrink_work, lock_shrink_worker);
 	INIT_LIST_HEAD(&linfo->shrink_list);
 	atomic64_set(&linfo->next_refresh_gen, 0);
+	INIT_WORK(&linfo->inv_iput_work, lock_inv_iput_worker);
+	init_llist_head(&linfo->inv_iput_llist);
 	scoutfs_tseq_tree_init(&linfo->tseq_tree, lock_tseq_show);
 
 	sbi->lock_info = linfo;

kmod/src/lock.h

@@ -10,6 +10,8 @@
 
 #define SCOUTFS_LOCK_NR_MODES		SCOUTFS_LOCK_INVALID
 
+struct scoutfs_omap_lock;
+
 /*
  * A few fields (start, end, refresh_gen, write_version, granted_mode)
  * are referenced by code outside lock.c.
@@ -23,15 +25,13 @@ struct scoutfs_lock {
 	u64 refresh_gen;
 	u64 write_version;
 	u64 dirty_trans_seq;
-	struct scoutfs_net_roots roots;
 	struct list_head lru_head;
 	wait_queue_head_t waitq;
-	ktime_t grace_deadline;
 	unsigned long request_pending:1,
 		      invalidate_pending:1;
 
 	struct list_head grant_head;
-	struct scoutfs_net_lock_grant_response grant_resp;
+	struct scoutfs_net_lock grant_nl;
 	struct list_head inv_head;
 	struct scoutfs_net_lock inv_nl;
 	u64 inv_net_id;
@@ -48,6 +48,10 @@ struct scoutfs_lock {
 
 	/* the forest tracks which log tree last saw bloom bit updates */
 	atomic64_t forest_bloom_nr;
+
+	/* open ino mapping has a valid map for a held write lock */
+	spinlock_t omap_spinlock;
+	struct scoutfs_omap_lock_data *omap_data;
 };
 
 struct scoutfs_lock_coverage {
@@ -57,7 +61,7 @@ struct scoutfs_lock_coverage {
 };
 
 int scoutfs_lock_grant_response(struct super_block *sb,
-				struct scoutfs_net_lock_grant_response *gr);
+				struct scoutfs_net_lock *nl);
 int scoutfs_lock_invalidate_request(struct super_block *sb, u64 net_id,
 				    struct scoutfs_net_lock *nl);
 int scoutfs_lock_recover_request(struct super_block *sb, u64 net_id,
@@ -96,9 +100,10 @@ void scoutfs_lock_del_coverage(struct super_block *sb,
 bool scoutfs_lock_protected(struct scoutfs_lock *lock, struct scoutfs_key *key,
 			    enum scoutfs_lock_mode mode);
 
-void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr);
+void scoutfs_free_unused_locks(struct super_block *sb);
 
 int scoutfs_lock_setup(struct super_block *sb);
+void scoutfs_lock_unmount_begin(struct super_block *sb);
 void scoutfs_lock_shutdown(struct super_block *sb);
 void scoutfs_lock_destroy(struct super_block *sb);
 

kmod/src/lock_server.c

@@ -20,10 +20,10 @@
 #include "tseq.h"
 #include "spbm.h"
 #include "block.h"
-#include "btree.h"
 #include "msg.h"
 #include "scoutfs_trace.h"
 #include "lock_server.h"
+#include "recov.h"
 
 /*
  * The scoutfs server implements a simple lock service.  Client mounts
@@ -56,14 +56,11 @@
  * Message requests and responses are reliably delivered in order across
  * reconnection.
  *
- * The server maintains a persistent record of connected clients.  A new
- * server instance discovers these and waits for previously connected
- * clients to reconnect and recover their state before proceeding.  If
- * clients don't reconnect they are forcefully prevented from unsafely
- * accessing the shared persistent storage.  (fenced, according to the
- * rules of the platform.. could range from being powered off to having
- * their switch port disabled to having their local block device set
- * read-only.)
+ * As a new server comes up it recovers lock state from existing clients
+ * which were connected to a previous lock server.  Recover requests are
+ * sent to clients as they connect and they respond with all there
+ * locks.  Once all clients and locks are accounted for normal
+ * processing can resume.
  *
  * The lock server doesn't respond to memory pressure.  The only way
  * locks are freed is if they are invalidated to null on behalf of a
@@ -77,14 +74,12 @@ struct lock_server_info {
 	struct super_block *sb;
 
 	spinlock_t lock;
-	struct mutex mutex;
 	struct rb_root locks_root;
 
-	struct scoutfs_spbm recovery_pending;
-	struct delayed_work recovery_dwork;
-
 	struct scoutfs_tseq_tree tseq_tree;
 	struct dentry *tseq_dentry;
+	struct scoutfs_tseq_tree stats_tseq_tree;
+	struct dentry *stats_tseq_dentry;
 
 	struct scoutfs_alloc *alloc;
 	struct scoutfs_block_writer *wri;
@@ -116,6 +111,9 @@ struct server_lock_node {
 	struct list_head granted;
 	struct list_head requested;
 	struct list_head invalidated;
+
+	struct scoutfs_tseq_entry stats_tseq_entry;
+	u64 stats[SLT_NR];
 };
 
 /*
@@ -305,6 +303,8 @@ static struct server_lock_node *alloc_server_lock(struct lock_server_info *inf,
 			snode = get_server_lock(inf, key, ins, false);
 			if (snode != ins)
 				kfree(ins);
+			else
+				scoutfs_tseq_add(&inf->stats_tseq_tree, &snode->stats_tseq_entry);
 		}
 	}
 
@@ -334,8 +334,10 @@ static void put_server_lock(struct lock_server_info *inf,
 
 	mutex_unlock(&snode->mutex);
 
-	if (should_free)
+	if (should_free) {
+		scoutfs_tseq_del(&inf->stats_tseq_tree, &snode->stats_tseq_entry);
 		kfree(snode);
+	}
 }
 
 static struct client_lock_entry *find_entry(struct server_lock_node *snode,
@@ -397,6 +399,8 @@ int scoutfs_lock_server_request(struct super_block *sb, u64 rid,
 		goto out;
 	}
 
+	snode->stats[SLT_REQUEST]++;
+
 	clent->snode = snode;
 	add_client_entry(snode, &snode->requested, clent);
 	scoutfs_tseq_add(&inf->tseq_tree, &clent->tseq_entry);
@@ -430,13 +434,15 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 rid,
 		goto out;
 	}
 
-	/* XXX should always have a server lock here?  recovery? */
+	/* XXX should always have a server lock here? */
 	snode = get_server_lock(inf, &nl->key, NULL, false);
 	if (!snode) {
 		ret = -EINVAL;
 		goto out;
 	}
 
+	snode->stats[SLT_RESPONSE]++;
+
 	clent = find_entry(snode, &snode->invalidated, rid);
 	if (!clent) {
 		put_server_lock(inf, snode);
@@ -473,18 +479,14 @@ out:
  * so we unlock the snode mutex.
  *
  * All progress must wait for all clients to finish with recovery
- * because we don't know which locks they'll hold.  The unlocked
- * recovery_pending test here is OK.  It's filled by setup before
- * anything runs.  It's emptied by recovery completion.  We can get a
- * false nonempty result if we race with recovery completion, but that's
- * OK because recovery completion processes all the locks that have
- * requests after emptying, including the unlikely loser of that race.
+ * because we don't know which locks they'll hold.  Once recover
+ * finishes the server calls us to kick all the locks that were waiting
+ * during recovery.
  */
 static int process_waiting_requests(struct super_block *sb,
 				    struct server_lock_node *snode)
 {
 	DECLARE_LOCK_SERVER_INFO(sb, inf);
-	struct scoutfs_net_lock_grant_response gres;
 	struct scoutfs_net_lock nl;
 	struct client_lock_entry *req;
 	struct client_lock_entry *req_tmp;
@@ -497,7 +499,7 @@ static int process_waiting_requests(struct super_block *sb,
 
 	/* processing waits for all invalidation responses or recovery */
 	if (!list_empty(&snode->invalidated) ||
-	    !scoutfs_spbm_empty(&inf->recovery_pending)) {
+	    scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_LOCKS) != 0) {
 		ret = 0;
 		goto out;
 	}
@@ -521,6 +523,7 @@ static int process_waiting_requests(struct super_block *sb,
 			trace_scoutfs_lock_message(sb, SLT_SERVER,
 						   SLT_INVALIDATE, SLT_REQUEST,
 						   gr->rid, 0, &nl);
+			snode->stats[SLT_INVALIDATE]++;
 
 			add_client_entry(snode, &snode->invalidated, gr);
 		}
@@ -547,17 +550,15 @@ static int process_waiting_requests(struct super_block *sb,
 			nl.write_version = cpu_to_le64(wv);
 		}
 
-		gres.nl = nl;
-		scoutfs_server_get_roots(sb, &gres.roots);
-
 		ret = scoutfs_server_lock_response(sb, req->rid,
-						   req->net_id, &gres);
+						   req->net_id, &nl);
 		if (ret)
 			goto out;
 
 		trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_GRANT,
 					   SLT_RESPONSE, req->rid,
 					   req->net_id, &nl);
+		snode->stats[SLT_GRANT]++;
 
 		/* don't track null client locks, track all else */ 
 		if (req->mode == SCOUTFS_LOCK_NULL)
@@ -573,85 +574,39 @@ out:
 	return ret;
 }
 
-static void init_lock_clients_key(struct scoutfs_key *key, u64 rid)
-{
-	*key = (struct scoutfs_key) {
-		.sk_zone = SCOUTFS_LOCK_CLIENTS_ZONE,
-		.sklc_rid = cpu_to_le64(rid),
-	};
-}
-
 /*
  * The server received a greeting from a client for the first time.  If
- * the client had already talked to the server then we must find an
- * existing record for it and should begin recovery.  If it doesn't have
- * a record then its timed out and we can't allow it to reconnect.  If
- * its connecting for the first time then we insert a new record.  If
+ * the client is in lock recovery then we send the initial lock request.
  *
  * This is running in concurrent client greeting processing contexts.
  */
-int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid,
-				 bool should_exist)
+int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid)
 {
-	DECLARE_LOCK_SERVER_INFO(sb, inf);
-	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
-	SCOUTFS_BTREE_ITEM_REF(iref);
 	struct scoutfs_key key;
 	int ret;
 
-	init_lock_clients_key(&key, rid);
-
-	mutex_lock(&inf->mutex);
-	if (should_exist) {
-		ret = scoutfs_btree_lookup(sb, &super->lock_clients, &key,
-					   &iref);
-		if (ret == 0)
-			scoutfs_btree_put_iref(&iref);
-	} else {
-		ret = scoutfs_btree_insert(sb, inf->alloc, inf->wri,
-					   &super->lock_clients,
-					   &key, NULL, 0);
-	}
-	mutex_unlock(&inf->mutex);
-
-	if (should_exist && ret == 0) {
+	if (scoutfs_recov_is_pending(sb, rid, SCOUTFS_RECOV_LOCKS)) {
 		scoutfs_key_set_zeros(&key);
 		ret = scoutfs_server_lock_recover_request(sb, rid, &key);
-		if (ret)
-			goto out;
+	} else {
+		ret = 0;
 	}
 
-out:
 	return ret;
 }
 
 /*
- * A client sent their last recovery response and can exit recovery.  If
- * they were the last client in recovery then we can process all the
- * server locks that had requests.
+ * All clients have finished lock recovery, we can make forward process
+ * on all the queued requests that were waiting on recovery.
  */
-static int finished_recovery(struct super_block *sb, u64 rid, bool cancel)
+int scoutfs_lock_server_finished_recovery(struct super_block *sb)
 {
 	DECLARE_LOCK_SERVER_INFO(sb, inf);
 	struct server_lock_node *snode;
 	struct scoutfs_key key;
-	bool still_pending;
 	int ret = 0;
 
-	spin_lock(&inf->lock);
-	scoutfs_spbm_clear(&inf->recovery_pending, rid);
-	still_pending = !scoutfs_spbm_empty(&inf->recovery_pending);
-	spin_unlock(&inf->lock);
-	if (still_pending)
-		return 0;
-
-	if (cancel)
-		cancel_delayed_work_sync(&inf->recovery_dwork);
-
 	scoutfs_key_set_zeros(&key);
-
-	scoutfs_info(sb, "all lock clients recovered");
-
 	while ((snode = get_server_lock(inf, &key, NULL, true))) {
 
 		key = snode->key;
@@ -695,16 +650,15 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid,
 	int i;
 
 	/* client must be in recovery */
-	spin_lock(&inf->lock);
-	if (!scoutfs_spbm_test(&inf->recovery_pending, rid))
+	if (!scoutfs_recov_is_pending(sb, rid, SCOUTFS_RECOV_LOCKS)) {
 		ret = -EINVAL;
-	spin_unlock(&inf->lock);
-	if (ret)
 		goto out;
+	}
 
 	/* client has sent us all their locks */
 	if (nlr->nr == 0) {
-		ret = finished_recovery(sb, rid, true);
+		scoutfs_server_recov_finish(sb, rid, SCOUTFS_RECOV_LOCKS);
+		ret = 0;
 		goto out;
 	}
 
@@ -755,101 +709,15 @@ out:
 	return ret;
 }
 
-static int get_rid_and_put_ref(struct scoutfs_btree_item_ref *iref, u64 *rid)
-{
-	int ret;
-
-	if (iref->val_len == 0) {
-		*rid = le64_to_cpu(iref->key->sklc_rid);
-		ret = 0;
-	} else {
-		ret = -EIO;
-	}
-	scoutfs_btree_put_iref(iref);
-	return ret;
-}
-
-/*
- * This work executes if enough time passes without all of the clients
- * finishing with recovery and canceling the work.  We walk through the
- * client records and find any that still have their recovery pending.
- */
-static void scoutfs_lock_server_recovery_timeout(struct work_struct *work)
-{
-	struct lock_server_info *inf = container_of(work,
-						    struct lock_server_info,
-						    recovery_dwork.work);
-	struct super_block *sb = inf->sb;
-	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
-	SCOUTFS_BTREE_ITEM_REF(iref);
-	struct scoutfs_key key;
-	bool timed_out;
-	u64 rid;
-	int ret;
-
-	ret = scoutfs_server_hold_commit(sb);
-	if (ret)
-		goto out;
-
-	/* we enter recovery if there are any client records */
-	for (rid = 0; ; rid++) {
-		init_lock_clients_key(&key, rid);
-		ret = scoutfs_btree_next(sb, &super->lock_clients, &key, &iref);
-		if (ret == -ENOENT) {
-			ret = 0;
-			break;
-		}
-		if (ret == 0)
-			ret = get_rid_and_put_ref(&iref, &rid);
-		if (ret < 0)
-			break;
-
-		spin_lock(&inf->lock);
-		if (scoutfs_spbm_test(&inf->recovery_pending, rid)) {
-			scoutfs_spbm_clear(&inf->recovery_pending, rid);
-			timed_out = true;
-		} else {
-			timed_out = false;
-		}
-		spin_unlock(&inf->lock);
-
-		if (!timed_out)
-			continue;
-
-		scoutfs_err(sb, "client rid %016llx lock recovery timed out",
-			    rid);
-
-		init_lock_clients_key(&key, rid);
-		ret = scoutfs_btree_delete(sb, inf->alloc, inf->wri,
-					   &super->lock_clients, &key);
-		if (ret)
-			break;
-	}
-
-	ret = scoutfs_server_apply_commit(sb, ret);
-out:
-	/* force processing all pending lock requests */
-	if (ret == 0)
-		ret = finished_recovery(sb, 0, false);
-
-	if (ret < 0) {
-		scoutfs_err(sb, "lock server saw err %d while timing out clients, shutting down", ret);
-		scoutfs_server_abort(sb);
-	}
-}
-
 /*
  * A client is leaving the lock service.  They aren't using locks and
  * won't send any more requests.  We tear down all the state we had for
  * them.  This can be called multiple times for a given client as their
  * farewell is resent to new servers.  It's OK to not find any state.
- * If we fail to delete a persistent entry then we have to shut down and
- * hope that the next server has more luck.
  */
 int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid)
 {
 	DECLARE_LOCK_SERVER_INFO(sb, inf);
-	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
 	struct client_lock_entry *clent;
 	struct client_lock_entry *tmp;
 	struct server_lock_node *snode;
@@ -858,20 +726,7 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid)
 	bool freed;
 	int ret = 0;
 
-	mutex_lock(&inf->mutex);
-	init_lock_clients_key(&key, rid);
-	ret = scoutfs_btree_delete(sb, inf->alloc, inf->wri,
-				   &super->lock_clients, &key);
-	mutex_unlock(&inf->mutex);
-	if (ret == -ENOENT) {
-		ret = 0;
-		goto out;
-	}
-	if (ret < 0)
-		goto out;
-
 	scoutfs_key_set_zeros(&key);
-
 	while ((snode = get_server_lock(inf, &key, NULL, true))) {
 
 		freed = false;
@@ -954,25 +809,26 @@ static void lock_server_tseq_show(struct seq_file *m,
 		   clent->net_id);
 }
 
+static void stats_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent)
+{
+	struct server_lock_node *snode = container_of(ent, struct server_lock_node,
+						      stats_tseq_entry);
+
+	seq_printf(m, SK_FMT" req %llu inv %llu rsp %llu gr %llu\n",
+		   SK_ARG(&snode->key), snode->stats[SLT_REQUEST], snode->stats[SLT_INVALIDATE],
+		   snode->stats[SLT_RESPONSE], snode->stats[SLT_GRANT]);
+}
+
 /*
  * Setup the lock server.  This is called before networking can deliver
- * requests.  If we find existing client records then we enter recovery.
- * Lock request processing is deferred until recovery is resolved for
- * all the existing clients, either they reconnect and replay locks or
- * we time them out.
+ * requests.
  */
 int scoutfs_lock_server_setup(struct super_block *sb,
 			      struct scoutfs_alloc *alloc,
 			      struct scoutfs_block_writer *wri, u64 max_vers)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
-	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
 	struct lock_server_info *inf;
-	SCOUTFS_BTREE_ITEM_REF(iref);
-	struct scoutfs_key key;
-	unsigned int nr;
-	u64 rid;
-	int ret;
 
 	inf = kzalloc(sizeof(struct lock_server_info), GFP_KERNEL);
 	if (!inf)
@@ -980,12 +836,9 @@ int scoutfs_lock_server_setup(struct super_block *sb,
 
 	inf->sb = sb;
 	spin_lock_init(&inf->lock);
-	mutex_init(&inf->mutex);
 	inf->locks_root = RB_ROOT;
-	scoutfs_spbm_init(&inf->recovery_pending);
-	INIT_DELAYED_WORK(&inf->recovery_dwork,
-			  scoutfs_lock_server_recovery_timeout);
 	scoutfs_tseq_tree_init(&inf->tseq_tree, lock_server_tseq_show);
+	scoutfs_tseq_tree_init(&inf->stats_tseq_tree, stats_tseq_show);
 	inf->alloc = alloc;
 	inf->wri = wri;
 	atomic64_set(&inf->write_version, max_vers); /* inc_return gives +1 */
@@ -997,38 +850,17 @@ int scoutfs_lock_server_setup(struct super_block *sb,
 		return -ENOMEM;
 	}
 
+	inf->stats_tseq_dentry = scoutfs_tseq_create("tmp_lock_stats", sbi->debug_root,
+					       &inf->stats_tseq_tree);
+	if (!inf->stats_tseq_dentry) {
+		debugfs_remove(inf->tseq_dentry);
+		kfree(inf);
+		return -ENOMEM;
+	}
+
 	sbi->lock_server_info = inf;
 
-	/* we enter recovery if there are any client records */
-	nr = 0;
-	for (rid = 0; ; rid++) {
-		init_lock_clients_key(&key, rid);
-		ret = scoutfs_btree_next(sb, &super->lock_clients, &key, &iref);
-		if (ret == -ENOENT)
-			break;
-		if (ret == 0)
-			ret = get_rid_and_put_ref(&iref, &rid);
-		if (ret < 0)
-			goto out;
-
-		ret = scoutfs_spbm_set(&inf->recovery_pending, rid);
-		if (ret)
-			goto out;
-		nr++;
-
-		if (rid == U64_MAX)
-			break;
-	}
-	ret = 0;
-
-	if (nr) {
-		schedule_delayed_work(&inf->recovery_dwork,
-				msecs_to_jiffies(LOCK_SERVER_RECOVERY_MS));
-		scoutfs_info(sb, "waiting for %u lock clients to recover", nr);
-	}
-
-out:
-	return ret;
+	return 0;
 }
 
 /*
@@ -1046,9 +878,8 @@ void scoutfs_lock_server_destroy(struct super_block *sb)
 	LIST_HEAD(list);
 
 	if (inf) {
-		cancel_delayed_work_sync(&inf->recovery_dwork);
-
 		debugfs_remove(inf->tseq_dentry);
+		debugfs_remove(inf->stats_tseq_dentry);
 
 		rbtree_postorder_for_each_entry_safe(snode, stmp,
 						     &inf->locks_root, node) {
@@ -1066,8 +897,6 @@ void scoutfs_lock_server_destroy(struct super_block *sb)
 			kfree(snode);
 		}
 
-		scoutfs_spbm_destroy(&inf->recovery_pending);
-
 		kfree(inf);
 		sbi->lock_server_info = NULL;
 	}

kmod/src/lock_server.h

@@ -3,10 +3,10 @@
 
 int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid,
 					 struct scoutfs_net_lock_recover *nlr);
+int scoutfs_lock_server_finished_recovery(struct super_block *sb);
 int scoutfs_lock_server_request(struct super_block *sb, u64 rid,
 				u64 net_id, struct scoutfs_net_lock *nl);
-int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid,
-				 bool should_exist);
+int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid);
 int scoutfs_lock_server_response(struct super_block *sb, u64 rid,
 				 struct scoutfs_net_lock *nl);
 int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid);

kmod/src/net.c

@@ -944,7 +944,6 @@ static void scoutfs_net_listen_worker(struct work_struct *work)
 	struct scoutfs_net_connection *acc_conn;
 	DECLARE_WAIT_QUEUE_HEAD(waitq);
 	struct socket *acc_sock;
-	LIST_HEAD(conn_list);
 	int ret;
 
 	trace_scoutfs_net_listen_work_enter(sb, 0, 0);
@@ -1546,9 +1545,8 @@ void scoutfs_net_client_greeting(struct super_block *sb,
  * response and they can disconnect cleanly.
  *
  * At this point our connection is idle except for send submissions and
- * shutdown being queued.  Once we shut down a We completely own a We
- * have exclusive access to a previous conn once its shutdown and we set
- * _freeing.
+ * shutdown being queued.  We have exclusive access to the previous conn
+ * once it's shutdown and we set _freeing.
  */
 void scoutfs_net_server_greeting(struct super_block *sb,
 				 struct scoutfs_net_connection *conn,

kmod/src/net.h

@@ -90,19 +90,13 @@ enum conn_flags {
 #define SIN_ARG(sin)	sin, be16_to_cpu((sin)->sin_port)
 
 static inline void scoutfs_addr_to_sin(struct sockaddr_in *sin,
-				       struct scoutfs_inet_addr *addr)
+				       union scoutfs_inet_addr *addr)
 {
-	sin->sin_family = AF_INET;
-	sin->sin_addr.s_addr = cpu_to_be32(le32_to_cpu(addr->addr));
-	sin->sin_port = cpu_to_be16(le16_to_cpu(addr->port));
-}
+	BUG_ON(addr->v4.family != cpu_to_le16(SCOUTFS_AF_IPV4));
 
-static inline void scoutfs_addr_from_sin(struct scoutfs_inet_addr *addr,
-					 struct sockaddr_in *sin)
-{
-	addr->addr = be32_to_le32(sin->sin_addr.s_addr);
-	addr->port = be16_to_le16(sin->sin_port);
-	memset(addr->__pad, 0, sizeof(addr->__pad));
+	sin->sin_family = AF_INET;
+	sin->sin_addr.s_addr = cpu_to_be32(le32_to_cpu(addr->v4.addr));
+	sin->sin_port = cpu_to_be16(le16_to_cpu(addr->v4.port));
 }
 
 struct scoutfs_net_connection *

kmod/src/omap.h

@@ -0,0 +1,24 @@
+#ifndef _SCOUTFS_OMAP_H_
+#define _SCOUTFS_OMAP_H_
+
+int scoutfs_omap_inc(struct super_block *sb, u64 ino);
+void scoutfs_omap_dec(struct super_block *sb, u64 ino);
+int scoutfs_omap_should_delete(struct super_block *sb, struct inode *inode,
+			       struct scoutfs_lock **lock_ret);
+void scoutfs_omap_free_lock_data(struct scoutfs_omap_lock_data *ldata);
+int scoutfs_omap_client_handle_request(struct super_block *sb, u64 id,
+				       struct scoutfs_open_ino_map_args *args);
+
+int scoutfs_omap_add_rid(struct super_block *sb, u64 rid);
+int scoutfs_omap_remove_rid(struct super_block *sb, u64 rid);
+int scoutfs_omap_finished_recovery(struct super_block *sb);
+int scoutfs_omap_server_handle_request(struct super_block *sb, u64 rid, u64 id,
+				       struct scoutfs_open_ino_map_args *args);
+int scoutfs_omap_server_handle_response(struct super_block *sb, u64 rid,
+					struct scoutfs_open_ino_map *resp_map);
+void scoutfs_omap_server_shutdown(struct super_block *sb);
+
+int scoutfs_omap_setup(struct super_block *sb);
+void scoutfs_omap_destroy(struct super_block *sb);
+
+#endif

kmod/src/options.c

@@ -28,7 +28,7 @@
 #include "super.h"
 
 static const match_table_t tokens = {
-	{Opt_server_addr, "server_addr=%s"},
+	{Opt_quorum_slot_nr, "quorum_slot_nr=%s"},
 	{Opt_metadev_path, "metadev_path=%s"},
 	{Opt_err, NULL}
 };
@@ -43,46 +43,6 @@ u32 scoutfs_option_u32(struct super_block *sb, int token)
 	return 0;
 }
 
-/* The caller's string is null terminted and can be clobbered */
-static int parse_ipv4(struct super_block *sb, char *str,
-		      struct sockaddr_in *sin)
-{
-	unsigned long port = 0;
-	__be32 addr;
-	char *c;
-	int ret;
-
-	/* null term port, if specified */
-	c = strchr(str, ':');
-	if (c)
-		*c = '\0';
-
-	/* parse addr */
-	addr = in_aton(str);
-	if (ipv4_is_multicast(addr) || ipv4_is_lbcast(addr) ||
-	    ipv4_is_zeronet(addr) ||
-	    ipv4_is_local_multicast(addr)) {
-		scoutfs_err(sb, "invalid unicast ipv4 address: %s", str);
-		return -EINVAL;
-	}
-
-	/* parse port, if specified */
-	if (c) {
-		c++;
-		ret = kstrtoul(c, 0, &port);
-		if (ret != 0 || port == 0 || port >= U16_MAX) {
-			scoutfs_err(sb, "invalid port in ipv4 address: %s", c);
-			return -EINVAL;
-		}
-	}
-
-	sin->sin_family = AF_INET;
-	sin->sin_addr.s_addr = addr;
-	sin->sin_port = cpu_to_be16(port);
-
-	return 0;
-}
-
 static int parse_bdev_path(struct super_block *sb, substring_t *substr,
 			      char **bdev_path_ret)
 {
@@ -132,14 +92,15 @@ out:
 int scoutfs_parse_options(struct super_block *sb, char *options,
 			  struct mount_options *parsed)
 {
-	char ipstr[INET_ADDRSTRLEN + 1];
 	substring_t args[MAX_OPT_ARGS];
+	int nr;
 	int token;
 	char *p;
 	int ret;
 
 	/* Set defaults */
 	memset(parsed, 0, sizeof(*parsed));
+	parsed->quorum_slot_nr = -1;
 
 	while ((p = strsep(&options, ",")) != NULL) {
 		if (!*p)
@@ -147,12 +108,23 @@ int scoutfs_parse_options(struct super_block *sb, char *options,
 
 		token = match_token(p, tokens, args);
 		switch (token) {
-		case Opt_server_addr:
+		case Opt_quorum_slot_nr:
 
-			match_strlcpy(ipstr, args, ARRAY_SIZE(ipstr));
-			ret = parse_ipv4(sb, ipstr, &parsed->server_addr);
-			if (ret < 0)
+			if (parsed->quorum_slot_nr != -1) {
+				scoutfs_err(sb, "multiple quorum_slot_nr options provided, only provide one.");
+				return -EINVAL;
+			}
+
+			ret = match_int(args, &nr);
+			if (ret < 0 || nr < 0 ||
+			    nr >= SCOUTFS_QUORUM_MAX_SLOTS) {
+				scoutfs_err(sb, "invalid quorum_slot_nr option, must be between 0 and %u",
+					    SCOUTFS_QUORUM_MAX_SLOTS - 1);
+				if (ret == 0)
+					ret = -EINVAL;
 				return ret;
+			}
+			parsed->quorum_slot_nr = nr;
 			break;
 		case Opt_metadev_path:
 

kmod/src/options.h

@@ -6,13 +6,13 @@
 #include "format.h"
 
 enum scoutfs_mount_options {
-	Opt_server_addr,
+	Opt_quorum_slot_nr,
 	Opt_metadev_path,
 	Opt_err,
 };
 
 struct mount_options {
-	struct sockaddr_in server_addr;
+	int quorum_slot_nr;
 	char *metadev_path;
 };
 

kmod/src/quorum.h

@@ -1,10 +1,15 @@
 #ifndef _SCOUTFS_QUORUM_H_
 #define _SCOUTFS_QUORUM_H_
 
-int scoutfs_quorum_election(struct super_block *sb, ktime_t timeout_abs,
-			    u64 prev_term, u64 *elected_term);
-void scoutfs_quorum_clear_leader(struct super_block *sb);
+int scoutfs_quorum_server_sin(struct super_block *sb, struct sockaddr_in *sin);
+void scoutfs_quorum_server_shutdown(struct super_block *sb);
+
+u8 scoutfs_quorum_votes_needed(struct super_block *sb);
+void scoutfs_quorum_slot_sin(struct scoutfs_super_block *super, int i,
+			     struct sockaddr_in *sin);
 
 int scoutfs_quorum_setup(struct super_block *sb);
+void scoutfs_quorum_shutdown(struct super_block *sb);
 void scoutfs_quorum_destroy(struct super_block *sb);
+
 #endif

kmod/src/recov.c

@@ -0,0 +1,280 @@
+/*
+ * Copyright (C) 2021 Versity Software, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/rhashtable.h>
+#include <linux/rcupdate.h>
+
+#include "super.h"
+#include "recov.h"
+
+/*
+ * There are a few server messages which can't be processed until they
+ * know that they have state for all possibly active clients.  These
+ * little helpers track which clients have recovered what state and give
+ * those message handlers a call to check if recovery has completed.  We
+ * track the timeout here, but all we do is call back into the server to
+ * take steps to evict timed out clients and then let us know that their
+ * recovery has finished.
+ */
+
+struct recov_info {
+	struct super_block *sb;
+	spinlock_t lock;
+	struct list_head pending;
+	struct timer_list timer;
+	void (*timeout_fn)(struct super_block *);
+};
+
+#define DECLARE_RECOV_INFO(sb, name) \
+	struct recov_info *name = SCOUTFS_SB(sb)->recov_info
+
+struct recov_pending {
+	struct list_head head;
+	u64 rid;
+	int which;
+};
+
+static struct recov_pending *find_pending(struct recov_info *recinf, u64 rid, int which)
+{
+	struct recov_pending *pend;
+
+	list_for_each_entry(pend, &recinf->pending, head) {
+		if ((rid == 0 || pend->rid == rid) && (pend->which & which))
+			return pend;
+	}
+
+	return NULL;
+}
+
+/*
+ * Record that we'll be waiting for a client to recover something.
+ * _finished will eventually be called for every _prepare, either
+ * because recovery naturally finished or because it timed out and the
+ * server evicted the client. 
+ */
+int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which)
+{
+	DECLARE_RECOV_INFO(sb, recinf);
+	struct recov_pending *alloc;
+	struct recov_pending *pend;
+
+	if (WARN_ON_ONCE(which & SCOUTFS_RECOV_INVALID))
+		return -EINVAL;
+
+	alloc = kmalloc(sizeof(*pend), GFP_NOFS);
+	if (!alloc)
+		return -ENOMEM;
+
+	spin_lock(&recinf->lock);
+
+	pend = find_pending(recinf, rid, SCOUTFS_RECOV_ALL);
+	if (pend) {
+		pend->which |= which;
+	} else {
+		swap(pend, alloc);
+		pend->rid = rid;
+		pend->which = which;
+		list_add(&pend->head, &recinf->pending);
+	}
+
+	spin_unlock(&recinf->lock);
+
+	kfree(alloc);
+	return 0;
+}
+
+/*
+ * Recovery is only finished once we've begun (which sets the timer) and
+ * all clients have finished.  If we didn't test the timer we could
+ * claim it finished prematurely as clients are being prepared.
+ */
+static int recov_finished(struct recov_info *recinf)
+{
+	return !!(recinf->timeout_fn != NULL && list_empty(&recinf->pending));
+}
+
+static void timer_callback(struct timer_list *timer)
+{
+	struct recov_info *recinf = from_timer(recinf, timer, timer);
+
+	recinf->timeout_fn(recinf->sb);
+}
+
+/*
+ * Begin waiting for recovery once we've prepared all the clients.  If
+ * the timeout period elapses before _finish is called on all prepared
+ * clients then the timer will call the callback.
+ *
+ * Returns > 0 if all the prepared clients finish recovery before begin
+ * is called.
+ */
+int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_block *),
+			unsigned int timeout_ms)
+{
+	DECLARE_RECOV_INFO(sb, recinf);
+	int ret;
+
+	spin_lock(&recinf->lock);
+
+	recinf->timeout_fn = timeout_fn;
+	recinf->timer.expires = jiffies + msecs_to_jiffies(timeout_ms);
+	add_timer(&recinf->timer);
+
+	ret = recov_finished(recinf);
+
+	spin_unlock(&recinf->lock);
+
+	if (ret > 0)
+		del_timer_sync(&recinf->timer);
+
+	return ret;
+}
+
+/*
+ * A given client has recovered the given state.  If it's finished all
+ * recovery then we free it, and if all clients have finished recovery
+ * then we cancel the timeout timer.
+ *
+ * Returns > 0 if _begin has been called and all clients have finished.
+ * The caller will only see > 0 returned once.
+ */
+int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which)
+{
+	DECLARE_RECOV_INFO(sb, recinf);
+	struct recov_pending *pend;
+	int ret = 0;
+
+	spin_lock(&recinf->lock);
+
+	pend = find_pending(recinf, rid, which);
+	if (pend) {
+		pend->which &= ~which;
+		if (pend->which) {
+			pend = NULL;
+		} else {
+			list_del(&pend->head);
+			ret = recov_finished(recinf);
+		}
+	}
+
+	spin_unlock(&recinf->lock);
+
+	if (ret > 0)
+		del_timer_sync(&recinf->timer);
+
+	kfree(pend);
+
+	return ret;
+}
+
+/*
+ * Returns true if the given client is still trying to recover
+ * the given state.
+ */
+bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which)
+{
+	DECLARE_RECOV_INFO(sb, recinf);
+	bool is_pending;
+
+	spin_lock(&recinf->lock);
+	is_pending = find_pending(recinf, rid, which) != NULL;
+	spin_unlock(&recinf->lock);
+
+	return is_pending;
+}
+
+/*
+ * Returns 0 if there are no rids waiting for the given state to be
+ * recovered.  Returns the rid of a client still waiting if there are
+ * any, in no specified order.
+ *
+ * This is inherently racey.  Callers are responsible for resolving any
+ * actions taken based on pending with the recovery finishing, perhaps
+ * before we return.
+ */
+u64 scoutfs_recov_next_pending(struct super_block *sb, int which)
+{
+	DECLARE_RECOV_INFO(sb, recinf);
+	struct recov_pending *pend;
+	u64 rid;
+
+	spin_lock(&recinf->lock);
+	pend = find_pending(recinf, 0, which);
+	rid = pend ? pend->rid : 0;
+	spin_unlock(&recinf->lock);
+
+	return rid;
+}
+
+/*
+ * The server is shutting down and doesn't need to worry about recovery
+ * anymore.  It'll be built up again by the next server, if needed.
+ */
+void scoutfs_recov_shutdown(struct super_block *sb)
+{
+	DECLARE_RECOV_INFO(sb, recinf);
+	struct recov_pending *pend;
+	struct recov_pending *tmp;
+	LIST_HEAD(list);
+
+	del_timer_sync(&recinf->timer);
+
+	spin_lock(&recinf->lock);
+	list_splice_init(&recinf->pending, &list);
+	recinf->timeout_fn = NULL;
+	spin_unlock(&recinf->lock);
+
+	list_for_each_entry_safe(pend, tmp, &recinf->pending, head) {
+		list_del(&pend->head);
+		kfree(pend);
+	}
+}
+
+int scoutfs_recov_setup(struct super_block *sb)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	struct recov_info *recinf;
+	int ret;
+
+	recinf = kzalloc(sizeof(struct recov_info), GFP_KERNEL);
+	if (!recinf) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	recinf->sb = sb;
+	spin_lock_init(&recinf->lock);
+	INIT_LIST_HEAD(&recinf->pending);
+	timer_setup(&recinf->timer, timer_callback, 0);
+
+	sbi->recov_info = recinf;
+	ret = 0;
+out:
+	return ret;
+}
+
+void scoutfs_recov_destroy(struct super_block *sb)
+{
+	DECLARE_RECOV_INFO(sb, recinf);
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+
+	if (recinf) {
+		scoutfs_recov_shutdown(sb);
+
+		kfree(recinf);
+		sbi->recov_info = NULL;
+	}
+}

kmod/src/recov.h

@@ -0,0 +1,23 @@
+#ifndef _SCOUTFS_RECOV_H_
+#define _SCOUTFS_RECOV_H_
+
+enum {
+	SCOUTFS_RECOV_GREETING	= ( 1 <<  0),
+	SCOUTFS_RECOV_LOCKS	= ( 1 <<  1),
+
+	SCOUTFS_RECOV_INVALID	= (~0 <<  2),
+	SCOUTFS_RECOV_ALL	= (~SCOUTFS_RECOV_INVALID),
+};
+
+int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which);
+int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_block *),
+			unsigned int timeout_ms);
+int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which);
+bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which);
+u64 scoutfs_recov_next_pending(struct super_block *sb, int which);
+void scoutfs_recov_shutdown(struct super_block *sb);
+
+int scoutfs_recov_setup(struct super_block *sb);
+void scoutfs_recov_destroy(struct super_block *sb);
+
+#endif

kmod/src/scoutfs_trace.h

@@ -423,61 +423,34 @@ TRACE_EVENT(scoutfs_trans_write_func,
 	TP_printk(SCSBF" dirty %lu", SCSB_TRACE_ARGS, __entry->dirty)
 );
 
-TRACE_EVENT(scoutfs_release_trans,
-	TP_PROTO(struct super_block *sb, void *rsv, unsigned int rsv_holders,
-		 unsigned int tri_holders,
-		 unsigned int tri_writing),
+DECLARE_EVENT_CLASS(scoutfs_trans_hold_release_class,
+	TP_PROTO(struct super_block *sb, void *journal_info, int holders),
 
-	TP_ARGS(sb, rsv, rsv_holders, tri_holders, tri_writing),
+	TP_ARGS(sb, journal_info, holders),
 
 	TP_STRUCT__entry(
 		SCSB_TRACE_FIELDS
-		__field(void *, rsv)
-		__field(unsigned int, rsv_holders)
-		__field(unsigned int, tri_holders)
-		__field(unsigned int, tri_writing)
+		__field(unsigned long, journal_info)
+		__field(int, holders)
 	),
 
 	TP_fast_assign(
 		SCSB_TRACE_ASSIGN(sb);
-		__entry->rsv = rsv;
-		__entry->rsv_holders = rsv_holders;
-		__entry->tri_holders = tri_holders;
-		__entry->tri_writing = tri_writing;
+		__entry->journal_info = (unsigned long)journal_info;
+		__entry->holders = holders;
 	),
 
-	TP_printk(SCSBF" rsv %p holders %u trans holders %u writing %u",
-		  SCSB_TRACE_ARGS, __entry->rsv, __entry->rsv_holders,
-		  __entry->tri_holders, __entry->tri_writing)
+	TP_printk(SCSBF" journal_info 0x%0lx holders %d",
+		  SCSB_TRACE_ARGS, __entry->journal_info, __entry->holders)
 );
 
-TRACE_EVENT(scoutfs_trans_acquired_hold,
-	TP_PROTO(struct super_block *sb,
-		 void *rsv, unsigned int rsv_holders,
-		 unsigned int tri_holders,
-		 unsigned int tri_writing),
-
-	TP_ARGS(sb, rsv, rsv_holders, tri_holders, tri_writing),
-
-	TP_STRUCT__entry(
-		SCSB_TRACE_FIELDS
-		__field(void *, rsv)
-		__field(unsigned int, rsv_holders)
-		__field(unsigned int, tri_holders)
-		__field(unsigned int, tri_writing)
-	),
-
-	TP_fast_assign(
-		SCSB_TRACE_ASSIGN(sb);
-		__entry->rsv = rsv;
-		__entry->rsv_holders = rsv_holders;
-		__entry->tri_holders = tri_holders;
-		__entry->tri_writing = tri_writing;
-	),
-
-	TP_printk(SCSBF" rsv %p holders %u trans holders %u writing %u",
-		  SCSB_TRACE_ARGS, __entry->rsv, __entry->rsv_holders,
-		  __entry->tri_holders, __entry->tri_writing)
+DEFINE_EVENT(scoutfs_trans_hold_release_class, scoutfs_trans_acquired_hold,
+	TP_PROTO(struct super_block *sb, void *journal_info, int holders),
+	TP_ARGS(sb, journal_info, holders)
+);
+DEFINE_EVENT(scoutfs_trans_hold_release_class, scoutfs_release_trans,
+	TP_PROTO(struct super_block *sb, void *journal_info, int holders),
+	TP_ARGS(sb, journal_info, holders)
 );
 
 TRACE_EVENT(scoutfs_ioc_release,
@@ -717,15 +690,16 @@ TRACE_EVENT(scoutfs_evict_inode,
 
 TRACE_EVENT(scoutfs_drop_inode,
 	TP_PROTO(struct super_block *sb, __u64 ino, unsigned int nlink,
-		 unsigned int unhashed),
+		 unsigned int unhashed, bool drop_invalidated),
 
-	TP_ARGS(sb, ino, nlink, unhashed),
+	TP_ARGS(sb, ino, nlink, unhashed, drop_invalidated),
 
 	TP_STRUCT__entry(
 		SCSB_TRACE_FIELDS
 		__field(__u64, ino)
 		__field(unsigned int, nlink)
 		__field(unsigned int, unhashed)
+		__field(unsigned int, drop_invalidated)
 	),
 
 	TP_fast_assign(
@@ -733,10 +707,12 @@ TRACE_EVENT(scoutfs_drop_inode,
 		__entry->ino = ino;
 		__entry->nlink = nlink;
 		__entry->unhashed = unhashed;
+		__entry->drop_invalidated = !!drop_invalidated;
 	),
 
-	TP_printk(SCSBF" ino %llu nlink %u unhashed %d", SCSB_TRACE_ARGS,
-		  __entry->ino, __entry->nlink, __entry->unhashed)
+	TP_printk(SCSBF" ino %llu nlink %u unhashed %d drop_invalidated %u", SCSB_TRACE_ARGS,
+		  __entry->ino, __entry->nlink, __entry->unhashed,
+		  __entry->drop_invalidated)
 );
 
 TRACE_EVENT(scoutfs_inode_walk_writeback,
@@ -1611,7 +1587,7 @@ TRACE_EVENT(scoutfs_get_name,
 );
 
 TRACE_EVENT(scoutfs_btree_read_error,
-	TP_PROTO(struct super_block *sb, struct scoutfs_btree_ref *ref),
+	TP_PROTO(struct super_block *sb, struct scoutfs_block_ref *ref),
 
 	TP_ARGS(sb, ref),
 
@@ -1631,37 +1607,10 @@ TRACE_EVENT(scoutfs_btree_read_error,
 		  SCSB_TRACE_ARGS, __entry->blkno, __entry->seq)
 );
 
-TRACE_EVENT(scoutfs_btree_dirty_block,
-	TP_PROTO(struct super_block *sb, u64 blkno, u64 seq,
-		 u64 bt_blkno, u64 bt_seq),
-
-	TP_ARGS(sb, blkno, seq, bt_blkno, bt_seq),
-
-	TP_STRUCT__entry(
-		SCSB_TRACE_FIELDS
-		__field(__u64, blkno)
-		__field(__u64, seq)
-		__field(__u64, bt_blkno)
-		__field(__u64, bt_seq)
-	),
-
-	TP_fast_assign(
-		SCSB_TRACE_ASSIGN(sb);
-		__entry->blkno = blkno;
-		__entry->seq = seq;
-		__entry->bt_blkno = bt_blkno;
-		__entry->bt_seq = bt_seq;
-	),
-
-	TP_printk(SCSBF" blkno %llu seq %llu bt_blkno %llu bt_seq %llu",
-		  SCSB_TRACE_ARGS, __entry->blkno, __entry->seq,
-		  __entry->bt_blkno, __entry->bt_seq)
-);
-
 TRACE_EVENT(scoutfs_btree_walk,
 	TP_PROTO(struct super_block *sb, struct scoutfs_btree_root *root,
 		 struct scoutfs_key *key, int flags, int level,
-		 struct scoutfs_btree_ref *ref),
+		 struct scoutfs_block_ref *ref),
 
 	TP_ARGS(sb, root, key, flags, level, ref),
 
@@ -1797,118 +1746,69 @@ TRACE_EVENT(scoutfs_lock_message,
 		  __entry->old_mode, __entry->new_mode)
 );
 
+DECLARE_EVENT_CLASS(scoutfs_quorum_message_class,
+	TP_PROTO(struct super_block *sb, u64 term, u8 type, int nr),
 
-TRACE_EVENT(scoutfs_quorum_election,
-	TP_PROTO(struct super_block *sb, u64 prev_term),
-
-	TP_ARGS(sb, prev_term),
+	TP_ARGS(sb, term, type, nr),
 
 	TP_STRUCT__entry(
 		SCSB_TRACE_FIELDS
-		__field(__u64, prev_term)
-	),
-
-	TP_fast_assign(
-		SCSB_TRACE_ASSIGN(sb);
-		__entry->prev_term = prev_term;
-	),
-
-	TP_printk(SCSBF" prev_term %llu",
-		  SCSB_TRACE_ARGS, __entry->prev_term)
-);
-
-TRACE_EVENT(scoutfs_quorum_election_ret,
-	TP_PROTO(struct super_block *sb, int ret, u64 elected_term),
-
-	TP_ARGS(sb, ret, elected_term),
-
-	TP_STRUCT__entry(
-		SCSB_TRACE_FIELDS
-		__field(int, ret)
-		__field(__u64, elected_term)
-	),
-
-	TP_fast_assign(
-		SCSB_TRACE_ASSIGN(sb);
-		__entry->ret = ret;
-		__entry->elected_term = elected_term;
-	),
-
-	TP_printk(SCSBF" ret %d elected_term %llu",
-		  SCSB_TRACE_ARGS, __entry->ret, __entry->elected_term)
-);
-
-TRACE_EVENT(scoutfs_quorum_election_vote,
-	TP_PROTO(struct super_block *sb, int role, u64 term, u64 vote_for_rid,
-		 int votes, int log_cycles, int quorum_count),
-
-	TP_ARGS(sb, role, term, vote_for_rid, votes, log_cycles, quorum_count),
-
-	TP_STRUCT__entry(
-		SCSB_TRACE_FIELDS
-		__field(int, role)
 		__field(__u64, term)
-		__field(__u64, vote_for_rid)
-		__field(int, votes)
-		__field(int, log_cycles)
-		__field(int, quorum_count)
+		__field(__u8, type)
+		__field(int, nr)
 	),
 
 	TP_fast_assign(
 		SCSB_TRACE_ASSIGN(sb);
-		__entry->role = role;
 		__entry->term = term;
-		__entry->vote_for_rid = vote_for_rid;
-		__entry->votes = votes;
-		__entry->log_cycles = log_cycles;
-		__entry->quorum_count = quorum_count;
+		__entry->type = type;
+		__entry->nr = nr;
 	),
 
-	TP_printk(SCSBF" role %d term %llu vote_for_rid %016llx votes %d log_cycles %d quorum_count %d",
-		  SCSB_TRACE_ARGS, __entry->role, __entry->term,
-		  __entry->vote_for_rid, __entry->votes, __entry->log_cycles,
-		  __entry->quorum_count)
+	TP_printk(SCSBF" term %llu type %u nr %d",
+		  SCSB_TRACE_ARGS, __entry->term, __entry->type, __entry->nr)
+);
+DEFINE_EVENT(scoutfs_quorum_message_class, scoutfs_quorum_send_message,
+	TP_PROTO(struct super_block *sb, u64 term, u8 type, int nr),
+	TP_ARGS(sb, term, type, nr)
+);
+DEFINE_EVENT(scoutfs_quorum_message_class, scoutfs_quorum_recv_message,
+	TP_PROTO(struct super_block *sb, u64 term, u8 type, int nr),
+	TP_ARGS(sb, term, type, nr)
 );
 
-DECLARE_EVENT_CLASS(scoutfs_quorum_block_class,
-	TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk),
+TRACE_EVENT(scoutfs_quorum_loop,
+	TP_PROTO(struct super_block *sb, int role, u64 term, int vote_for,
+		 unsigned long vote_bits, struct timespec64 timeout),
 
-	TP_ARGS(sb, blk),
+	TP_ARGS(sb, role, term, vote_for, vote_bits, timeout),
 
 	TP_STRUCT__entry(
 		SCSB_TRACE_FIELDS
-		__field(__u64, blkno)
 		__field(__u64, term)
-		__field(__u64, write_nr)
-		__field(__u64, voter_rid)
-		__field(__u64, vote_for_rid)
-		__field(__u32, crc)
-		__field(__u8, log_nr)
+		__field(int, role)
+		__field(int, vote_for)
+		__field(unsigned long, vote_bits)
+		__field(unsigned long, vote_count)
+		__field(unsigned long long, timeout_sec)
+		__field(int, timeout_nsec)
 	),
 
 	TP_fast_assign(
 		SCSB_TRACE_ASSIGN(sb);
-		__entry->blkno = le64_to_cpu(blk->blkno);
-		__entry->term = le64_to_cpu(blk->term);
-		__entry->write_nr = le64_to_cpu(blk->write_nr);
-		__entry->voter_rid = le64_to_cpu(blk->voter_rid);
-		__entry->vote_for_rid = le64_to_cpu(blk->vote_for_rid);
-		__entry->crc = le32_to_cpu(blk->crc);
-		__entry->log_nr = blk->log_nr;
+		__entry->term = term;
+		__entry->role = role;
+		__entry->vote_for = vote_for;
+		__entry->vote_bits = vote_bits;
+		__entry->vote_count = hweight_long(vote_bits);
+		__entry->timeout_sec = timeout.tv_sec;
+		__entry->timeout_nsec = timeout.tv_nsec;
 	),
 
-	TP_printk(SCSBF" blkno %llu term %llu write_nr %llu voter_rid %016llx vote_for_rid %016llx crc 0x%08x log_nr %u",
-		  SCSB_TRACE_ARGS, __entry->blkno, __entry->term,
-		  __entry->write_nr, __entry->voter_rid, __entry->vote_for_rid,
-		  __entry->crc, __entry->log_nr)
-);
-DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_read_block,
-	TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk),
-	TP_ARGS(sb, blk)
-);
-DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_write_block,
-	TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk),
-	TP_ARGS(sb, blk)
+	TP_printk(SCSBF" term %llu role %d vote_for %d vote_bits 0x%lx vote_count %lu timeout %llu.%u",
+		  SCSB_TRACE_ARGS, __entry->term, __entry->role,
+		  __entry->vote_for, __entry->vote_bits, __entry->vote_count,
+		  __entry->timeout_sec, __entry->timeout_nsec)
 );
 
 /*
@@ -2038,8 +1938,8 @@ DEFINE_EVENT(scoutfs_forest_bloom_class, scoutfs_forest_bloom_search,
 );
 
 TRACE_EVENT(scoutfs_forest_prepare_commit,
-	TP_PROTO(struct super_block *sb, struct scoutfs_btree_ref *item_ref,
-		 struct scoutfs_btree_ref *bloom_ref),
+	TP_PROTO(struct super_block *sb, struct scoutfs_block_ref *item_ref,
+		 struct scoutfs_block_ref *bloom_ref),
 	TP_ARGS(sb, item_ref, bloom_ref),
 	TP_STRUCT__entry(
 		SCSB_TRACE_FIELDS
@@ -2105,18 +2005,45 @@ TRACE_EVENT(scoutfs_forest_init_our_log,
 		  __entry->blkno, __entry->seq)
 );
 
+TRACE_EVENT(scoutfs_block_dirty_ref,
+	TP_PROTO(struct super_block *sb, u64 ref_blkno, u64 ref_seq,
+		 u64 block_blkno, u64 block_seq),
+
+	TP_ARGS(sb, ref_blkno, ref_seq, block_blkno, block_seq),
+
+	TP_STRUCT__entry(
+		SCSB_TRACE_FIELDS
+		__field(__u64, ref_blkno)
+		__field(__u64, ref_seq)
+		__field(__u64, block_blkno)
+		__field(__u64, block_seq)
+	),
+
+	TP_fast_assign(
+		SCSB_TRACE_ASSIGN(sb);
+		__entry->ref_blkno = ref_blkno;
+		__entry->ref_seq = ref_seq;
+		__entry->block_blkno = block_blkno;
+		__entry->block_seq = block_seq;
+	),
+
+	TP_printk(SCSBF" ref_blkno %llu ref_seq %llu block_blkno %llu block_seq %llu",
+		  SCSB_TRACE_ARGS, __entry->ref_blkno, __entry->ref_seq,
+		  __entry->block_blkno, __entry->block_seq)
+);
+
 DECLARE_EVENT_CLASS(scoutfs_block_class,
-	TP_PROTO(struct super_block *sb, void *bp, u64 blkno,
-		 int refcount, int io_count, unsigned long bits, u64 lru_moved),
-	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved),
+	TP_PROTO(struct super_block *sb, void *bp, u64 blkno, int refcount, int io_count,
+		 unsigned long bits, __u64 accessed),
+	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, accessed),
 	TP_STRUCT__entry(
 		SCSB_TRACE_FIELDS
 		__field(void *, bp)
 		__field(__u64, blkno)
 		__field(int, refcount)
 		__field(int, io_count)
-		__field(unsigned long, bits)
-		__field(__u64, lru_moved)
+		__field(long, bits)
+		__field(__u64, accessed)
 	),
 	TP_fast_assign(
 		SCSB_TRACE_ASSIGN(sb);
@@ -2125,57 +2052,71 @@ DECLARE_EVENT_CLASS(scoutfs_block_class,
 		__entry->refcount = refcount;
 		__entry->io_count = io_count;
 		__entry->bits = bits;
-		__entry->lru_moved = lru_moved;
+		__entry->accessed = accessed;
 	),
-	TP_printk(SCSBF" bp %p blkno %llu refcount %d io_count %d bits 0x%lx lru_moved %llu",
-		  SCSB_TRACE_ARGS, __entry->bp, __entry->blkno,
-		  __entry->refcount, __entry->io_count, __entry->bits,
-		  __entry->lru_moved)
+	TP_printk(SCSBF" bp %p blkno %llu refcount %d io_count %d bits 0x%lx accessed %llu",
+		  SCSB_TRACE_ARGS, __entry->bp, __entry->blkno, __entry->refcount,
+		  __entry->io_count, __entry->bits, __entry->accessed)
 );
 DEFINE_EVENT(scoutfs_block_class, scoutfs_block_allocate,
 	TP_PROTO(struct super_block *sb, void *bp, u64 blkno,
-		 int refcount, int io_count, unsigned long bits, u64 lru_moved),
-	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved)
+		 int refcount, int io_count, unsigned long bits,
+		 __u64 accessed),
+	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, accessed)
 );
 DEFINE_EVENT(scoutfs_block_class, scoutfs_block_free,
 	TP_PROTO(struct super_block *sb, void *bp, u64 blkno,
-		 int refcount, int io_count, unsigned long bits, u64 lru_moved),
-	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved)
+		 int refcount, int io_count, unsigned long bits,
+		 __u64 accessed),
+	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, accessed)
 );
 DEFINE_EVENT(scoutfs_block_class, scoutfs_block_insert,
 	TP_PROTO(struct super_block *sb, void *bp, u64 blkno,
-		 int refcount, int io_count, unsigned long bits, u64 lru_moved),
-	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved)
+		 int refcount, int io_count, unsigned long bits,
+		 __u64 accessed),
+	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, accessed)
+);
+DEFINE_EVENT(scoutfs_block_class, scoutfs_block_remove,
+	TP_PROTO(struct super_block *sb, void *bp, u64 blkno,
+		 int refcount, int io_count, unsigned long bits,
+		 __u64 accessed),
+	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, accessed)
 );
 DEFINE_EVENT(scoutfs_block_class, scoutfs_block_end_io,
 	TP_PROTO(struct super_block *sb, void *bp, u64 blkno,
-		 int refcount, int io_count, unsigned long bits, u64 lru_moved),
-	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved)
+		 int refcount, int io_count, unsigned long bits,
+		 __u64 accessed),
+	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, accessed)
 );
 DEFINE_EVENT(scoutfs_block_class, scoutfs_block_submit,
 	TP_PROTO(struct super_block *sb, void *bp, u64 blkno,
-		 int refcount, int io_count, unsigned long bits, u64 lru_moved),
-	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved)
+		 int refcount, int io_count, unsigned long bits,
+		 __u64 accessed),
+	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, accessed)
 );
 DEFINE_EVENT(scoutfs_block_class, scoutfs_block_invalidate,
 	TP_PROTO(struct super_block *sb, void *bp, u64 blkno,
-		 int refcount, int io_count, unsigned long bits, u64 lru_moved),
-	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved)
+		 int refcount, int io_count, unsigned long bits,
+		 __u64 accessed),
+	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, accessed)
 );
 DEFINE_EVENT(scoutfs_block_class, scoutfs_block_mark_dirty,
 	TP_PROTO(struct super_block *sb, void *bp, u64 blkno,
-		 int refcount, int io_count, unsigned long bits, u64 lru_moved),
-	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved)
+		 int refcount, int io_count, unsigned long bits,
+		 __u64 accessed),
+	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, accessed)
 );
 DEFINE_EVENT(scoutfs_block_class, scoutfs_block_forget,
 	TP_PROTO(struct super_block *sb, void *bp, u64 blkno,
-		 int refcount, int io_count, unsigned long bits, u64 lru_moved),
-	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved)
+		 int refcount, int io_count, unsigned long bits,
+		 __u64 accessed),
+	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, accessed)
 );
 DEFINE_EVENT(scoutfs_block_class, scoutfs_block_shrink,
 	TP_PROTO(struct super_block *sb, void *bp, u64 blkno,
-		 int refcount, int io_count, unsigned long bits, u64 lru_moved),
-	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved)
+		 int refcount, int io_count, unsigned long bits,
+		 __u64 accessed),
+	TP_ARGS(sb, bp, blkno, refcount, io_count, bits, accessed)
 );
 
 DECLARE_EVENT_CLASS(scoutfs_ext_next_class,
@@ -2464,6 +2405,89 @@ TRACE_EVENT(scoutfs_item_invalidate_page,
 		  sk_trace_args(pg_start), sk_trace_args(pg_end), __entry->pgi)
 );
 
+DECLARE_EVENT_CLASS(scoutfs_omap_group_class,
+	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
+		 int bit_nr, int bit_count),
+
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count),
+
+	TP_STRUCT__entry(
+		SCSB_TRACE_FIELDS
+		__field(void *, grp)
+		__field(__u64, group_nr)
+		__field(unsigned int, group_total)
+		__field(int, bit_nr)
+		__field(int, bit_count)
+	),
+
+	TP_fast_assign(
+		SCSB_TRACE_ASSIGN(sb);
+		__entry->grp = grp;
+		__entry->group_nr = group_nr;
+		__entry->group_total = group_total;
+		__entry->bit_nr = bit_nr;
+		__entry->bit_count = bit_count;
+	),
+
+	TP_printk(SCSBF" grp %p group_nr %llu group_total %u bit_nr %d bit_count %d",
+		  SCSB_TRACE_ARGS, __entry->grp, __entry->group_nr, __entry->group_total,
+		  __entry->bit_nr, __entry->bit_count)
+);
+
+DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_alloc,
+	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
+		 int bit_nr, int bit_count),
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count)
+);
+DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_free,
+	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
+		 int bit_nr, int bit_count),
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count)
+);
+DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_inc,
+	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
+		 int bit_nr, int bit_count),
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count)
+);
+DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_dec,
+	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
+		 int bit_nr, int bit_count),
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count)
+);
+DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_request,
+	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
+		 int bit_nr, int bit_count),
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count)
+);
+DEFINE_EVENT(scoutfs_omap_group_class, scoutfs_omap_group_destroy,
+	TP_PROTO(struct super_block *sb, void *grp, u64 group_nr, unsigned int group_total,
+		 int bit_nr, int bit_count),
+	TP_ARGS(sb, grp, group_nr, group_total, bit_nr, bit_count)
+);
+
+TRACE_EVENT(scoutfs_omap_should_delete,
+	TP_PROTO(struct super_block *sb, u64 ino, unsigned int nlink, int ret),
+
+	TP_ARGS(sb, ino, nlink, ret),
+
+	TP_STRUCT__entry(
+		SCSB_TRACE_FIELDS
+		__field(__u64, ino)
+		__field(unsigned int, nlink)
+		__field(int, ret)
+	),
+
+	TP_fast_assign(
+		SCSB_TRACE_ASSIGN(sb);
+		__entry->ino = ino;
+		__entry->nlink = nlink;
+		__entry->ret = ret;
+	),
+
+	TP_printk(SCSBF" ino %llu nlink %u ret %d",
+		  SCSB_TRACE_ARGS, __entry->ino, __entry->nlink, __entry->ret)
+);
+
 #endif /* _TRACE_SCOUTFS_H */
 
 /* This part must be outside protection */

kmod/src/server.c

@@ -38,6 +38,8 @@
 #include "srch.h"
 #include "alloc.h"
 #include "forest.h"
+#include "recov.h"
+#include "omap.h"
 
 /*
  * Every active mount can act as the server that listens on a net
@@ -59,7 +61,6 @@ struct server_info {
 	int err;
 	bool shutting_down;
 	struct completion start_comp;
-	struct sockaddr_in listen_sin;
 	u64 term;
 	struct scoutfs_net_connection *conn;
 
@@ -75,7 +76,7 @@ struct server_info {
 	unsigned long nr_clients;
 
 	/* track clients waiting in unmmount for farewell response */
-	struct mutex farewell_mutex;
+	spinlock_t farewell_lock;
 	struct list_head farewell_requests;
 	struct work_struct farewell_work;
 
@@ -92,10 +93,14 @@ struct server_info {
 
 	struct mutex logs_mutex;
 	struct mutex srch_mutex;
+	struct mutex mounted_clients_mutex;
 
 	/* stable versions stored from commits, given in locks and rpcs */
 	seqcount_t roots_seqcount;
 	struct scoutfs_net_roots roots;
+
+	/* recovery timeout fences from work */
+	struct work_struct fence_pending_recov_work;
 };
 
 #define DECLARE_SERVER_INFO(sb, name) \
@@ -182,7 +187,7 @@ int scoutfs_server_apply_commit(struct super_block *sb, int err)
 	return err;
 }
 
-void scoutfs_server_get_roots(struct super_block *sb,
+static void get_roots(struct super_block *sb,
 			      struct scoutfs_net_roots *roots)
 {
 	DECLARE_SERVER_INFO(sb, server);
@@ -556,7 +561,7 @@ static int server_get_roots(struct super_block *sb,
 		memset(&roots, 0, sizeof(roots));
 		ret = -EINVAL;
 	}  else {
-		scoutfs_server_get_roots(sb, &roots);
+		get_roots(sb, &roots);
 		ret = 0;
 	}
 
@@ -862,13 +867,13 @@ int scoutfs_server_lock_request(struct super_block *sb, u64 rid,
 }
 
 int scoutfs_server_lock_response(struct super_block *sb, u64 rid, u64 id,
-				 struct scoutfs_net_lock_grant_response *gr)
+				 struct scoutfs_net_lock *nl)
 {
 	struct server_info *server = SCOUTFS_SB(sb)->server_info;
 
 	return scoutfs_net_response_node(sb, server->conn, rid,
 					 SCOUTFS_NET_CMD_LOCK, id, 0,
-					 gr, sizeof(*gr));
+					 nl, sizeof(*nl));
 }
 
 static bool invalid_recover(struct scoutfs_net_lock_recover *nlr,
@@ -1016,6 +1021,60 @@ out:
 	return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0);
 }
 
+/* The server is receiving an omap response from the client */
+static int open_ino_map_response(struct super_block *sb, struct scoutfs_net_connection *conn,
+				 void *resp, unsigned int resp_len, int error, void *data)
+{
+	u64 rid = scoutfs_net_client_rid(conn);
+
+	if (resp_len != sizeof(struct scoutfs_open_ino_map))
+		return -EINVAL;
+
+	return scoutfs_omap_server_handle_response(sb, rid, resp);
+}
+
+/* The server is sending an omap request to the client */
+int scoutfs_server_send_omap_request(struct super_block *sb, u64 rid,
+				     struct scoutfs_open_ino_map_args *args)
+{
+	struct server_info *server = SCOUTFS_SB(sb)->server_info;
+
+	return scoutfs_net_submit_request_node(sb, server->conn, rid, SCOUTFS_NET_CMD_OPEN_INO_MAP,
+					      args, sizeof(*args),
+					      open_ino_map_response, NULL, NULL);
+}
+
+/* The server is sending an omap response to the client */
+int scoutfs_server_send_omap_response(struct super_block *sb, u64 rid, u64 id,
+				      struct scoutfs_open_ino_map *map, int err)
+{
+	struct server_info *server = SCOUTFS_SB(sb)->server_info;
+
+	return scoutfs_net_response_node(sb, server->conn, rid,
+					 SCOUTFS_NET_CMD_OPEN_INO_MAP, id, err,
+					 map, sizeof(*map));
+}
+
+/* The server is receiving an omap request from the client */
+static int server_open_ino_map(struct super_block *sb, struct scoutfs_net_connection *conn,
+			       u8 cmd, u64 id, void *arg, u16 arg_len)
+{
+	u64 rid = scoutfs_net_client_rid(conn);
+	int ret;
+
+	if (arg_len != sizeof(struct scoutfs_open_ino_map_args)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	ret = scoutfs_omap_server_handle_request(sb, rid, id, arg);
+out:
+	if (ret < 0)
+		return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0);
+
+	return 0;
+}
+
 static void init_mounted_client_key(struct scoutfs_key *key, u64 rid)
 {
 	*key = (struct scoutfs_key) {
@@ -1024,6 +1083,12 @@ static void init_mounted_client_key(struct scoutfs_key *key, u64 rid)
 	};
 }
 
+/*
+ * Insert a new mounted client item for a client that is sending us a
+ * greeting that hasn't yet seen a response.  The greeting can be
+ * retransmitted to a new server after the previous inserted the item so
+ * it's acceptable to see -EEXIST.
+ */
 static int insert_mounted_client(struct super_block *sb, u64 rid,
 				 u64 gr_flags)
 {
@@ -1031,15 +1096,22 @@ static int insert_mounted_client(struct super_block *sb, u64 rid,
 	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
 	struct scoutfs_mounted_client_btree_val mcv;
 	struct scoutfs_key key;
+	int ret;
 
 	init_mounted_client_key(&key, rid);
 	mcv.flags = 0;
-	if (gr_flags & SCOUTFS_NET_GREETING_FLAG_VOTER)
-		mcv.flags |= SCOUTFS_MOUNTED_CLIENT_VOTER;
+	if (gr_flags & SCOUTFS_NET_GREETING_FLAG_QUORUM)
+		mcv.flags |= SCOUTFS_MOUNTED_CLIENT_QUORUM;
 
-	return scoutfs_btree_insert(sb, &server->alloc, &server->wri,
-				    &super->mounted_clients, &key, &mcv,
-				    sizeof(mcv));
+	mutex_lock(&server->mounted_clients_mutex);
+	ret = scoutfs_btree_insert(sb, &server->alloc, &server->wri,
+				   &super->mounted_clients, &key, &mcv,
+				   sizeof(mcv));
+	if (ret == -EEXIST)
+		ret = 0;
+	mutex_unlock(&server->mounted_clients_mutex);
+
+	return ret;
 }
 
 /*
@@ -1047,9 +1119,6 @@ static int insert_mounted_client(struct super_block *sb, u64 rid,
  * removed if we're processing a farewell on behalf of a client that
  * already had a previous server process its farewell.
  *
- * When we remove the last mounted client that's voting we write a new
- * quorum block with the updated unmount_barrier.
- *
  * The caller has to serialize with farewell processing.
  */
 static int delete_mounted_client(struct super_block *sb, u64 rid)
@@ -1061,8 +1130,10 @@ static int delete_mounted_client(struct super_block *sb, u64 rid)
 
 	init_mounted_client_key(&key, rid);
 
+	mutex_lock(&server->mounted_clients_mutex);
 	ret = scoutfs_btree_delete(sb, &server->alloc, &server->wri,
 				   &super->mounted_clients, &key);
+	mutex_unlock(&server->mounted_clients_mutex);
 	if (ret == -ENOENT)
 		ret = 0;
 
@@ -1149,7 +1220,6 @@ static int server_greeting(struct super_block *sb,
 	struct scoutfs_net_greeting *gr = arg;
 	struct scoutfs_net_greeting greet;
 	DECLARE_SERVER_INFO(sb, server);
-	__le64 umb = 0;
 	bool reconnecting;
 	bool first_contact;
 	bool farewell;
@@ -1182,28 +1252,24 @@ static int server_greeting(struct super_block *sb,
 		if (ret < 0)
 			goto send_err;
 
-		spin_lock(&server->lock);
-		umb = super->unmount_barrier;
-		spin_unlock(&server->lock);
-
-		mutex_lock(&server->farewell_mutex);
 		ret = insert_mounted_client(sb, le64_to_cpu(gr->rid),
 					    le64_to_cpu(gr->flags));
-		mutex_unlock(&server->farewell_mutex);
 
 		ret = scoutfs_server_apply_commit(sb, ret);
 		queue_work(server->wq, &server->farewell_work);
-	} else {
-		umb = gr->unmount_barrier;
+		if (ret < 0)
+			goto send_err;
 	}
 
+	scoutfs_server_recov_finish(sb, le64_to_cpu(gr->rid), SCOUTFS_RECOV_GREETING);
+	ret = 0;
+
 send_err:
 	err = ret;
 
 	greet.fsid = super->hdr.fsid;
 	greet.version = super->version;
 	greet.server_term = cpu_to_le64(server->term);
-	greet.unmount_barrier = umb;
 	greet.rid = gr->rid;
 	greet.flags = 0;
 
@@ -1226,17 +1292,10 @@ send_err:
 	scoutfs_net_server_greeting(sb, conn, le64_to_cpu(gr->rid), id,
 				    reconnecting, first_contact, farewell);
 
-	/* lock server might send recovery request */
+	/* let layers know we have a client connecting for the first time */
 	if (le64_to_cpu(gr->server_term) != server->term) {
-
-		/* we're now doing two commits per greeting, not great */
-		ret = scoutfs_server_hold_commit(sb);
-		if (ret)
-			goto out;
-
-		ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid),
-						   gr->server_term != 0);
-		ret = scoutfs_server_apply_commit(sb, ret);
+		ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid)) ?:
+		      scoutfs_omap_add_rid(sb, le64_to_cpu(gr->rid));
 		if (ret)
 			goto out;
 	}
@@ -1257,21 +1316,38 @@ static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref)
 			sizeof(struct scoutfs_mounted_client_btree_val));
 }
 
+static int reclaim_rid(struct super_block *sb, u64 rid)
+{
+	int ret;
+
+	ret = scoutfs_server_hold_commit(sb);
+	if (ret < 0)
+		return ret;
+
+	/* delete mounted client last, client reconnect looks for it */
+	ret = scoutfs_lock_server_farewell(sb, rid) ?:
+	      remove_trans_seq(sb, rid) ?:
+	      reclaim_log_trees(sb, rid) ?:
+	      cancel_srch_compact(sb, rid) ?:
+	      delete_mounted_client(sb, rid) ?:
+	      scoutfs_omap_remove_rid(sb, rid);
+
+	return scoutfs_server_apply_commit(sb, ret);
+}
+
 /*
  * This work processes farewell requests asynchronously.  Requests from
- * voting clients can be held until only the final quorum remains and
+ * quorum members can be held until only the final majority remains and
  * they've all sent farewell requests.
  *
- * When we remove the last mounted client record for the last voting
- * client then we increase the unmount_barrier and write it to the super
- * block.  If voting clients don't get their farewell response they'll
- * see the greater umount_barrier in the super and will know that their
- * farewell has been processed and that they can exit.
+ * A client can be disconnected before receiving our farewell response.
+ * Before reconnecting they check for their mounted client item, if it's
+ * been removed then they know that their farewell has been processed
+ * and that they finish unmounting without reconnecting.
  *
- * Responses that are waiting for clients who aren't voting are
- * immediately sent.  Clients that don't have a mounted client record
- * have already had their farewell processed by another server and can
- * proceed.
+ * Responses for clients who aren't quorum members are immediately sent.
+ * Clients that don't have a mounted client record have already had
+ * their farewell processed by another server and can proceed.
  *
  * Farewell responses are unique in that sending them causes the server
  * to shutdown the connection to the client next time the socket
@@ -1293,56 +1369,26 @@ static void farewell_worker(struct work_struct *work)
 	struct farewell_request *tmp;
 	struct farewell_request *fw;
 	SCOUTFS_BTREE_ITEM_REF(iref);
-	unsigned int nr_unmounting = 0;
-	unsigned int nr_mounted = 0;
+	unsigned int quo_reqs = 0;
+	unsigned int quo_mnts = 0;
+	unsigned int non_mnts = 0;
 	struct scoutfs_key key;
 	LIST_HEAD(reqs);
 	LIST_HEAD(send);
-	bool deleted = false;
-	bool voting;
 	bool more_reqs;
 	int ret;
 
-	/* grab all the requests that are waiting */
-	mutex_lock(&server->farewell_mutex);
+	spin_lock(&server->farewell_lock);
 	list_splice_init(&server->farewell_requests, &reqs);
-	mutex_unlock(&server->farewell_mutex);
+	spin_unlock(&server->farewell_lock);
 
-	/* count how many reqs requests are from voting clients */
-	nr_unmounting = 0;
-	list_for_each_entry_safe(fw, tmp, &reqs, entry) {
-		init_mounted_client_key(&key, fw->rid);
-		ret = scoutfs_btree_lookup(sb, &super->mounted_clients, &key,
-					   &iref);
-		if (ret == 0 && invalid_mounted_client_item(&iref)) {
-			scoutfs_btree_put_iref(&iref);
-			ret = -EIO;
-		}
-		if (ret < 0) {
-			if (ret == -ENOENT) {
-				list_move_tail(&fw->entry, &send);
-				continue;
-			}
-			goto out;
-		}
-
-		mcv = iref.val;
-		voting = (mcv->flags & SCOUTFS_MOUNTED_CLIENT_VOTER) != 0;
-		scoutfs_btree_put_iref(&iref);
-
-		if (!voting) {
-			list_move_tail(&fw->entry, &send);
-			continue;
-		}
-
-		nr_unmounting++;
-	}
-
-	/* see how many mounted clients could vote for quorum */
+	/* first count mounted clients who could send requests */
 	init_mounted_client_key(&key, 0);
 	for (;;) {
+		mutex_lock(&server->mounted_clients_mutex);
 		ret = scoutfs_btree_next(sb, &super->mounted_clients, &key,
 					 &iref);
+		mutex_unlock(&server->mounted_clients_mutex);
 		if (ret == 0 && invalid_mounted_client_item(&iref)) {
 			scoutfs_btree_put_iref(&iref);
 			ret = -EIO;
@@ -1356,51 +1402,67 @@ static void farewell_worker(struct work_struct *work)
 		key = *iref.key;
 		mcv = iref.val;
 
-		if (mcv->flags & SCOUTFS_MOUNTED_CLIENT_VOTER)
-			nr_mounted++;
+		if (mcv->flags & SCOUTFS_MOUNTED_CLIENT_QUORUM)
+			quo_mnts++;
+		else
+			non_mnts++;
 
 		scoutfs_btree_put_iref(&iref);
 		scoutfs_key_inc(&key);
 	}
 
-	/* send as many responses as we can to maintain quorum */
-	while ((fw = list_first_entry_or_null(&reqs, struct farewell_request,
-					      entry)) &&
-	       (nr_mounted > super->quorum_count ||
-		nr_unmounting >= nr_mounted)) {
+	/* walk requests, checking their mounted client items */
+	list_for_each_entry_safe(fw, tmp, &reqs, entry) {
+		init_mounted_client_key(&key, fw->rid);
+		mutex_lock(&server->mounted_clients_mutex);
+		ret = scoutfs_btree_lookup(sb, &super->mounted_clients, &key,
+					   &iref);
+		mutex_unlock(&server->mounted_clients_mutex);
+		if (ret == 0 && invalid_mounted_client_item(&iref)) {
+			scoutfs_btree_put_iref(&iref);
+			ret = -EIO;
+		}
+		if (ret < 0) {
+			/* missing items means we've already processed */
+			if (ret == -ENOENT) {
+				list_move(&fw->entry, &send);
+				continue;
+			}
+			goto out;
+		}
 
-		list_move_tail(&fw->entry, &send);
-		nr_mounted--;
-		nr_unmounting--;
-		deleted = true;
+		mcv = iref.val;
+
+		/* count quo reqs, can always send to non-quo clients */
+		if (mcv->flags & SCOUTFS_MOUNTED_CLIENT_QUORUM) {
+			quo_reqs++;
+		} else {
+			list_move(&fw->entry, &send);
+			non_mnts--;
+		}
+
+		scoutfs_btree_put_iref(&iref);
+	}
+
+	/*
+	 * Only requests from quorum members remain and we've counted
+	 * them and remaining mounts.  Send responses as long as enough
+	 * quorum clients remain for a majority, or all the requests are
+	 * from the final majority of quorum clients they're the only
+	 * mounted clients.
+	 */
+	list_for_each_entry_safe(fw, tmp, &reqs, entry) {
+		if ((quo_mnts > scoutfs_quorum_votes_needed(sb)) ||
+		    ((quo_reqs == quo_mnts) && (non_mnts == 0))) {
+			list_move_tail(&fw->entry, &send);
+			quo_mnts--;
+			quo_reqs--;
+		}
 	}
 
 	/* process and send farewell responses */
 	list_for_each_entry_safe(fw, tmp, &send, entry) {
-		ret = scoutfs_server_hold_commit(sb);
-		if (ret)
-			goto out;
-
-		ret = scoutfs_lock_server_farewell(sb, fw->rid) ?:
-		      remove_trans_seq(sb, fw->rid) ?:
-		      reclaim_log_trees(sb, fw->rid) ?:
-		      delete_mounted_client(sb, fw->rid) ?:
-		      cancel_srch_compact(sb, fw->rid);
-
-		ret = scoutfs_server_apply_commit(sb, ret);
-		if (ret)
-			goto out;
-	}
-
-	/* update the unmount barrier if we deleted all voting clients */
-	if (deleted && nr_mounted == 0) {
-		ret = scoutfs_server_hold_commit(sb);
-		if (ret)
-			goto out;
-
-		le64_add_cpu(&super->unmount_barrier, 1);
-
-		ret = scoutfs_server_apply_commit(sb, ret);
+		ret = reclaim_rid(sb, fw->rid);
 		if (ret)
 			goto out;
 	}
@@ -1420,11 +1482,11 @@ static void farewell_worker(struct work_struct *work)
 
 	ret = 0;
 out:
-	mutex_lock(&server->farewell_mutex);
+	spin_lock(&server->farewell_lock);
 	more_reqs = !list_empty(&server->farewell_requests);
 	list_splice_init(&reqs, &server->farewell_requests);
 	list_splice_init(&send, &server->farewell_requests);
-	mutex_unlock(&server->farewell_mutex);
+	spin_unlock(&server->farewell_lock);
 
 	if (ret < 0)
 		stop_server(server);
@@ -1437,15 +1499,17 @@ static void free_farewell_requests(struct super_block *sb, u64 rid)
 	struct server_info *server = SCOUTFS_SB(sb)->server_info;
 	struct farewell_request *tmp;
 	struct farewell_request *fw;
+	LIST_HEAD(rid_list);
 
-	mutex_lock(&server->farewell_mutex);
+	spin_lock(&server->farewell_lock);
 	list_for_each_entry_safe(fw, tmp, &server->farewell_requests, entry) {
-		if (rid == 0 || fw->rid == rid) {
-			list_del_init(&fw->entry);
-			kfree(fw);
-		}
+		if (rid == 0 || fw->rid == rid)
+			list_move_tail(&fw->entry, &rid_list);
 	}
-	mutex_unlock(&server->farewell_mutex);
+	spin_unlock(&server->farewell_lock);
+
+	list_for_each_entry_safe(fw, tmp, &rid_list, entry)
+		kfree(fw);
 }
 
 /*
@@ -1479,9 +1543,9 @@ static int server_farewell(struct super_block *sb,
 	fw->rid = rid;
 	fw->net_id = id;
 
-	mutex_lock(&server->farewell_mutex);
+	spin_lock(&server->farewell_lock);
 	list_add_tail(&fw->entry, &server->farewell_requests);
-	mutex_unlock(&server->farewell_mutex);
+	spin_unlock(&server->farewell_lock);
 
 	queue_farewell_work(server);
 
@@ -1500,6 +1564,7 @@ static scoutfs_net_request_t server_req_funcs[] = {
 	[SCOUTFS_NET_CMD_LOCK]			= server_lock,
 	[SCOUTFS_NET_CMD_SRCH_GET_COMPACT]	= server_srch_get_compact,
 	[SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT]	= server_srch_commit_compact,
+	[SCOUTFS_NET_CMD_OPEN_INO_MAP]		= server_open_ino_map,
 	[SCOUTFS_NET_CMD_FAREWELL]		= server_farewell,
 };
 
@@ -1541,6 +1606,143 @@ static void server_notify_down(struct super_block *sb,
 	}
 }
 
+/*
+ * All clients have recovered all state.  Now we can kick all the work
+ * that was waiting on recovery.
+ *
+ * It's a bit of a false dependency to have all work wait for completion
+ * before any work can make progress, but recovery is naturally
+ * concerned about in-memory state.  It should all be quick to recover
+ * once a client arrives.
+ */
+static void finished_recovery(struct super_block *sb)
+{
+	DECLARE_SERVER_INFO(sb, server);
+	int ret = 0;
+
+	scoutfs_info(sb, "all clients recovered");
+
+	ret = scoutfs_omap_finished_recovery(sb) ?:
+	      scoutfs_lock_server_finished_recovery(sb);
+	if (ret < 0) {
+		scoutfs_err(sb, "error %d resuming after recovery finished, shutting down", ret);
+		stop_server(server);
+	}
+}
+
+void scoutfs_server_recov_finish(struct super_block *sb, u64 rid, int which)
+{
+	if (scoutfs_recov_finish(sb, rid, which) > 0)
+		finished_recovery(sb);
+}
+
+/*
+ * If the recovery timeout is too short we'll prematurely evict mounts
+ * that would have recovered.  They need time to have their sockets
+ * timeout, reconnect to the current server, and fully recover their
+ * state.
+ *
+ * If it's too long we'll needlessly delay resuming operations after
+ * clients crash and will never recover.
+ */
+#define SERVER_RECOV_TIMEOUT_MS (30 * MSEC_PER_SEC)
+
+/*
+ * Not all clients recovered in time.  We fence them and reclaim
+ * whatever resources they were using.  If we see a rid here then we're
+ * going to fence it, regardless of if it manages to finish recovery
+ * while we're fencing it.
+ */
+static void fence_pending_recov_worker(struct work_struct *work)
+{
+	struct server_info *server = container_of(work, struct server_info,
+						  fence_pending_recov_work);
+	struct super_block *sb = server->sb;
+	u64 rid;
+	int ret;
+
+	while ((rid = scoutfs_recov_next_pending(sb, SCOUTFS_RECOV_ALL)) > 0) {
+		scoutfs_err(sb, "%lu ms recovery timeout expired for client rid %016llx, fencing",
+			    SERVER_RECOV_TIMEOUT_MS, rid);
+
+		ret = reclaim_rid(sb, rid);
+		if (ret < 0) {
+			scoutfs_err(sb, "error %d reclaiming rid %016llx, shutting down", ret, rid);
+			stop_server(server);
+			break;
+		}
+
+		scoutfs_server_recov_finish(sb, rid, SCOUTFS_RECOV_ALL);
+	}
+}
+
+static void recovery_timeout(struct super_block *sb)
+{
+	DECLARE_SERVER_INFO(sb, server);
+
+	if (!server->shutting_down)
+		queue_work(server->wq, &server->fence_pending_recov_work);
+}
+
+/*
+ * As the server starts up it needs to start waiting for recovery from
+ * any clients which were previously still mounted in the last running
+ * server.  This is done before networking is started so we won't
+ * receive any messages from clients until we've prepared them all.  If
+ * the clients don't recover in time then they'll be fenced.
+ */
+static int start_recovery(struct super_block *sb)
+{
+	DECLARE_SERVER_INFO(sb, server);
+	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
+	SCOUTFS_BTREE_ITEM_REF(iref);
+	struct scoutfs_key key;
+	unsigned int nr = 0;
+	u64 rid;
+	int ret;
+
+	for (rid = 0; ; rid++) {
+		init_mounted_client_key(&key, rid);
+		ret = scoutfs_btree_next(sb, &super->mounted_clients, &key, &iref);
+		if (ret == -ENOENT) {
+			ret = 0;
+			break;
+		}
+		if (ret == 0) {
+			rid = le64_to_cpu(iref.key->skmc_rid);
+			scoutfs_btree_put_iref(&iref);
+		}
+		if (ret < 0)
+			goto out;
+
+		ret = scoutfs_recov_prepare(sb, rid, SCOUTFS_RECOV_ALL);
+		if (ret < 0) {
+			scoutfs_err(sb, "error %d preparing recovery for client rid %016llx, shutting down",
+				     ret, rid);
+			goto out;
+		}
+
+		nr++;
+	}
+
+	if (nr > 0) {
+		scoutfs_info(sb, "waiting for %u clients to recover", nr);
+
+		ret = scoutfs_recov_begin(sb, recovery_timeout, SERVER_RECOV_TIMEOUT_MS);
+		if (ret > 0) {
+			finished_recovery(sb);
+			ret = 0;
+		}
+	}
+
+out:
+	if (ret < 0) {
+		scoutfs_err(sb, "error %d starting recovery, shutting down", ret);
+		stop_server(server);
+	}
+	return ret;
+}
+
 static void scoutfs_server_worker(struct work_struct *work)
 {
 	struct server_info *server = container_of(work, struct server_info,
@@ -1548,18 +1750,16 @@ static void scoutfs_server_worker(struct work_struct *work)
 	struct super_block *sb = server->sb;
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
 	struct scoutfs_super_block *super = &sbi->super;
+	struct mount_options *opts = &sbi->opts;
 	struct scoutfs_net_connection *conn = NULL;
 	DECLARE_WAIT_QUEUE_HEAD(waitq);
 	struct sockaddr_in sin;
-	LIST_HEAD(conn_list);
 	u64 max_vers;
 	int ret;
-	int err;
 
 	trace_scoutfs_server_work_enter(sb, 0, 0);
 
-	sin = server->listen_sin;
-
+	scoutfs_quorum_slot_sin(super, opts->quorum_slot_nr, &sin);
 	scoutfs_info(sb, "server setting up at "SIN_FMT, SIN_ARG(&sin));
 
 	conn = scoutfs_net_alloc_conn(sb, server_notify_up, server_notify_down,
@@ -1579,9 +1779,6 @@ static void scoutfs_server_worker(struct work_struct *work)
 		goto out;
 	}
 
-	if (ret)
-		goto out;
-
 	/* start up the server subsystems before accepting */
 	ret = scoutfs_read_super(sb, super);
 	if (ret < 0)
@@ -1616,24 +1813,11 @@ static void scoutfs_server_worker(struct work_struct *work)
 		goto shutdown;
 	}
 
-	ret = scoutfs_lock_server_setup(sb, &server->alloc, &server->wri,
-					max_vers);
+	ret = scoutfs_lock_server_setup(sb, &server->alloc, &server->wri, max_vers) ?:
+	      start_recovery(sb);
 	if (ret)
 		goto shutdown;
 
-	/*
-	 * Write our address in the super before it's possible for net
-	 * processing to start writing the super as part of
-	 * transactions.  In theory clients could be trying to connect
-	 * to our address without having seen it in the super (maybe
-	 * they saw it a long time ago).
-	 */
-	scoutfs_addr_from_sin(&super->server_addr, &sin);
-	super->quorum_server_term = cpu_to_le64(server->term);
-	ret = scoutfs_write_super(sb, super);
-	if (ret < 0)
-		goto shutdown;
-
 	/* start accepting connections and processing work */
 	server->conn = conn;
 	scoutfs_net_listen(sb, conn);
@@ -1654,36 +1838,25 @@ shutdown:
 	scoutfs_net_shutdown(sb, conn);
 	server->conn = NULL;
 
+	/* stop tracking recovery, cancel timer, flush any fencing */
+	scoutfs_recov_shutdown(sb);
+	flush_work(&server->fence_pending_recov_work);
+
 	/* wait for extra queues by requests, won't find waiters */
 	flush_work(&server->commit_work);
 
 	scoutfs_lock_server_destroy(sb);
+	scoutfs_omap_server_shutdown(sb);
 
 out:
-	scoutfs_quorum_clear_leader(sb);
 	scoutfs_net_free_conn(sb, conn);
 
+	/* let quorum know that we've shutdown */
+	scoutfs_quorum_server_shutdown(sb);
+
 	scoutfs_info(sb, "server stopped at "SIN_FMT, SIN_ARG(&sin));
 	trace_scoutfs_server_work_exit(sb, 0, ret);
 
-	/*
-	 * Always try to clear our presence in the super so that we're
-	 * not fenced.  We do this last because other mounts will try to
-	 * reach quorum the moment they see zero here.  The later we do
-	 * this the longer we have to finish shutdown while clients
-	 * timeout.
-	 */
-	err = scoutfs_read_super(sb, super);
-	if (err == 0) {
-		super->quorum_fenced_term = cpu_to_le64(server->term);
-		memset(&super->server_addr, 0, sizeof(super->server_addr));
-		err = scoutfs_write_super(sb, super);
-	}
-	if (err < 0) {
-		scoutfs_err(sb, "failed to clear election term %llu at "SIN_FMT", this mount could be fenced",
-			    server->term, SIN_ARG(&sin));
-	}
-
 	server->err = ret;
 	complete(&server->start_comp);
 }
@@ -1693,14 +1866,12 @@ out:
  * the super block's fence_term has been set to the new server's term so
  * that it won't be fenced.
  */
-int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin,
-			 u64 term)
+int scoutfs_server_start(struct super_block *sb, u64 term)
 {
 	DECLARE_SERVER_INFO(sb, server);
 
 	server->err = 0;
 	server->shutting_down = false;
-	server->listen_sin = *sin;
 	server->term = term;
 	init_completion(&server->start_comp);
 
@@ -1753,13 +1924,15 @@ int scoutfs_server_setup(struct super_block *sb)
 	INIT_WORK(&server->commit_work, scoutfs_server_commit_func);
 	init_rwsem(&server->seq_rwsem);
 	INIT_LIST_HEAD(&server->clients);
-	mutex_init(&server->farewell_mutex);
+	spin_lock_init(&server->farewell_lock);
 	INIT_LIST_HEAD(&server->farewell_requests);
 	INIT_WORK(&server->farewell_work, farewell_worker);
 	mutex_init(&server->alloc_mutex);
 	mutex_init(&server->logs_mutex);
 	mutex_init(&server->srch_mutex);
+	mutex_init(&server->mounted_clients_mutex);
 	seqcount_init(&server->roots_seqcount);
+	INIT_WORK(&server->fence_pending_recov_work, fence_pending_recov_worker);
 
 	server->wq = alloc_workqueue("scoutfs_server",
 				     WQ_UNBOUND | WQ_NON_REENTRANT, 0);

kmod/src/server.h

@@ -59,18 +59,21 @@ do {								\
 int scoutfs_server_lock_request(struct super_block *sb, u64 rid,
 				struct scoutfs_net_lock *nl);
 int scoutfs_server_lock_response(struct super_block *sb, u64 rid, u64 id,
-				 struct scoutfs_net_lock_grant_response *gr);
+				 struct scoutfs_net_lock *nl);
 int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid,
 					struct scoutfs_key *key);
-void scoutfs_server_get_roots(struct super_block *sb,
-			      struct scoutfs_net_roots *roots);
 int scoutfs_server_hold_commit(struct super_block *sb);
 int scoutfs_server_apply_commit(struct super_block *sb, int err);
+void scoutfs_server_recov_finish(struct super_block *sb, u64 rid, int which);
+
+int scoutfs_server_send_omap_request(struct super_block *sb, u64 rid,
+				     struct scoutfs_open_ino_map_args *args);
+int scoutfs_server_send_omap_response(struct super_block *sb, u64 rid, u64 id,
+				      struct scoutfs_open_ino_map *map, int err);
 
 struct sockaddr_in;
 struct scoutfs_quorum_elected_info;
-int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin,
-			 u64 term);
+int scoutfs_server_start(struct super_block *sb, u64 term);
 void scoutfs_server_abort(struct super_block *sb);
 void scoutfs_server_stop(struct super_block *sb);
 

kmod/src/srch.c

@@ -255,24 +255,9 @@ static u8 height_for_blk(u64 blk)
 	return hei;
 }
 
-static void init_file_block(struct super_block *sb, struct scoutfs_block *bl,
-			    int level)
+static inline u32 srch_level_magic(int level)
 {
-	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
-	struct scoutfs_block_header *hdr;
-
-	/* don't leak uninit kernel mem.. block should do this for us? */
-	memset(bl->data, 0, SCOUTFS_BLOCK_LG_SIZE);
-
-	hdr = bl->data;
-	hdr->fsid = super->hdr.fsid;
-	hdr->blkno = cpu_to_le64(bl->blkno);
-	prandom_bytes(&hdr->seq, sizeof(hdr->seq));
-
-	if (level)
-		hdr->magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SRCH_PARENT);
-	else
-		hdr->magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK);
+	return level ? SCOUTFS_BLOCK_MAGIC_SRCH_PARENT : SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK;
 }
 
 /*
@@ -284,39 +269,15 @@ static void init_file_block(struct super_block *sb, struct scoutfs_block *bl,
  */
 static int read_srch_block(struct super_block *sb,
 			   struct scoutfs_block_writer *wri, int level,
-			   struct scoutfs_srch_ref *ref,
+			   struct scoutfs_block_ref *ref,
 			   struct scoutfs_block **bl_ret)
 {
-	struct scoutfs_block *bl;
-	int retries = 0;
-	int ret = 0;
-	int mag;
+	u32 magic = srch_level_magic(level);
+	int ret;
 
-	mag = level ? SCOUTFS_BLOCK_MAGIC_SRCH_PARENT :
-		      SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK;
-retry:
-	bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno));
-	if (!IS_ERR_OR_NULL(bl) &&
-	    !scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno, mag)) {
-
-		scoutfs_inc_counter(sb, srch_inconsistent_ref);
-		scoutfs_block_writer_forget(sb, wri, bl);
-		scoutfs_block_invalidate(sb, bl);
-		scoutfs_block_put(sb, bl);
-		bl = NULL;
-
-		if (retries++ == 0)
-			goto retry;
-
-		bl = ERR_PTR(-ESTALE);
+	ret = scoutfs_block_read_ref(sb, ref, magic, bl_ret);
+	if (ret == -ESTALE)
 		scoutfs_inc_counter(sb, srch_read_stale);
-	}
-	if (IS_ERR(bl)) {
-		ret = PTR_ERR(bl);
-		bl = NULL;
-	}
-
-	*bl_ret = bl;
 	return ret;
 }
 
@@ -333,7 +294,7 @@ static int read_path_block(struct super_block *sb,
 {
 	struct scoutfs_block *bl = NULL;
 	struct scoutfs_srch_parent *srp;
-	struct scoutfs_srch_ref ref;
+	struct scoutfs_block_ref ref;
 	int level;
 	int ind;
 	int ret;
@@ -392,12 +353,10 @@ static int get_file_block(struct super_block *sb,
 	struct scoutfs_block_header *hdr;
 	struct scoutfs_block *bl = NULL;
 	struct scoutfs_srch_parent *srp;
-	struct scoutfs_block *new_bl;
-	struct scoutfs_srch_ref *ref;
-	u64 blkno = 0;
+	struct scoutfs_block_ref new_root_ref;
+	struct scoutfs_block_ref *ref;
 	int level;
 	int ind;
-	int err;
 	int ret;
 	u8 hei;
 
@@ -409,29 +368,21 @@ static int get_file_block(struct super_block *sb,
 			goto out;
 		}
 
-		ret = scoutfs_alloc_meta(sb, alloc, wri, &blkno);
+		memset(&new_root_ref, 0, sizeof(new_root_ref));
+		level = sfl->height;
+
+		ret = scoutfs_block_dirty_ref(sb, alloc, wri, &new_root_ref,
+					      srch_level_magic(level), &bl, 0, NULL);
 		if (ret < 0)
 			goto out;
 
-		bl = scoutfs_block_create(sb, blkno);
-		if (IS_ERR(bl)) {
-			ret = PTR_ERR(bl);
-			goto out;
-		}
-		blkno = 0;
-
-		scoutfs_block_writer_mark_dirty(sb, wri, bl);
-
-		init_file_block(sb, bl, sfl->height);
-		if (sfl->height) {
+		if (level) {
 			srp = bl->data;
-			srp->refs[0].blkno = sfl->ref.blkno;
-			srp->refs[0].seq = sfl->ref.seq;
+			srp->refs[0] = sfl->ref;
 		}
 
 		hdr = bl->data;
-		sfl->ref.blkno = hdr->blkno;
-		sfl->ref.seq = hdr->seq;
+		sfl->ref = new_root_ref;
 		sfl->height++;
 		scoutfs_block_put(sb, bl);
 		bl = NULL;
@@ -447,54 +398,13 @@ static int get_file_block(struct super_block *sb,
 			goto out;
 		}
 
-		/* read an existing block */
-		if (ref->blkno) {
-			ret = read_srch_block(sb, wri, level, ref, &bl);
-			if (ret < 0)
-				goto out;
-		}
-
-		/* allocate a new block if we need it */
-		if (!ref->blkno || ((flags & GFB_DIRTY) &&
-				    !scoutfs_block_writer_is_dirty(sb, bl))) {
-			ret = scoutfs_alloc_meta(sb, alloc, wri, &blkno);
-			if (ret < 0)
-				goto out;
-
-			new_bl = scoutfs_block_create(sb, blkno);
-			if (IS_ERR(new_bl)) {
-				ret = PTR_ERR(new_bl);
-				goto out;
-			}
-
-			if (bl) {
-				/* cow old block if we have one */
-				ret = scoutfs_free_meta(sb, alloc, wri,
-							bl->blkno);
-				if (ret)
-					goto out;
-
-				memcpy(new_bl->data, bl->data,
-				       SCOUTFS_BLOCK_LG_SIZE);
-				scoutfs_block_put(sb, bl);
-				bl = new_bl;
-				hdr = bl->data;
-				hdr->blkno = cpu_to_le64(bl->blkno);
-				prandom_bytes(&hdr->seq, sizeof(hdr->seq));
-			} else {
-				/* init new allocated block */
-				bl = new_bl;
-				init_file_block(sb, bl, level);
-			}
-
-			blkno = 0;
-			scoutfs_block_writer_mark_dirty(sb, wri, bl);
-
-			/* update file or parent block ref */
-			hdr = bl->data;
-			ref->blkno = hdr->blkno;
-			ref->seq = hdr->seq;
-		}
+		if (flags & GFB_DIRTY)
+			ret = scoutfs_block_dirty_ref(sb, alloc, wri, ref, srch_level_magic(level),
+						      &bl, 0, NULL);
+		else
+			ret = scoutfs_block_read_ref(sb, ref, srch_level_magic(level), &bl);
+		if (ret < 0)
+			goto out;
 
 		if (level == 0) {
 			ret = 0;
@@ -514,12 +424,6 @@ static int get_file_block(struct super_block *sb,
 out:
 	scoutfs_block_put(sb, parent);
 
-	/* return allocated blkno on error */
-	if (blkno > 0) {
-		err = scoutfs_free_meta(sb, alloc, wri, blkno);
-		BUG_ON(err); /* radix should have been dirty */
-	}
-
 	if (ret < 0) {
 		scoutfs_block_put(sb, bl);
 		bl = NULL;
@@ -2252,7 +2156,8 @@ static void scoutfs_srch_compact_worker(struct work_struct *work)
 	if (ret < 0)
 		goto commit;
 
-	ret = scoutfs_block_writer_write(sb, &wri);
+	ret = scoutfs_alloc_prepare_commit(sb, &alloc, &wri) ?:
+	      scoutfs_block_writer_write(sb, &wri);
 commit:
 	/* the server won't use our partial compact if _ERROR is set */
 	sc->meta_avail = alloc.avail;

kmod/src/super.c

@@ -44,6 +44,8 @@
 #include "srch.h"
 #include "item.h"
 #include "alloc.h"
+#include "recov.h"
+#include "omap.h"
 #include "scoutfs_trace.h"
 
 static struct dentry *scoutfs_debugfs_root;
@@ -166,7 +168,7 @@ out:
 	 * try to free as many locks as possible.
 	 */
 	if (scoutfs_trigger(sb, STATFS_LOCK_PURGE))
-		scoutfs_free_unused_locks(sb, -1UL);
+		scoutfs_free_unused_locks(sb);
 
 	return ret;
 }
@@ -176,7 +178,8 @@ static int scoutfs_show_options(struct seq_file *seq, struct dentry *root)
 	struct super_block *sb = root->d_sb;
 	struct mount_options *opts = &SCOUTFS_SB(sb)->opts;
 
-	seq_printf(seq, ",server_addr="SIN_FMT, SIN_ARG(&opts->server_addr));
+	if (opts->quorum_slot_nr >= 0)
+		seq_printf(seq, ",quorum_slot_nr=%d", opts->quorum_slot_nr);
 	seq_printf(seq, ",metadev_path=%s", opts->metadev_path);
 
 	return 0;
@@ -192,20 +195,19 @@ static ssize_t metadev_path_show(struct kobject *kobj,
 }
 SCOUTFS_ATTR_RO(metadev_path);
 
-static ssize_t server_addr_show(struct kobject *kobj,
+static ssize_t quorum_server_nr_show(struct kobject *kobj,
 			      struct kobj_attribute *attr, char *buf)
 {
 	struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj);
 	struct mount_options *opts = &SCOUTFS_SB(sb)->opts;
 
-	return snprintf(buf, PAGE_SIZE, SIN_FMT"\n",
-			SIN_ARG(&opts->server_addr));
+	return snprintf(buf, PAGE_SIZE, "%d\n", opts->quorum_slot_nr);
 }
-SCOUTFS_ATTR_RO(server_addr);
+SCOUTFS_ATTR_RO(quorum_server_nr);
 
 static struct attribute *mount_options_attrs[] = {
 	SCOUTFS_ATTR_PTR(metadev_path),
-	SCOUTFS_ATTR_PTR(server_addr),
+	SCOUTFS_ATTR_PTR(quorum_server_nr),
 	NULL,
 };
 
@@ -243,28 +245,26 @@ static void scoutfs_put_super(struct super_block *sb)
 
 	trace_scoutfs_put_super(sb);
 
-	sbi->shutdown = true;
-
-	scoutfs_data_destroy(sb);
 	scoutfs_srch_destroy(sb);
 
 	scoutfs_unlock(sb, sbi->rid_lock, SCOUTFS_LOCK_WRITE);
 	sbi->rid_lock = NULL;
 
+	scoutfs_lock_shutdown(sb);
+
 	scoutfs_shutdown_trans(sb);
 	scoutfs_client_destroy(sb);
 	scoutfs_inode_destroy(sb);
 	scoutfs_item_destroy(sb);
 	scoutfs_forest_destroy(sb);
+	scoutfs_data_destroy(sb);
 
-	/* the server locks the listen address and compacts */
-	scoutfs_lock_shutdown(sb);
+	scoutfs_quorum_destroy(sb);
 	scoutfs_server_destroy(sb);
+	scoutfs_recov_destroy(sb);
 	scoutfs_net_destroy(sb);
 	scoutfs_lock_destroy(sb);
-
-	/* server clears quorum leader flag during shutdown */
-	scoutfs_quorum_destroy(sb);
+	scoutfs_omap_destroy(sb);
 
 	scoutfs_block_destroy(sb);
 	scoutfs_destroy_triggers(sb);
@@ -309,6 +309,34 @@ int scoutfs_write_super(struct super_block *sb,
 				      sizeof(struct scoutfs_super_block));
 }
 
+static bool invalid_blkno_limits(struct super_block *sb, char *which,
+				 u64 start, __le64 first, __le64 last,
+				 struct block_device *bdev, int shift)
+{
+	u64 blkno;
+
+	if (le64_to_cpu(first) < start) {
+		scoutfs_err(sb, "super block first %s blkno %llu is within first valid blkno %llu",
+			which, le64_to_cpu(first), start);
+		return true;
+	}
+
+	if (le64_to_cpu(first) > le64_to_cpu(last)) {
+		scoutfs_err(sb, "super block first %s blkno %llu is greater than last %s blkno %llu",
+			which, le64_to_cpu(first), which, le64_to_cpu(last));
+		return true;
+	}
+
+	blkno = (i_size_read(bdev->bd_inode) >> shift) - 1;
+	if (le64_to_cpu(last) > blkno) {
+		scoutfs_err(sb, "super block last %s blkno %llu is beyond device size last blkno %llu",
+			which, le64_to_cpu(last), blkno);
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * Read super, specifying bdev.
  */
@@ -316,9 +344,9 @@ static int scoutfs_read_super_from_bdev(struct super_block *sb,
 					struct block_device *bdev,
 					struct scoutfs_super_block *super_res)
 {
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
 	struct scoutfs_super_block *super;
 	__le32 calc;
-	u64 blkno;
 	int ret;
 
 	super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS);
@@ -362,48 +390,17 @@ static int scoutfs_read_super_from_bdev(struct super_block *sb,
 
 	/* XXX do we want more rigorous invalid super checking? */
 
-	if (super->quorum_count == 0 ||
-	    super->quorum_count > SCOUTFS_QUORUM_MAX_COUNT) {
-		scoutfs_err(sb, "super block has invalid quorum count %u, must be > 0 and <= %u",
-			    super->quorum_count, SCOUTFS_QUORUM_MAX_COUNT);
+	if (invalid_blkno_limits(sb, "meta",
+			         SCOUTFS_META_DEV_START_BLKNO,
+				 super->first_meta_blkno,
+				 super->last_meta_blkno, sbi->meta_bdev,
+				 SCOUTFS_BLOCK_LG_SHIFT) ||
+	    invalid_blkno_limits(sb, "data",
+			         SCOUTFS_DATA_DEV_START_BLKNO,
+				 super->first_data_blkno,
+				 super->last_data_blkno, sb->s_bdev,
+				 SCOUTFS_BLOCK_SM_SHIFT)) {
 		ret = -EINVAL;
-		goto out;
-	}
-
-	blkno = (SCOUTFS_QUORUM_BLKNO + SCOUTFS_QUORUM_BLOCKS) >>
-		SCOUTFS_BLOCK_SM_LG_SHIFT;
-	if (le64_to_cpu(super->first_meta_blkno) < blkno) {
-		scoutfs_err(sb, "super block first meta blkno %llu is within quorum blocks",
-			le64_to_cpu(super->first_meta_blkno));
-		ret = -EINVAL;
-		goto out;
-	}
-
-	if (le64_to_cpu(super->first_meta_blkno) >
-	    le64_to_cpu(super->last_meta_blkno)) {
-		scoutfs_err(sb, "super block first meta blkno %llu is greater than last meta blkno %llu",
-			le64_to_cpu(super->first_meta_blkno),
-			le64_to_cpu(super->last_meta_blkno));
-		ret = -EINVAL;
-		goto out;
-	}
-
-	if (le64_to_cpu(super->first_data_blkno) >
-	    le64_to_cpu(super->last_data_blkno)) {
-		scoutfs_err(sb, "super block first data blkno %llu is greater than last data blkno %llu",
-			le64_to_cpu(super->first_data_blkno),
-			le64_to_cpu(super->last_data_blkno));
-		ret = -EINVAL;
-		goto out;
-	}
-
-	blkno = (i_size_read(sb->s_bdev->bd_inode) >>
-		 SCOUTFS_BLOCK_SM_SHIFT) - 1;
-	if (le64_to_cpu(super->last_data_blkno) > blkno) {
-		scoutfs_err(sb, "super block last data blkno %llu is outsite device size last blkno %llu",
-			le64_to_cpu(super->last_data_blkno), blkno);
-		ret = -EINVAL;
-		goto out;
 	}
 
 out:
@@ -597,10 +594,12 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 	      scoutfs_inode_setup(sb) ?:
 	      scoutfs_data_setup(sb) ?:
 	      scoutfs_setup_trans(sb) ?:
+	      scoutfs_omap_setup(sb) ?:
 	      scoutfs_lock_setup(sb) ?:
 	      scoutfs_net_setup(sb) ?:
-	      scoutfs_quorum_setup(sb) ?:
+	      scoutfs_recov_setup(sb) ?:
 	      scoutfs_server_setup(sb) ?:
+	      scoutfs_quorum_setup(sb) ?:
 	      scoutfs_client_setup(sb) ?:
 	      scoutfs_lock_rid(sb, SCOUTFS_LOCK_WRITE, 0, sbi->rid,
 				   &sbi->rid_lock) ?:
@@ -649,6 +648,9 @@ static void scoutfs_kill_sb(struct super_block *sb)
 {
 	trace_scoutfs_kill_sb(sb);
 
+	if (SCOUTFS_HAS_SBI(sb))
+		scoutfs_lock_unmount_begin(sb);
+
 	kill_block_super(sb);
 }
 

kmod/src/super.h

@@ -26,6 +26,8 @@ struct net_info;
 struct block_info;
 struct forest_info;
 struct srch_info;
+struct recov_info;
+struct omap_info;
 
 struct scoutfs_sb_info {
 	struct super_block *sb;
@@ -48,6 +50,7 @@ struct scoutfs_sb_info {
 	struct block_info *block_info;
 	struct forest_info *forest_info;
 	struct srch_info *srch_info;
+	struct omap_info *omap_info;
 	struct item_cache_info *item_cache_info;
 
 	wait_queue_head_t trans_hold_wq;
@@ -70,6 +73,7 @@ struct scoutfs_sb_info {
 	struct lock_server_info *lock_server_info;
 	struct client_info *client_info;
 	struct server_info *server_info;
+	struct recov_info *recov_info;
 	struct sysfs_info *sfsinfo;
 
 	struct scoutfs_counters *counters;
@@ -81,8 +85,6 @@ struct scoutfs_sb_info {
 
 	struct dentry *debug_root;
 
-	bool shutdown;
-
 	unsigned long corruption_messages_once[SC_NR_LONGS];
 };
 

kmod/src/trans.c

@@ -39,17 +39,15 @@
  * track the relationships between dirty blocks so there's only ever one
  * transaction being built.
  *
- * The copy of the on-disk super block in the fs sb info has its header
- * sequence advanced so that new dirty blocks inherit this dirty
- * sequence number.  It's only advanced once all those dirty blocks are
- * reachable after having first written them all out and then the new
- * super with that seq.  It's first incremented at mount.
+ * Committing the current dirty transaction can be triggered by sync, a
+ * regular background commit interval, reaching a dirty block threshold,
+ * or the transaction running out of its private allocator resources.
+ * Once all the current holders release the writing func writes out the
+ * dirty blocks while excluding holders until it finishes.
  *
- * Unfortunately writers can nest.  We don't bother trying to special
- * case holding a transaction that you're already holding because that
- * requires per-task storage.  We just let anyone hold transactions
- * regardless of waiters waiting to write, which risks waiters waiting a
- * very long time.
+ * Unfortunately writing holders can nest.  We track nested hold callers
+ * with the per-task journal_info pointer to avoid deadlocks between
+ * holders that might otherwise wait for a pending commit.
  */
 
 /* sync dirty data at least this often */
@@ -59,9 +57,7 @@
  * XXX move the rest of the super trans_ fields here.
  */
 struct trans_info {
-	spinlock_t lock;
-	unsigned holders;
-	bool writing;
+	atomic_t holders;
 
 	struct scoutfs_log_trees lt;
 	struct scoutfs_alloc alloc;
@@ -71,17 +67,9 @@ struct trans_info {
 #define DECLARE_TRANS_INFO(sb, name) \
 	struct trans_info *name = SCOUTFS_SB(sb)->trans_info
 
-static bool drained_holders(struct trans_info *tri)
-{
-	bool drained;
-
-	spin_lock(&tri->lock);
-	tri->writing = true;
-	drained = tri->holders == 0;
-	spin_unlock(&tri->lock);
-
-	return drained;
-}
+/* avoid the high sign bit out of an abundance of caution*/
+#define TRANS_HOLDERS_WRITE_FUNC_BIT	(1 << 30)
+#define TRANS_HOLDERS_COUNT_MASK	(TRANS_HOLDERS_WRITE_FUNC_BIT - 1)
 
 static int commit_btrees(struct super_block *sb)
 {
@@ -126,6 +114,36 @@ bool scoutfs_trans_has_dirty(struct super_block *sb)
 	return scoutfs_block_writer_has_dirty(sb, &tri->wri);
 }
 
+/*
+ * This is racing with wait_event conditions, make sure our atomic
+ * stores and waitqueue loads are ordered.
+ */
+static void sub_holders_and_wake(struct super_block *sb, int val)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	DECLARE_TRANS_INFO(sb, tri);
+
+	atomic_sub(val, &tri->holders);
+	smp_mb(); /* make sure sub is visible before we wake */
+	if (waitqueue_active(&sbi->trans_hold_wq))
+		wake_up(&sbi->trans_hold_wq);
+}
+
+/*
+ * called as a wait_event condition, needs to be careful to not change
+ * task state and is racing with waking paths that sub_return, test, and
+ * wake.
+ */
+static bool drained_holders(struct trans_info *tri)
+{
+	int holders;
+
+	smp_mb(); /* make sure task in wait_event queue before atomic read */
+	holders = atomic_read(&tri->holders) & TRANS_HOLDERS_COUNT_MASK;
+
+	return holders == 0;
+}
+
 /*
  * This work func is responsible for writing out all the dirty blocks
  * that make up the current dirty transaction.  It prevents writers from
@@ -162,6 +180,9 @@ void scoutfs_trans_write_func(struct work_struct *work)
 
 	sbi->trans_task = current;
 
+	/* mark that we're writing so holders wait for us to finish and clear our bit */
+	atomic_add(TRANS_HOLDERS_WRITE_FUNC_BIT, &tri->holders);
+
 	wait_event(sbi->trans_hold_wq, drained_holders(tri));
 
 	trace_scoutfs_trans_write_func(sb,
@@ -213,11 +234,8 @@ out:
 	spin_unlock(&sbi->trans_write_lock);
 	wake_up(&sbi->trans_write_wq);
 
-	spin_lock(&tri->lock);
-	tri->writing = false;
-	spin_unlock(&tri->lock);
-
-	wake_up(&sbi->trans_hold_wq);
+	/* we're done, wake waiting holders */
+	sub_holders_and_wake(sb, TRANS_HOLDERS_WRITE_FUNC_BIT);
 
 	sbi->trans_task = NULL;
 
@@ -309,53 +327,83 @@ void scoutfs_trans_restart_sync_deadline(struct super_block *sb)
 }
 
 /*
- * Each thread reserves space in the segment for their dirty items while
- * they hold the transaction.  This is calculated before the first
- * transaction hold is acquired.  It includes all the potential nested
- * item manipulation that could happen with the transaction held.
- * Including nested holds avoids having to deal with writing out partial
- * transactions while a caller still holds the transaction.
+ * We store nested holders in the lower bits of journal_info.  We use
+ * some higher bits as a magic value to detect if something goes
+ * horribly wrong and it gets clobbered.
  */
+#define TRANS_JI_MAGIC		0xd5700000
+#define TRANS_JI_MAGIC_MASK	0xfff00000
+#define TRANS_JI_COUNT_MASK	0x000fffff
 
-#define SCOUTFS_RESERVATION_MAGIC 0xd57cd13b
-struct scoutfs_reservation {
-	unsigned magic;
-	unsigned holders;
-};
+/* returns true if a caller already had a holder counted in journal_info */
+static bool inc_journal_info_holders(void)
+{
+	unsigned long holders = (unsigned long)current->journal_info;
+
+	WARN_ON_ONCE(holders != 0 && ((holders & TRANS_JI_MAGIC_MASK) != TRANS_JI_MAGIC));
+
+	if (holders == 0)
+		holders = TRANS_JI_MAGIC;
+	holders++;
+
+	current->journal_info = (void *)holders;
+	return (holders > (TRANS_JI_MAGIC | 1));
+}
+
+static void dec_journal_info_holders(void)
+{
+	unsigned long holders = (unsigned long)current->journal_info;
+
+	WARN_ON_ONCE(holders != 0 && ((holders & TRANS_JI_MAGIC_MASK) != TRANS_JI_MAGIC));
+	WARN_ON_ONCE((holders & TRANS_JI_COUNT_MASK) == 0);
+
+	holders--;
+	if (holders == TRANS_JI_MAGIC)
+		holders = 0;
+
+	current->journal_info = (void *)holders;
+}
 
 /*
- * Try to hold the transaction.  If a caller already holds the trans then
- * we piggy back on their hold.  We wait if the writer is trying to
- * write out the transation.  And if our items won't fit then we kick off
- * a write.
+ * This is called as the wait_event condition for holding a transaction.
+ * Increment the holder count unless the writer is present.  We return
+ * false to wait until the writer finishes and wakes us.
  *
- * This is called as a condition for wait_event.  It is very limited in
- * the locking (blocking) it can do because the caller has set the task
- * state before testing the condition safely race with waking after
- * setting the condition.  Our checking the amount of dirty metadata
- * blocks and free data blocks is racy, but we don't mind the risk of
- * delaying or prematurely forcing commits.
+ * This can be racing with itself while there's no waiters.  We retry
+ * the cmpxchg instead of returning and waiting.
  */
-static bool acquired_hold(struct super_block *sb,
-			  struct scoutfs_reservation *rsv)
+static bool inc_holders_unless_writer(struct trans_info *tri)
 {
-	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
-	DECLARE_TRANS_INFO(sb, tri);
-	bool acquired = false;
+	int holders;
 
-	spin_lock(&tri->lock);
+	do {
+		smp_mb(); /* make sure we read after wait puts task in queue */
+		holders = atomic_read(&tri->holders);
+		if (holders & TRANS_HOLDERS_WRITE_FUNC_BIT)
+			return false;
 
-	trace_scoutfs_trans_acquired_hold(sb, rsv, rsv->holders,
-					  tri->holders, tri->writing);
+	} while (atomic_cmpxchg(&tri->holders, holders, holders + 1) != holders);
 
-	/* use a caller's existing reservation */
-	if (rsv->holders)
-		goto hold;
+	return true;
+}
 
-	/* wait until the writing thread is finished */
-	if (tri->writing)
-		goto out;
+/*
+ * As we drop the last trans holder we try to wake a writing thread that
+ * was waiting for us to finish.
+ */
+static void release_holders(struct super_block *sb)
+{
+	dec_journal_info_holders();
+	sub_holders_and_wake(sb, 1);
+}
 
+/*
+ * The caller has incremented holders so it is blocking commits.  We
+ * make some quick checks to see if we need to trigger and wait for
+ * another commit before proceeding.
+ */
+static bool commit_before_hold(struct super_block *sb, struct trans_info *tri)
+{
 	/*
 	 * In theory each dirty item page could be straddling two full
 	 * blocks, requiring 4 allocations for each item cache page.
@@ -365,11 +413,9 @@ static bool acquired_hold(struct super_block *sb,
 	 * that it accounts for having to dirty parent blocks and
 	 * whatever dirtying is done during the transaction hold.
 	 */
-	if (scoutfs_alloc_meta_low(sb, &tri->alloc,
-				   scoutfs_item_dirty_pages(sb) * 2)) {
+	if (scoutfs_alloc_meta_low(sb, &tri->alloc, scoutfs_item_dirty_pages(sb) * 2)) {
 		scoutfs_inc_counter(sb, trans_commit_dirty_meta_full);
-		queue_trans_work(sbi);
-		goto out;
+		return true;
 	}
 
 	/*
@@ -381,57 +427,74 @@ static bool acquired_hold(struct super_block *sb,
 	 */
 	if (scoutfs_alloc_meta_low(sb, &tri->alloc, 16)) {
 		scoutfs_inc_counter(sb, trans_commit_meta_alloc_low);
-		queue_trans_work(sbi);
-		goto out;
+		return true;
 	}
 
 	/* Try to refill data allocator before premature enospc */
 	if (scoutfs_data_alloc_free_bytes(sb) <= SCOUTFS_TRANS_DATA_ALLOC_LWM) {
 		scoutfs_inc_counter(sb, trans_commit_data_alloc_low);
-		queue_trans_work(sbi);
+		return true;
+	}
+
+	return false;
+}
+
+static bool acquired_hold(struct super_block *sb)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	DECLARE_TRANS_INFO(sb, tri);
+	bool acquired;
+
+	/* if a caller already has a hold we acquire unconditionally */
+	if (inc_journal_info_holders()) {
+		atomic_inc(&tri->holders);
+		acquired = true;
 		goto out;
 	}
 
-hold:
-	rsv->holders++;
-	tri->holders++;
+	/* wait if the writer is blocking holds */
+	if (!inc_holders_unless_writer(tri)) {
+		dec_journal_info_holders();
+		acquired = false;
+		goto out;
+	}
+
+	/* wait if we're triggering another commit */
+	if (commit_before_hold(sb, tri)) {
+		release_holders(sb);
+		queue_trans_work(sbi);
+		acquired = false;
+		goto out;
+	}
+
+	trace_scoutfs_trans_acquired_hold(sb, current->journal_info, atomic_read(&tri->holders));
 	acquired = true;
-
 out:
-
-	spin_unlock(&tri->lock);
-
 	return acquired;
 }
 
+/*
+ * Try to hold the transaction.  Holding the transaction prevents it
+ * from being committed.  If a transaction is currently being written
+ * then we'll block until it's done and our hold can be granted.
+ *
+ * If a caller already holds the trans then we unconditionally acquire
+ * our hold and return to avoid deadlocks with our caller, the writing
+ * thread, and us.  We record nested holds in a call stack with the
+ * journal_info pointer in the task_struct.
+ *
+ * The writing thread marks itself as a global trans_task which
+ * short-circuits all the hold machinery so it can call code that would
+ * otherwise try to hold transactions while it is writing.
+ */
 int scoutfs_hold_trans(struct super_block *sb)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
-	struct scoutfs_reservation *rsv;
-	int ret;
 
 	if (current == sbi->trans_task)
 		return 0;
 
-	rsv = current->journal_info;
-	if (rsv == NULL) {
-		rsv = kzalloc(sizeof(struct scoutfs_reservation), GFP_NOFS);
-		if (!rsv)
-			return -ENOMEM;
-
-		rsv->magic = SCOUTFS_RESERVATION_MAGIC;
-		current->journal_info = rsv;
-	}
-
-	BUG_ON(rsv->magic != SCOUTFS_RESERVATION_MAGIC);
-
-	ret = wait_event_interruptible(sbi->trans_hold_wq,
-				       acquired_hold(sb, rsv));
-	if (ret && rsv->holders == 0) {
-		current->journal_info = NULL;
-		kfree(rsv);
-	}
-	return ret;
+	return wait_event_interruptible(sbi->trans_hold_wq, acquired_hold(sb));
 }
 
 /*
@@ -441,50 +504,22 @@ int scoutfs_hold_trans(struct super_block *sb)
  */
 bool scoutfs_trans_held(void)
 {
-	struct scoutfs_reservation *rsv = current->journal_info;
+	unsigned long holders = (unsigned long)current->journal_info;
 
-	return rsv && rsv->magic == SCOUTFS_RESERVATION_MAGIC;
+	return (holders != 0 && ((holders & TRANS_JI_MAGIC_MASK) == TRANS_JI_MAGIC));
 }
 
-/*
- * As we drop the last hold in the reservation we try and wake other
- * hold attempts that were waiting for space.  As we drop the last trans
- * holder we try to wake a writing thread that was waiting for us to
- * finish.
- */
 void scoutfs_release_trans(struct super_block *sb)
 {
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
-	struct scoutfs_reservation *rsv;
 	DECLARE_TRANS_INFO(sb, tri);
-	bool wake = false;
 
 	if (current == sbi->trans_task)
 		return;
 
-	rsv = current->journal_info;
-	BUG_ON(!rsv || rsv->magic != SCOUTFS_RESERVATION_MAGIC);
+	release_holders(sb);
 
-	spin_lock(&tri->lock);
-
-	trace_scoutfs_release_trans(sb, rsv, rsv->holders, tri->holders, tri->writing);
-
-	BUG_ON(rsv->holders <= 0);
-	BUG_ON(tri->holders <= 0);
-
-	if (--rsv->holders == 0) {
-		current->journal_info = NULL;
-		kfree(rsv);
-		wake = true;
-	}
-
-	if (--tri->holders == 0)
-		wake = true;
-
-	spin_unlock(&tri->lock);
-
-	if (wake)
-		wake_up(&sbi->trans_hold_wq);
+	trace_scoutfs_release_trans(sb, current->journal_info, atomic_read(&tri->holders));
 }
 
 /*
@@ -513,7 +548,7 @@ int scoutfs_setup_trans(struct super_block *sb)
 	if (!tri)
 		return -ENOMEM;
 
-	spin_lock_init(&tri->lock);
+	atomic_set(&tri->holders, 0);
 	scoutfs_block_writer_init(sb, &tri->wri);
 
 	sbi->trans_write_workq = alloc_workqueue("scoutfs_trans",
@@ -529,8 +564,15 @@ int scoutfs_setup_trans(struct super_block *sb)
 }
 
 /*
- * kill_sb calls sync before getting here so we know that dirty data
- * should be in flight.  We just have to wait for it to quiesce.
+ * While the vfs will have done an fs level sync before calling
+ * put_super, we may have done work down in our level after all the fs
+ * ops were done.  An example is final inode deletion in iput, that's
+ * done in generic_shutdown_super after the sync and before calling our
+ * put_super.
+ *
+ * So we always try to write any remaining dirty transactions before
+ * shutting down.  Typically there won't be any dirty data and the
+ * worker will just return.
  */
 void scoutfs_shutdown_trans(struct super_block *sb)
 {
@@ -538,13 +580,18 @@ void scoutfs_shutdown_trans(struct super_block *sb)
 	DECLARE_TRANS_INFO(sb, tri);
 
 	if (tri) {
-		scoutfs_block_writer_forget_all(sb, &tri->wri);
 		if (sbi->trans_write_workq) {
+			/* immediately queues pending timer */
+			flush_delayed_work(&sbi->trans_write_work);
+			/* prevents re-arming if it has to wait */
 			cancel_delayed_work_sync(&sbi->trans_write_work);
 			destroy_workqueue(sbi->trans_write_workq);
 			/* trans work schedules after shutdown see null */
 			sbi->trans_write_workq = NULL;
 		}
+
+		scoutfs_block_writer_forget_all(sb, &tri->wri);
+
 		kfree(tri);
 		sbi->trans_info = NULL;
 	}

kmod/src/triggers.c

@@ -38,10 +38,7 @@ struct scoutfs_triggers {
 	struct scoutfs_triggers *name = SCOUTFS_SB(sb)->triggers
 
 static char *names[] = {
-	[SCOUTFS_TRIGGER_BTREE_STALE_READ] = "btree_stale_read",
-	[SCOUTFS_TRIGGER_BTREE_ADVANCE_RING_HALF] = "btree_advance_ring_half",
-	[SCOUTFS_TRIGGER_HARD_STALE_ERROR] = "hard_stale_error",
-	[SCOUTFS_TRIGGER_SEG_STALE_READ] = "seg_stale_read",
+	[SCOUTFS_TRIGGER_BLOCK_REMOVE_STALE] = "block_remove_stale",
 	[SCOUTFS_TRIGGER_STATFS_LOCK_PURGE] = "statfs_lock_purge",
 };
 

kmod/src/triggers.h

@@ -2,10 +2,7 @@
 #define _SCOUTFS_TRIGGERS_H_
 
 enum scoutfs_trigger {
-	SCOUTFS_TRIGGER_BTREE_STALE_READ,
-	SCOUTFS_TRIGGER_BTREE_ADVANCE_RING_HALF,
-	SCOUTFS_TRIGGER_HARD_STALE_ERROR,
-	SCOUTFS_TRIGGER_SEG_STALE_READ,
+	SCOUTFS_TRIGGER_BLOCK_REMOVE_STALE,
 	SCOUTFS_TRIGGER_STATFS_LOCK_PURGE,
 	SCOUTFS_TRIGGER_NR,
 };

tests/Makefile

@@ -1,4 +1,4 @@
-CFLAGS := -Wall -O2 -Werror -D_FILE_OFFSET_BITS=64 -fno-strict-aliasing 
+CFLAGS := -Wall -O2 -Werror -D_FILE_OFFSET_BITS=64 -fno-strict-aliasing -I ../kmod/src
 SHELL := /usr/bin/bash
 
 # each binary command is built from a single .c file
@@ -6,6 +6,7 @@ BIN := src/createmany			\
 	src/dumb_setxattr		\
 	src/handle_cat			\
 	src/bulk_create_paths		\
+	src/stage_tmpfile		\
 	src/find_xattrs
 
 DEPS := $(wildcard src/*.d)

tests/funcs/filter.sh

@@ -3,7 +3,7 @@
 t_filter_fs()
 {
 	sed -e 's@mnt/test\.[0-9]*@mnt/test@g' \
-	    -e 's@Device: [a-fA-F0-7]*h/[0-9]*d@Device: 0h/0d@g'
+	    -e 's@Device: [a-fA-F0-9]*h/[0-9]*d@Device: 0h/0d@g'
 }
 
 #
@@ -52,8 +52,8 @@ t_filter_dmesg()
 
 	# tests that drop unmount io triggers fencing
 	re="$re|scoutfs .* error: fencing "
-	re="$re|scoutfs .*: waiting for .* lock clients"
-	re="$re|scoutfs .*: all lock clients recovered"
+	re="$re|scoutfs .*: waiting for .* clients"
+	re="$re|scoutfs .*: all clients recovered"
 	re="$re|scoutfs .* error: client rid.*lock recovery timed out"
 
 	# some tests mount w/o options

tests/funcs/fs.sh

@@ -99,6 +99,19 @@ t_first_client_nr()
 	t_fail "t_first_client_nr didn't find any clients"
 }
 
+#
+# The number of quorum members needed to form a majority to start the
+# server.
+#
+t_majority_count()
+{
+	if [ "$T_QUORUM" -lt 3 ]; then
+		echo 1
+	else
+		echo $(((T_QUORUM / 2) + 1))
+	fi
+}
+
 t_mount()
 {
 	local nr="$1"
@@ -116,7 +129,7 @@ t_umount()
 	test "$nr" -lt "$T_NR_MOUNTS" || \
 		t_fail "fs nr $nr invalid"
 
-	eval t_quiet umount \$T_DB$i
+	eval t_quiet umount \$T_M$nr
 }
 
 #
@@ -196,12 +209,19 @@ t_trigger_show() {
 	echo "trigger $which $string: $(t_trigger_get $which $nr)"
 }
 
-t_trigger_arm() {
+t_trigger_arm_silent() {
 	local which="$1"
 	local nr="$2"
 	local path=$(t_trigger_path "$nr")
 
 	echo 1 > "$path/$which"
+}
+
+t_trigger_arm() {
+	local which="$1"
+	local nr="$2"
+
+	t_trigger_arm_silent $which $nr
 	t_trigger_show $which armed $nr
 }
 
@@ -216,16 +236,44 @@ t_counter() {
 	cat "$(t_sysfs_path $nr)/counters/$which"
 }
 
+#
+# output the difference between the current value of a counter and the
+# caller's provided previous value.
+#
+t_counter_diff_value() {
+	local which="$1"
+	local old="$2"
+	local nr="$3"
+	local new="$(t_counter $which $nr)"
+
+	echo "$((new - old))"
+}
+
 #
 # output the value of the given counter for the given mount, defaulting
-# to mount 0 if a mount isn't specified.
+# to mount 0 if a mount isn't specified.  For tests which expect a
+# specific difference in counters.
 #
 t_counter_diff() {
 	local which="$1"
 	local old="$2"
 	local nr="$3"
-	local new
 
-	new="$(t_counter $which $nr)"
-	echo "counter $which diff $((new - old))"
+	echo "counter $which diff $(t_counter_diff_value $which $old $nr)"
+}
+
+#
+# output a message indicating whether or not the counter value changed.
+# For tests that expect a difference, or not, but the amount of
+# difference isn't significant.
+#
+t_counter_diff_changed() {
+	local which="$1"
+	local old="$2"
+	local nr="$3"
+	local diff="$(t_counter_diff_value $which $old $nr)"
+
+	test "$diff" -eq 0 && \
+		echo "counter $which didn't change" ||
+		echo "counter $which changed"
 }

tests/golden/block-stale-reads

@@ -0,0 +1,52 @@
+== create shared test file
+== set and get xattrs between mount pairs while retrying
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="1"
+
+counter block_cache_remove_stale changed
+counter block_cache_remove_stale changed
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="2"
+
+counter block_cache_remove_stale changed
+counter block_cache_remove_stale changed
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="3"
+
+counter block_cache_remove_stale changed
+counter block_cache_remove_stale changed
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="4"
+
+counter block_cache_remove_stale changed
+counter block_cache_remove_stale changed
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="5"
+
+counter block_cache_remove_stale changed
+counter block_cache_remove_stale changed
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="6"
+
+counter block_cache_remove_stale changed
+counter block_cache_remove_stale changed
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="7"
+
+counter block_cache_remove_stale changed
+counter block_cache_remove_stale changed
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="8"
+
+counter block_cache_remove_stale changed
+counter block_cache_remove_stale changed
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="9"
+
+counter block_cache_remove_stale changed
+counter block_cache_remove_stale changed
+# file: /mnt/test/test/block-stale-reads/file
+user.xat="10"
+
+counter block_cache_remove_stale changed
+counter block_cache_remove_stale changed

tests/golden/inode-deletion

@@ -0,0 +1,27 @@
+== basic unlink deletes
+ino found in dseq index
+ino not found in dseq index
+== local open-unlink waits for close to delete
+contents after rm: contents
+ino found in dseq index
+ino not found in dseq index
+== multiple local opens are protected
+contents after rm 1: contents
+contents after rm 2: contents
+ino found in dseq index
+ino not found in dseq index
+== remote unopened unlink deletes
+ino not found in dseq index
+ino not found in dseq index
+== unlink wait for open on other mount
+mount 0 contents after mount 1 rm: contents
+ino found in dseq index
+ino found in dseq index
+stat: cannot stat ‘/mnt/test/test/inode-deletion/file’: No such file or directory
+ino not found in dseq index
+ino not found in dseq index
+== lots of deletions use one open map
+== open files survive remote scanning orphans
+mount 0 contents after mount 1 remounted: contents
+ino not found in dseq index
+ino not found in dseq index

tests/golden/lock-conflicting-batch-commit

@@ -1,4 +0,0 @@
-== create per mount files
-== time independent modification
-== time concurrent independent modification
-== time concurrent conflicting modification

tests/golden/stage-tmpfile

@@ -0,0 +1,18 @@
+total file size 33669120
+00000000  41 41 41 41 41 41 41 41  41 41 41 41 41 41 41 41  |AAAAAAAAAAAAAAAA|
+*
+00400000  42 42 42 42 42 42 42 42  42 42 42 42 42 42 42 42  |BBBBBBBBBBBBBBBB|
+*
+00801000  43 43 43 43 43 43 43 43  43 43 43 43 43 43 43 43  |CCCCCCCCCCCCCCCC|
+*
+00c03000  44 44 44 44 44 44 44 44  44 44 44 44 44 44 44 44  |DDDDDDDDDDDDDDDD|
+*
+01006000  45 45 45 45 45 45 45 45  45 45 45 45 45 45 45 45  |EEEEEEEEEEEEEEEE|
+*
+0140a000  46 46 46 46 46 46 46 46  46 46 46 46 46 46 46 46  |FFFFFFFFFFFFFFFF|
+*
+0180f000  47 47 47 47 47 47 47 47  47 47 47 47 47 47 47 47  |GGGGGGGGGGGGGGGG|
+*
+01c15000  48 48 48 48 48 48 48 48  48 48 48 48 48 48 48 48  |HHHHHHHHHHHHHHHH|
+*
+0201c000

tests/golden/stale-btree-read

@@ -1,11 +0,0 @@
-== create file for xattr ping pong
-# file: /mnt/test/test/stale-btree-read/file
-user.xat="initial"
-
-== retry btree block read
-trigger btree_stale_read armed: 1
-# file: /mnt/test/test/stale-btree-read/file
-user.xat="btree"
-
-trigger btree_stale_read after: 0
-counter btree_stale_read diff 1

tests/golden/xfstests

@@ -1,6 +1,7 @@
 Ran:
 generic/001
 generic/002
+generic/004
 generic/005
 generic/006
 generic/007
@@ -73,7 +74,6 @@ generic/376
 generic/377
 Not
 run:
-generic/004
 generic/008
 generic/009
 generic/012
@@ -278,4 +278,4 @@ shared/004
 shared/032
 shared/051
 shared/289
-Passed all 72 tests
+Passed all 73 tests

tests/run-tests.sh

@@ -52,11 +52,11 @@ $(basename $0) options:
               | the file system to be tested.  Will be clobbered by -m mkfs.
     -m        | Run mkfs on the device before mounting and running
               | tests.  Implies unmounting existing mounts first.
-    -n        | The number of devices and mounts to test.
+    -n <nr>   | The number of devices and mounts to test.
     -P        | Enable trace_printk.
     -p        | Exit script after preparing mounts only, don't run tests.
-    -q <nr>   | Specify the quorum count needed to mount.  This is
-              | used when running mkfs and is needed by a few tests.
+    -q <nr>   | The first <nr> mounts will be quorum members.  Must be
+              | at least 1 and no greater than -n number of mounts.
     -r <dir>  | Specify the directory in which to store results of
               | test runs.  The directory will be created if it doesn't
               | exist.  Previous results will be deleted as each test runs.
@@ -199,7 +199,6 @@ test -e "$T_EX_META_DEV" || die "extra meta device -f '$T_EX_META_DEV' doesn't e
 test -n "$T_EX_DATA_DEV" || die "must specify -e extra data device"
 test -e "$T_EX_DATA_DEV" || die "extra data device -e '$T_EX_DATA_DEV' doesn't exist"
 
-test -n "$T_MKFS" -a -z "$T_QUORUM" && die "mkfs (-m) requires quorum (-q)"
 test -n "$T_RESULTS" || die "must specify -r results dir"
 test -n "$T_XFSTESTS_REPO" -a -z "$T_XFSTESTS_BRANCH" -a -z "$T_SKIP_CHECKOUT" && \
 	die "-X xfstests repo requires -x xfstests branch"
@@ -209,6 +208,12 @@ test -n "$T_XFSTESTS_BRANCH" -a -z "$T_XFSTESTS_REPO" -a -z "$T_SKIP_CHECKOUT" &
 test -n "$T_NR_MOUNTS" || die "must specify -n nr mounts"
 test "$T_NR_MOUNTS" -ge 1 -a "$T_NR_MOUNTS" -le 8 || \
 	 die "-n nr mounts must be >= 1 and <= 8"
+test -n "$T_QUORUM" || \
+	 die "must specify -q number of mounts that are quorum members"
+test "$T_QUORUM" -ge "1" || \
+	 die "-q quorum mmembers must be at least 1"
+test "$T_QUORUM" -le "$T_NR_MOUNTS" || \
+	 die "-q quorum mmembers must not be greater than -n mounts"
 
 # top level paths
 T_KMOD=$(realpath "$(dirname $0)/../kmod")
@@ -307,8 +312,14 @@ if [ -n "$T_UNMOUNT" ]; then
 	unmount_all
 fi
 
+quo=""
 if [ -n "$T_MKFS" ]; then
-	cmd scoutfs mkfs -Q "$T_QUORUM" "$T_META_DEVICE" "$T_DATA_DEVICE" -f
+	for i in $(seq -0 $((T_QUORUM - 1))); do
+		quo="$quo -Q $i,127.0.0.1,$((42000 + i))"
+	done
+
+	msg "making new filesystem with $T_QUORUM quorum members"
+	cmd scoutfs mkfs -f $quo "$T_META_DEVICE" "$T_DATA_DEVICE"
 fi
 
 if [ -n "$T_INSMOD" ]; then
@@ -365,8 +376,12 @@ for i in $(seq 0 $((T_NR_MOUNTS - 1))); do
 	dir="/mnt/test.$i"
 	test -d "$dir" || cmd mkdir -p "$dir"
 
+	opts="-o metadev_path=$meta_dev"
+	if [ "$i" -lt "$T_QUORUM" ]; then
+		opts="$opts,quorum_slot_nr=$i"
+	fi
+
 	msg "mounting $meta_dev|$data_dev on $dir"
-	opts="-o server_addr=127.0.0.1,metadev_path=$meta_dev"
 	cmd mount -t scoutfs $opts "$data_dev" "$dir" &
 
 	p="$!"

tests/sequence

@@ -13,21 +13,23 @@ lock-refleak.sh
 lock-shrink-consistency.sh
 lock-pr-cw-conflict.sh
 lock-revoke-getcwd.sh
+export-lookup-evict-race.sh
 createmany-parallel.sh
 createmany-large-names.sh
 createmany-rename-large-dir.sh
 stage-release-race-alloc.sh
 stage-multi-part.sh
+stage-tmpfile.sh
 basic-posix-consistency.sh
 dirent-consistency.sh
+mkdir-rename-rmdir.sh
 lock-ex-race-processes.sh
-lock-conflicting-batch-commit.sh
 cross-mount-data-free.sh
 persistent-item-vers.sh
 setup-error-teardown.sh
-# failing in jenkins pr runners, zab's working on it
-#umount-unmount-race.sh
+mount-unmount-race.sh
 createmany-parallel-mounts.sh
 archive-light-cycle.sh
-stale-btree-read.sh
+block-stale-reads.sh
+inode-deletion.sh
 xfstests.sh

tests/src/stage_tmpfile.c

@@ -0,0 +1,145 @@
+/*
+ * Exercise O_TMPFILE creation as well as staging from tmpfiles into
+ * a released destination file.
+ *
+ * Copyright (C) 2021 Versity Software, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <linux/types.h>
+#include <assert.h>
+
+#include "ioctl.h"
+
+#define array_size(arr) (sizeof(arr) / sizeof(arr[0]))
+
+/*
+ * Write known data into 8 tmpfiles.
+ * Make a new file X and release it
+ * Move contents of 8 tmpfiles into X.
+ */
+
+struct sub_tmp_info {
+	int fd;
+	unsigned int offset;
+	unsigned int length;
+};
+
+#define SZ	4096
+char buf[SZ];
+
+int main(int argc, char **argv)
+{
+	struct scoutfs_ioctl_release ioctl_args = {0};
+	struct scoutfs_ioctl_move_blocks mb;
+	struct sub_tmp_info sub_tmps[8];
+	int tot_size = 0;
+	char *dest_file;
+	int dest_fd;
+	char *mnt;
+	int ret;
+	int i;
+
+	if (argc < 3) {
+		printf("%s <mountpoint> <dest_file>\n", argv[0]);
+		return 1;
+	}
+
+	mnt = argv[1];
+	dest_file = argv[2];
+
+	for (i = 0; i < array_size(sub_tmps); i++) {
+		struct sub_tmp_info *sub_tmp = &sub_tmps[i];
+		int remaining;
+
+		sub_tmp->fd = open(mnt, O_RDWR | O_TMPFILE, S_IRUSR | S_IWUSR);
+		if (sub_tmp->fd < 0) {
+			perror("error");
+			exit(1);
+		}
+
+		sub_tmp->offset = tot_size;
+
+		/* First tmp file is 4MB */
+		/* Each is 4k bigger than last */
+		sub_tmp->length = (i + 1024) * sizeof(buf);
+
+		remaining = sub_tmp->length;
+
+		/* Each sub tmpfile written with 'A', 'B', etc. */
+		memset(buf, 'A' + i, sizeof(buf));
+		while (remaining) {
+			int written;
+
+			written = write(sub_tmp->fd, buf, sizeof(buf));
+			assert(written == sizeof(buf));
+			tot_size += sizeof(buf);
+			remaining -= written;
+		}
+	}
+
+	printf("total file size %d\n", tot_size);
+
+	dest_fd = open(dest_file, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
+	if (dest_fd == -1) {
+		perror("error");
+		exit(1);
+	}
+
+	// make dest file big
+	ret = posix_fallocate(dest_fd, 0, tot_size);
+	if (ret) {
+		perror("error");
+		exit(1);
+	}
+
+	// release everything in dest file
+	ioctl_args.offset = 0;
+	ioctl_args.length = tot_size;
+	ioctl_args.data_version = 0;
+
+	ret = ioctl(dest_fd, SCOUTFS_IOC_RELEASE, &ioctl_args);
+	if (ret < 0) {
+		perror("error");
+		exit(1);
+	}
+
+	// move contents into dest in reverse order
+	for (i = array_size(sub_tmps) - 1; i >= 0 ; i--) {
+		struct sub_tmp_info *sub_tmp = &sub_tmps[i];
+
+		mb.from_fd = sub_tmp->fd;
+		mb.from_off = 0;
+		mb.len = sub_tmp->length;
+		mb.to_off = sub_tmp->offset;
+		mb.data_version = 0;
+		mb.flags = SCOUTFS_IOC_MB_STAGE;
+
+		ret = ioctl(dest_fd, SCOUTFS_IOC_MOVE_BLOCKS, &mb);
+		if (ret < 0) {
+			perror("error");
+			exit(1);
+		}
+
+	}
+
+	return 0;
+}

tests/tests/block-stale-reads.sh

@@ -0,0 +1,61 @@
+#
+# Exercise stale block reading.
+#
+# It would be very difficult to manipulate the allocators, cache, and
+# persistent blocks to create stable block reading scenarios.    Instead
+# we use triggers to exercise how readers encounter stale blocks.
+#
+
+t_require_commands touch setfattr getfattr
+
+inc_wrap_fs_nr()
+{
+	local nr="$(($1 + 1))"
+
+	if [ "$nr" == "$T_NR_MOUNTS" ]; then
+		nr=0
+	fi
+
+	echo $nr
+}
+
+GETFATTR="getfattr --absolute-names"
+SETFATTR="setfattr"
+
+echo "== create shared test file"
+touch "$T_D0/file"
+$SETFATTR -n user.xat -v 0 "$T_D0/file"
+
+#
+# Trigger retries in the block cache as we bounce xattr values around
+# between sequential pairs of mounts.  This is a little silly because if
+# either of the mounts are the server then they'll almost certaily have
+# their trigger fired prematurely by message handling btree calls while
+# working with the t_ helpers long before we work with the xattrs.  But
+# the block cache stale retry path is still being exercised.
+#
+echo "== set and get xattrs between mount pairs while retrying"
+set_nr=0
+get_nr=$(inc_wrap_fs_nr $set_nr)
+
+for i in $(seq 1 10); do
+	eval set_file="\$T_D${set_nr}/file"
+	eval get_file="\$T_D${get_nr}/file"
+
+	old_set=$(t_counter block_cache_remove_stale $set_nr)
+	old_get=$(t_counter block_cache_remove_stale $get_nr)
+
+	t_trigger_arm_silent block_remove_stale $set_nr
+	t_trigger_arm_silent block_remove_stale $get_nr
+
+	$SETFATTR -n user.xat -v $i "$set_file"
+	$GETFATTR -n user.xat "$get_file" 2>&1 | t_filter_fs
+
+	t_counter_diff_changed block_cache_remove_stale $old_set $set_nr
+	t_counter_diff_changed block_cache_remove_stale $old_get $get_nr
+
+	set_nr="$get_nr"
+	get_nr=$(inc_wrap_fs_nr $set_nr)
+done
+
+t_pass

tests/tests/export-lookup-evict-race.sh

@@ -0,0 +1,32 @@
+#
+# test racing fh_to_dentry with evict from lock invalidation.   We've
+# had deadlocks between the ordering of iget and evict when they acquire
+# cluster locks.
+# 
+
+t_require_commands touch stat handle_cat
+t_require_mounts 2
+
+CPUS=$(getconf _NPROCESSORS_ONLN)
+NR=$((CPUS * 4))
+END=$((SECONDS + 30))
+
+touch "$T_D0/file"
+ino=$(stat -c "%i" "$T_D0/file")
+
+while test $SECONDS -lt $END; do
+	for i in $(seq 1 $NR); do
+		fs=$((RANDOM % T_NR_MOUNTS))
+		eval dir="\$T_D${fs}"
+		write=$((RANDOM & 1))
+
+		if [ "$write" == 1 ]; then
+			touch "$dir/file" &
+		else
+			handle_cat "$dir" "$ino" &
+		fi
+	done
+	wait
+done
+
+t_pass

tests/tests/inode-deletion.sh

@@ -0,0 +1,98 @@
+#
+# test deleting an inode once all its links and references are gone.
+#
+
+t_require_commands cat scoutfs
+t_require_mounts 2
+
+FILE="$T_D0/file"
+
+check_ino_index() {
+	local ino="$1"
+	local dseq="$2"
+	local mnt="$3"
+
+	t_sync_seq_index
+
+	scoutfs walk-inodes -p "$mnt" -- data_seq $dseq $(($dseq + 1)) |
+		awk 'BEGIN { not = "not " }
+		     ($4 == '$ino') { not = ""; exit; }
+		     END { print "ino " not "found in dseq index" }'
+}
+
+echo "== basic unlink deletes"
+echo "contents" > "$FILE"
+ino=$(stat -c "%i" "$FILE")
+dseq=$(scoutfs stat -s data_seq "$FILE")
+check_ino_index "$ino" "$dseq" "$T_M0"
+rm -f "$FILE"
+check_ino_index "$ino" "$dseq" "$T_M0"
+
+echo "== local open-unlink waits for close to delete"
+echo "contents" > "$FILE"
+ino=$(stat -c "%i" "$FILE")
+dseq=$(scoutfs stat -s data_seq "$FILE")
+exec {FD}<"$FILE"  # open unused fd, assign to FD
+rm -f "$FILE"
+echo "contents after rm: $(cat <&$FD)"
+check_ino_index "$ino" "$dseq" "$T_M0"
+exec {FD}>&-  # close
+check_ino_index "$ino" "$dseq" "$T_M0"
+
+echo "== multiple local opens are protected"
+echo "contents" > "$FILE"
+ino=$(stat -c "%i" "$FILE")
+dseq=$(scoutfs stat -s data_seq "$FILE")
+exec {FD1}<"$FILE"
+exec {FD2}<"$FILE"
+rm -f "$FILE"
+echo "contents after rm 1: $(cat <&$FD1)"
+echo "contents after rm 2: $(cat <&$FD2)"
+check_ino_index "$ino" "$dseq" "$T_M0"
+exec {FD1}>&-  # close
+exec {FD2}>&-  # close
+check_ino_index "$ino" "$dseq" "$T_M0"
+
+echo "== remote unopened unlink deletes"
+echo "contents" > "$T_D0/file"
+ino=$(stat -c "%i" "$T_D0/file")
+dseq=$(scoutfs stat -s data_seq "$T_D0/file")
+rm -f "$T_D1/file"
+check_ino_index "$ino" "$dseq" "$T_M0"
+check_ino_index "$ino" "$dseq" "$T_M1"
+
+echo "== unlink wait for open on other mount"
+echo "contents" > "$T_D0/file"
+ino=$(stat -c "%i" "$T_D0/file")
+dseq=$(scoutfs stat -s data_seq "$T_D0/file")
+exec {FD}<"$T_D0/file"
+rm -f "$T_D1/file"
+echo "mount 0 contents after mount 1 rm: $(cat <&$FD)"
+check_ino_index "$ino" "$dseq" "$T_M0"
+check_ino_index "$ino" "$dseq" "$T_M1"
+exec {FD}>&-  # close
+# we know that revalidating will unhash the remote dentry
+stat "$T_D0/file" 2>&1 | t_filter_fs
+check_ino_index "$ino" "$dseq" "$T_M0"
+check_ino_index "$ino" "$dseq" "$T_M1"
+
+echo "== lots of deletions use one open map"
+mkdir "$T_D0/dir"
+touch "$T_D0/dir"/files-{1..5}
+rm -f "$T_D0/dir"/files-*
+rmdir "$T_D0/dir"
+
+echo "== open files survive remote scanning orphans"
+echo "contents" > "$T_D0/file"
+ino=$(stat -c "%i" "$T_D0/file")
+dseq=$(scoutfs stat -s data_seq "$T_D0/file")
+exec {FD}<"$T_D0/file"
+rm -f "$T_D0/file"
+t_umount 1
+t_mount 1
+echo "mount 0 contents after mount 1 remounted: $(cat <&$FD)"
+exec {FD}>&-  # close
+check_ino_index "$ino" "$dseq" "$T_M0"
+check_ino_index "$ino" "$dseq" "$T_M1"
+
+t_pass

tests/tests/lock-conflicting-batch-commit.sh

@@ -1,59 +0,0 @@
-#
-# If bulk work accidentally conflicts in the worst way we'd like to have
-# it not result in catastrophic performance.  Make sure that each
-# instance of bulk work is given the opportunity to get as much as it
-# can into the transaction under a lock before the lock is revoked
-# and the transaction is committed.
-#
-
-t_require_commands setfattr
-t_require_mounts 2
-
-NR=3000
-
-echo "== create per mount files" 
-for m in 0 1; do 
-	eval dir="\$T_D${m}/dir/$m"
-	t_quiet mkdir -p "$dir"
-	for a in $(seq 1 $NR); do touch "$dir/$a"; done
-done
-
-echo "== time independent modification"
-for m in 0 1; do 
-	eval dir="\$T_D${m}/dir/$m"
-	START=$SECONDS
-	for a in $(seq 1 $NR); do
-		setfattr -n user.test_grace -v $a "$dir/$a"
-	done
-	echo "mount $m: $((SECONDS - START))" >> $T_TMP.log
-done
-
-echo "== time concurrent independent modification"
-START=$SECONDS
-for m in 0 1; do 
-	eval dir="\$T_D${m}/dir/$m"
-	(for a in $(seq 1 $NR); do
-		setfattr -n user.test_grace -v $a "$dir/$a"; 
-	done) &
-done
-wait
-IND="$((SECONDS - START))"
-echo "ind: $IND" >> $T_TMP.log
-
-echo "== time concurrent conflicting modification"
-START=$SECONDS
-for m in 0 1; do 
-	eval dir="\$T_D${m}/dir/0"
-	(for a in $(seq 1 $NR); do
-		setfattr -n user.test_grace -v $a "$dir/$a"; 
-	done) &
-done
-wait
-CONF="$((SECONDS - START))"
-echo "conf: $CONF" >> $T_TMP.log
-
-if [ "$CONF" -gt "$((IND * 5))" ]; then
-	t_fail "conflicting $CONF secs is more than 5x independent $IND secs"
-fi
-
-t_pass

tests/tests/mkdir-rename-rmdir.sh

@@ -0,0 +1,59 @@
+#
+# Sequentially perform operations on a dir (mkdir; rename*2; rmdir) on
+# all possible combinations of different mounts that could perform the
+# operations.
+#
+# We're testing that the tracking of the entry key in our cached dirents
+# stays consitent with the persistent entry items as they're modified
+# around the cluster.
+#
+
+t_require_commands mkdir mv rmdir
+
+NR_OPS=4
+
+unset op_mnt
+for op in $(seq 0 $NR_OPS); do
+	op_mnt[$op]=0
+done
+
+if [ $T_NR_MOUNTS -gt $NR_OPS ]; then
+	NR_MNTS=$NR_OPS
+else
+	NR_MNTS=$T_NR_MOUNTS
+fi
+
+# test until final op mount dir wraps
+while [ ${op_mnt[$NR_OPS]} == 0 ]; do
+
+	# sequentially perform each op from its mount dir
+	for op in $(seq 0 $((NR_OPS - 1))); do
+		m=${op_mnt[$op]}
+		eval dir="\$T_D${m}/dir"
+
+		case "$op" in
+			0) mkdir "$dir" ;;
+			1) mv "$dir" "$dir-1" ;;
+			2) mv "$dir-1" "$dir-2" ;;
+			3) rmdir "$dir-2" ;;
+		esac
+
+		if [ $? != 0 ]; then
+			t_fail "${op_mnt[*]} failed at op $op"
+		fi
+	done
+
+	# advance through mnt nrs for each op
+	i=0
+	while [ ${op_mnt[$NR_OPS]} == 0 ]; do
+		((op_mnt[$i]++))
+		if [ ${op_mnt[$i]} -ge $NR_MNTS ]; then
+			op_mnt[$i]=0
+			((i++))
+		else
+			break
+		fi
+	done
+done
+
+t_pass

tests/tests/mount-unmount-race.sh

@@ -4,25 +4,23 @@
 # At the start of the test all mounts are mounted.  Each iteration
 # randomly decides to change each mount or to leave it alone.
 #
-# They create dirty items before unmounting to encourage compaction
-# while unmounting
+# Each iteration create dirty items across the mounts randomly, giving
+# unmount some work to do.
 #
 # For this test to be meaningful it needs multiple mounts beyond the
-# quorum set which can be racing to mount and unmount.  A reasonable
-# config would be 5 mounts with 3 quorum.  But the test will run with
-# whatever count it finds.
+# quorum majority which can be racing to mount and unmount.  A
+# reasonable config would be 5 mounts with 3 quorum members.  But the
+# test will run with whatever count it finds.
 #
-# This assumes that all the mounts are configured as voting servers.  We
-# could update it to be more clever and know that it can always safely
-# unmount mounts that aren't configured as servers.
+# The test assumes that the first mounts are the quorum members.
 #
 
-# nothing to do if we can't unmount
-test "$T_NR_MOUNTS" == "$T_QUORUM" && \
-	t_skip "only quorum members mounted, can't unmount"
+majority_nr=$(t_majority_count)
+quorum_nr=$T_QUORUM
 
-nr_mounted=$T_NR_MOUNTS
-nr_quorum=$T_QUORUM
+cur_quorum=$quorum_nr
+test "$cur_quorum" == "$majority_nr" && \
+	t_skip "all quorum members make up majority, need more mounts to unmount"
 
 echo "== create per mount files" 
 for i in $(t_fs_nrs); do
@@ -55,25 +53,42 @@ while [ "$SECONDS" -lt "$END" ]; do
 		fi
 
 		if [ "${mounted[$i]}" == 1 ]; then
-			if [ "$nr_mounted" -gt "$nr_quorum" ]; then
+			#
+			# can always unmount non-quorum mounts,
+			# can only unmount quorum members beyond majority
+			#
+			if [ "$i" -ge "$quorum_nr" -o \
+			     "$cur_quorum" -gt "$majority_nr" ]; then
 				t_umount $i &
 				pid=$!
+				echo "umount $i pid $pid quo $cur_quorum" \
+					>> $T_TMP.log
 				pids="$pids $pid"
 				mounted[$i]=0
-				(( nr_mounted-- ))
+				if [ "$i" -lt "$quorum_nr" ]; then
+					(( cur_quorum-- ))
+				fi
 			fi
 		else
 			t_mount $i &
 			pid=$!
 			pids="$pids $pid"
+			echo "mount $i pid $pid quo $cur_quorum" >> $T_TMP.log
 			mounted[$i]=1
-			(( nr_mounted++ ))
+			if [ "$i" -lt "$quorum_nr" ]; then
+				(( cur_quorum++ ))
+			fi
 		fi
 	done
 		
 	echo "waiting (secs $SECONDS)" >> $T_TMP.log
 	for p in $pids; do
-		t_quiet wait $p
+		wait $p
+		rc=$?
+		if [ "$rc" != 0 ]; then
+			echo "waiting for pid $p returned $rc"
+			t_fail "background mount/umount returned error"
+		fi
 	done
 	echo "done waiting (secs $SECONDS))" >> $T_TMP.log
 done

tests/tests/stage-tmpfile.sh

@@ -0,0 +1,15 @@
+#
+# Run tmpfile_stage and check the output with hexdump.
+#
+
+t_require_commands stage_tmpfile hexdump
+
+DEST_FILE="$T_D0/dest_file"
+
+stage_tmpfile $T_D0 $DEST_FILE
+
+hexdump -C "$DEST_FILE"
+
+rm -fr "$DEST_FILE"
+
+t_pass

tests/tests/stale-btree-read.sh

@@ -1,40 +0,0 @@
-#
-# verify stale btree block reading
-#
-
-t_require_commands touch stat setfattr getfattr createmany
-t_require_mounts 2
-
-GETFATTR="getfattr --absolute-names"
-SETFATTR="setfattr"
-
-#
-# This exercises the soft retry of btree blocks when
-# inconsistent cached versions are found.  It ensures that basic hard
-# error returning turns into EIO in the case where the persistent reread
-# blocks and segments really are inconsistent.
-#
-# The triggers apply across all execution in the file system.  So to
-# trigger btree block retries in the client we make sure that the server
-# is running on the other node.
-# 
-
-cl=$(t_first_client_nr)
-sv=$(t_server_nr)
-eval cl_dir="\$T_D${cl}"
-eval sv_dir="\$T_D${sv}"
-
-echo "== create file for xattr ping pong"
-touch "$sv_dir/file"
-$SETFATTR -n user.xat -v initial "$sv_dir/file"
-$GETFATTR -n user.xat "$sv_dir/file" 2>&1 | t_filter_fs
-
-echo "== retry btree block read"
-$SETFATTR -n user.xat -v btree "$sv_dir/file"
-t_trigger_arm btree_stale_read $cl
-old=$(t_counter btree_stale_read $cl)
-$GETFATTR -n user.xat "$cl_dir/file" 2>&1 | t_filter_fs
-t_trigger_show btree_stale_read "after" $cl
-t_counter_diff btree_stale_read $old $cl
-
-t_pass

tests/tests/xfstests.sh

@@ -37,17 +37,25 @@ t_quiet make
 t_quiet sync
 # pwd stays in xfstests dir to build config and run
 
+#
+# Each filesystem needs specific mkfs and mount options because we put
+# quorum member addresess in mkfs options and the metadata device in
+# mount options.
+#
 cat << EOF > local.config
 export FSTYP=scoutfs
-export MKFS_OPTIONS="-Q 1"
+export MKFS_OPTIONS="-f"
+export MKFS_TEST_OPTIONS="-Q 0,127.0.0.1,42000"
+export MKFS_SCRATCH_OPTIONS="-Q 0,127.0.0.1,43000"
+export MKFS_DEV_OPTIONS="-Q 0,127.0.0.1,44000"
 export TEST_DEV=$T_DB0
 export TEST_DIR=$T_M0
 export SCRATCH_META_DEV=$T_EX_META_DEV
 export SCRATCH_DEV=$T_EX_DATA_DEV
 export SCRATCH_MNT="$T_TMPDIR/mnt.scratch"
-export SCOUTFS_SCRATCH_MOUNT_OPTIONS="-o server_addr=127.0.0.1,metadev_path=$T_EX_META_DEV"
-export MOUNT_OPTIONS="-o server_addr=127.0.0.1,metadev_path=$T_MB0"
-export TEST_FS_MOUNT_OPTS="-o server_addr=127.0.0.1,metadev_path=$T_MB0"
+export SCOUTFS_SCRATCH_MOUNT_OPTIONS="-o quorum_slot_nr=0,metadev_path=$T_EX_META_DEV"
+export MOUNT_OPTIONS="-o quorum_slot_nr=0,metadev_path=$T_MB0"
+export TEST_FS_MOUNT_OPTS="-o quorum_slot_nr=0,metadev_path=$T_MB0"
 EOF
 
 cat << EOF > local.exclude

utils/man/scoutfs.5

@@ -21,21 +21,19 @@ contains the filesystem's metadata.
 .sp
 This option is required.
 .TP
-.B server_addr=<ipv4:port>
-The server_addr option indicates that this mount will participate in
-quorum election to try and run a server for all the mounts of its
-filesystem.   The option specifies the local TCP IPv4 address that the
-mount's elected server will listen on for connections from all other
-mounts of the filesystem.
+.B quorum_slot_nr=<number>
+The quorum_slot_nr option assigns a quorum member slot to the mount.
+The mount will use the slot assignment to claim exclusive ownership of
+the slot's configured address and an associated metadata device block.
+Each slot number must be used by only one mount at any given time.
 .sp
-The IPv4 address must be specified as a dotted quad, name resolution is
-not supported.  A specific port may be provided after a seperating
-colon.  If no port is specified then a random port will be chosen.  The
-address will be used for the lifetime of the mount and can not be
-changed.  The mount must be unmounted to specify a different address.
+When a mount is assigned a quorum slot it becomes a quorum member and
+will participate in the raft leader election process and could start
+the server for the filesystem if it is elected leader.
 .sp
-If server_addr is not specified then the mount will read the filesystem
-until it sees the address of an elected server to connect to.
+The assigned number must match one of the slots defined with \-Q options
+when the filesystem was created with mkfs.  If the number assigned
+doesn't match a number created during mkfs then the mount will fail.
 .SH FURTHER READING
 A
 .B scoutfs

utils/man/scoutfs.8

@@ -32,7 +32,7 @@ A path within a ScoutFS filesystem.
 .PD
 
 .TP
-.BI "mkfs META-DEVICE DATA-DEVICE {-Q|--quorum-count} NUM [-m|--max-meta-size SIZE] [-d|--max-data-size SIZE] [-f|--force]"
+.BI "mkfs META-DEVICE DATA-DEVICE {-Q|--quorum-slot} NR,ADDR,PORT [-m|--max-meta-size SIZE] [-d|--max-data-size SIZE] [-f|--force]"
 .sp
 Initialize a new ScoutFS filesystem on the target devices. Since ScoutFS uses
 separate block devices for its metadata and data storage, two are required.
@@ -57,18 +57,20 @@ a faster block device for the metadata device.
 The path to the block device to be used for ScoutFS file data.  If possible, use
 a larger block device for the data device.
 .TP
-.B "-Q, --quorum-count NUM"
-The number of mounts needed to reach quorum and elect one
-to be the server.  Mounts of the filesystem will hang until a quorum of
-mounts are operational.
-.sp
-Mounts with the
-.B server_addr
-mount option participate in quorum.  The safest quorum number is the
-smallest majority of an odd number of participating mounts.  For
-example,
-two out of three total mounts.  This ensures that there can only be one
-set of mounts that can establish quorum.
+.B "-Q, --quorum-slot NR,ADDR,PORT"
+Each \-Q option configures a quorum slot.  The NR specifies the number
+of the slot to configure which must be between 0 and 14.  Each slot
+number must only be used once, but they can be used in any order and
+they need not be consecutive.  This is to allow natural relationships
+between slot numbers and nodes which may have arbitrary numbering
+schemes.  ADDR and PORT are the numerical IPv4 address and port which
+will be used as the UDP endpoint for leader elections and as the TCP
+listening address for server connections.  The number of configured
+slots determines the size of the quorum of member mounts which must be
+present to start the server for the filesystem to operate.  A simple
+majority is typically required, while one mount is sufficient if only
+one or two slots are configured.  Until the majority quorum are present,
+all mounts will hang waiting for a server to connect to.
 .TP
 .B "-m, --max-meta-size SIZE"
 Limit the space used by ScoutFS on the metadata device to the

utils/src/btree.c

@@ -25,17 +25,13 @@ static void init_block(struct scoutfs_btree_block *bt, int level)
  */
 void btree_init_root_single(struct scoutfs_btree_root *root,
 			    struct scoutfs_btree_block *bt,
-			    u64 blkno, u64 seq, __le64 fsid)
+			    u64 seq, u64 blkno)
 {
 	root->ref.blkno = cpu_to_le64(blkno);
-	root->ref.seq = cpu_to_le64(1);
+	root->ref.seq = cpu_to_le64(seq);
 	root->height = 1;
 
 	memset(bt, 0, SCOUTFS_BLOCK_LG_SIZE);
-	bt->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_BTREE);
-	bt->hdr.fsid = fsid;
-	bt->hdr.blkno = cpu_to_le64(blkno);
-	bt->hdr.seq = cpu_to_le64(1);
 
 	init_block(bt, 0);
 }

utils/src/btree.h

@@ -3,7 +3,7 @@
 
 void btree_init_root_single(struct scoutfs_btree_root *root,
 			    struct scoutfs_btree_block *bt,
-			    u64 blkno, u64 seq, __le64 fsid);
+			    u64 seq, u64 blkno);
 
 void btree_append_item(struct scoutfs_btree_block *bt,
 		       struct scoutfs_key *key, void *val, int val_len);

utils/src/mkfs.c

@@ -32,12 +32,22 @@
 #include "leaf_item_hash.h"
 #include "blkid.h"
 
-static int write_raw_block(int fd, u64 blkno, int shift, void *blk)
+/*
+ * Update the block header fields and write out the block.
+ */
+static int write_block(int fd, u32 magic, __le64 fsid, u64 seq, u64 blkno,
+		       int shift, struct scoutfs_block_header *hdr)
 {
 	size_t size = 1ULL << shift;
 	ssize_t ret;
 
-	ret = pwrite(fd, blk, size, blkno << shift);
+	hdr->magic = cpu_to_le32(magic);
+	hdr->fsid = fsid;
+	hdr->blkno = cpu_to_le64(blkno);
+	hdr->seq = cpu_to_le64(seq);
+	hdr->crc = cpu_to_le32(crc_block(hdr, size));
+
+	ret = pwrite(fd, hdr, size, blkno << shift);
 	if (ret != size) {
 		fprintf(stderr, "write to blkno %llu returned %zd: %s (%d)\n",
 			blkno, ret, strerror(errno), errno);
@@ -47,35 +57,18 @@ static int write_raw_block(int fd, u64 blkno, int shift, void *blk)
 	return 0;
 }
 
-/*
- * Update the block's header and write it out.
- */
-static int write_block(int fd, u64 blkno, int shift,
-		       struct scoutfs_super_block *super,
-		       struct scoutfs_block_header *hdr)
-{
-	size_t size = 1ULL << shift;
-
-	if (super)
-		*hdr = super->hdr;
-	hdr->blkno = cpu_to_le64(blkno);
-	hdr->crc = cpu_to_le32(crc_block(hdr, size));
-
-	return write_raw_block(fd, blkno, shift, hdr);
-}
-
 /*
  * Write the single btree block that contains the blkno and len indexed
  * items to store the given extent, and update the root to point to it.
  */
-static int write_alloc_root(struct scoutfs_super_block *super, int fd,
+static int write_alloc_root(int fd, __le64 fsid,
 			    struct scoutfs_alloc_root *root,
 			    struct scoutfs_btree_block *bt,
-			    u64 blkno, u64 start, u64 len)
+			    u64 seq, u64 blkno, u64 start, u64 len)
 {
 	struct scoutfs_key key;
 
-	btree_init_root_single(&root->root, bt, blkno, 1, super->hdr.fsid);
+	btree_init_root_single(&root->root, bt, seq, blkno);
 	root->total_len = cpu_to_le64(len);
 
 	memset(&key, 0, sizeof(key));
@@ -94,19 +87,18 @@ static int write_alloc_root(struct scoutfs_super_block *super, int fd,
 	key.skfl_blkno = cpu_to_le64(start);
 	btree_append_item(bt, &key, NULL, 0);
 
-	bt->hdr.crc = cpu_to_le32(crc_block(&bt->hdr,
-					    SCOUTFS_BLOCK_LG_SIZE));
-
-	return write_raw_block(fd, blkno, SCOUTFS_BLOCK_LG_SHIFT, bt);
+	return write_block(fd, SCOUTFS_BLOCK_MAGIC_BTREE, fsid, seq, blkno,
+			   SCOUTFS_BLOCK_LG_SHIFT, &bt->hdr);
 }
 
 struct mkfs_args {
-	unsigned long long quorum_count;
 	char *meta_device;
 	char *data_device;
 	unsigned long long max_meta_size;
 	unsigned long long max_data_size;
 	bool force;
+	int nr_slots;
+	struct scoutfs_quorum_slot slots[SCOUTFS_QUORUM_MAX_SLOTS];
 };
 
 /*
@@ -124,12 +116,14 @@ static int do_mkfs(struct mkfs_args *args)
 	struct scoutfs_inode inode;
 	struct scoutfs_alloc_list_block *lblk;
 	struct scoutfs_btree_block *bt = NULL;
+	struct scoutfs_block_header *hdr;
 	struct scoutfs_key key;
 	struct timeval tv;
 	int meta_fd = -1;
 	int data_fd = -1;
 	char uuid_str[37];
 	void *zeros = NULL;
+	char *indent;
 	u64 blkno;
 	u64 meta_size;
 	u64 data_size;
@@ -139,10 +133,12 @@ static int do_mkfs(struct mkfs_args *args)
 	u64 last_data;
 	u64 meta_start;
 	u64 meta_len;
+	__le64 fsid;
 	int ret;
 	int i;
 
 	gettimeofday(&tv, NULL);
+	pseudo_random_bytes(&fsid, sizeof(fsid));
 
 	meta_fd = open(args->meta_device, O_RDWR | O_EXCL);
 	if (meta_fd < 0) {
@@ -191,10 +187,7 @@ static int do_mkfs(struct mkfs_args *args)
 	if (ret)
 		goto out;
 
-	/* metadata blocks start after the quorum blocks */
-	next_meta = (SCOUTFS_QUORUM_BLKNO + SCOUTFS_QUORUM_BLOCKS) >>
-		    SCOUTFS_BLOCK_SM_LG_SHIFT;
-	/* rest of meta dev is available for metadata blocks */
+	next_meta = SCOUTFS_META_DEV_START_BLKNO;
 	last_meta = (meta_size >> SCOUTFS_BLOCK_LG_SHIFT) - 1;
 	/* Data blocks go on the data dev */
 	first_data = SCOUTFS_DATA_DEV_START_BLKNO;
@@ -202,12 +195,9 @@ static int do_mkfs(struct mkfs_args *args)
 
 	/* partially initialize the super so we can use it to init others */
 	memset(super, 0, SCOUTFS_BLOCK_SM_SIZE);
-	pseudo_random_bytes(&super->hdr.fsid, sizeof(super->hdr.fsid));
-	super->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SUPER);
-	super->hdr.seq = cpu_to_le64(1);
 	super->version = cpu_to_le64(SCOUTFS_INTEROP_VERSION);
 	uuid_generate(super->uuid);
-	super->next_ino = cpu_to_le64(SCOUTFS_ROOT_INO + 1);
+	super->next_ino = cpu_to_le64(round_up(SCOUTFS_ROOT_INO + 1, SCOUTFS_LOCK_INODE_GROUP_NR));
 	super->next_trans_seq = cpu_to_le64(1);
 	super->total_meta_blocks = cpu_to_le64(last_meta + 1);
 	super->first_meta_blkno = cpu_to_le64(next_meta);
@@ -215,11 +205,14 @@ static int do_mkfs(struct mkfs_args *args)
 	super->total_data_blocks = cpu_to_le64(last_data - first_data + 1);
 	super->first_data_blkno = cpu_to_le64(first_data);
 	super->last_data_blkno = cpu_to_le64(last_data);
-	super->quorum_count = args->quorum_count;
+
+	assert(sizeof(args->slots) ==
+		     member_sizeof(struct scoutfs_super_block, qconf.slots));
+	memcpy(super->qconf.slots, args->slots, sizeof(args->slots));
 
 	/* fs root starts with root inode and its index items */
 	blkno = next_meta++;
-	btree_init_root_single(&super->fs_root, bt, blkno, 1, super->hdr.fsid);
+	btree_init_root_single(&super->fs_root, bt, 1, blkno);
 
 	memset(&key, 0, sizeof(key));
 	key.sk_zone = SCOUTFS_INODE_INDEX_ZONE;
@@ -244,10 +237,8 @@ static int do_mkfs(struct mkfs_args *args)
 	inode.mtime.nsec = inode.atime.nsec;
 	btree_append_item(bt, &key, &inode, sizeof(inode));
 
-	bt->hdr.crc = cpu_to_le32(crc_block(&bt->hdr,
-					    SCOUTFS_BLOCK_LG_SIZE));
-
-	ret = write_raw_block(meta_fd, blkno, SCOUTFS_BLOCK_LG_SHIFT, bt);
+	ret = write_block(meta_fd, SCOUTFS_BLOCK_MAGIC_BTREE, fsid, 1, blkno,
+			  SCOUTFS_BLOCK_LG_SHIFT, &bt->hdr);
 	if (ret)
 		goto out;
 
@@ -256,11 +247,6 @@ static int do_mkfs(struct mkfs_args *args)
 	lblk = (void *)bt;
 	memset(lblk, 0, SCOUTFS_BLOCK_LG_SIZE);
 
-	lblk->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_ALLOC_LIST);
-	lblk->hdr.fsid = super->hdr.fsid;
-	lblk->hdr.blkno = cpu_to_le64(blkno);
-	lblk->hdr.seq = cpu_to_le64(1);
-
 	meta_len = (64 * 1024 * 1024) >> SCOUTFS_BLOCK_LG_SHIFT;
 	for (i = 0; i < meta_len; i++) {
 		lblk->blknos[i] = cpu_to_le64(next_meta);
@@ -268,20 +254,20 @@ static int do_mkfs(struct mkfs_args *args)
 	}
 	lblk->nr = cpu_to_le32(i);
 
-	super->server_meta_avail[0].ref.blkno = lblk->hdr.blkno;
-	super->server_meta_avail[0].ref.seq = lblk->hdr.seq;
+	super->server_meta_avail[0].ref.blkno = cpu_to_le64(blkno);
+	super->server_meta_avail[0].ref.seq = cpu_to_le64(1);
 	super->server_meta_avail[0].total_nr = le32_to_le64(lblk->nr);
 	super->server_meta_avail[0].first_nr = lblk->nr;
 
-	lblk->hdr.crc = cpu_to_le32(crc_block(&bt->hdr, SCOUTFS_BLOCK_LG_SIZE));
-	ret = write_raw_block(meta_fd, blkno, SCOUTFS_BLOCK_LG_SHIFT, lblk);
+	ret = write_block(meta_fd, SCOUTFS_BLOCK_MAGIC_ALLOC_LIST, fsid, 1,
+			  blkno, SCOUTFS_BLOCK_LG_SHIFT, &lblk->hdr);
 	if (ret)
 		goto out;
 
 	/* the data allocator has a single extent */
 	blkno = next_meta++;
-	ret = write_alloc_root(super, meta_fd, &super->data_alloc, bt,
-			       blkno, first_data,
+	ret = write_alloc_root(meta_fd, fsid, &super->data_alloc, bt,
+			       1, blkno, first_data,
 			       le64_to_cpu(super->total_data_blocks));
 	if (ret < 0)
 		goto out;
@@ -298,8 +284,8 @@ static int do_mkfs(struct mkfs_args *args)
 	/* each meta alloc root contains a portion of free metadata extents */
 	for (i = 0; i < array_size(super->meta_alloc); i++) {
 		blkno = next_meta++;
-		ret = write_alloc_root(super, meta_fd, &super->meta_alloc[i], bt,
-				       blkno, meta_start,
+		ret = write_alloc_root(meta_fd, fsid, &super->meta_alloc[i], bt,
+				       1, blkno, meta_start,
 				       min(meta_len,
 					   last_meta - meta_start + 1));
 		if (ret < 0)
@@ -309,9 +295,11 @@ static int do_mkfs(struct mkfs_args *args)
 	}
 
 	/* zero out quorum blocks */
+	hdr = zeros;
 	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
-		ret = write_raw_block(meta_fd, SCOUTFS_QUORUM_BLKNO + i,
-				      SCOUTFS_BLOCK_SM_SHIFT, zeros);
+		ret = write_block(meta_fd, SCOUTFS_BLOCK_MAGIC_QUORUM, fsid,
+				  1, SCOUTFS_QUORUM_BLKNO + i,
+				  SCOUTFS_BLOCK_SM_SHIFT, hdr);
 		if (ret < 0) {
 			fprintf(stderr, "error zeroing quorum block: %s (%d)\n",
 				strerror(-errno), -errno);
@@ -320,9 +308,9 @@ static int do_mkfs(struct mkfs_args *args)
 	}
 
 	/* write the super block to data dev and meta dev*/
-	super->hdr.seq = cpu_to_le64(1);
-	ret = write_block(data_fd, SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT,
-			  NULL, &super->hdr);
+	ret = write_block(data_fd, SCOUTFS_BLOCK_MAGIC_SUPER, fsid, 1,
+			  SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT,
+			  &super->hdr);
 	if (ret)
 		goto out;
 
@@ -334,8 +322,9 @@ static int do_mkfs(struct mkfs_args *args)
 	}
 
 	super->flags |= cpu_to_le64(SCOUTFS_FLAG_IS_META_BDEV);
-	ret = write_block(meta_fd, SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT,
-			  NULL, &super->hdr);
+	ret = write_block(meta_fd, SCOUTFS_BLOCK_MAGIC_SUPER, fsid,
+			  1, SCOUTFS_SUPER_BLKNO, SCOUTFS_BLOCK_SM_SHIFT,
+			  &super->hdr);
 	if (ret)
 		goto out;
 
@@ -356,7 +345,7 @@ static int do_mkfs(struct mkfs_args *args)
 	       "  uuid:                 %s\n"
 	       "  64KB metadata blocks: "SIZE_FMT"\n"
 	       "  4KB data blocks:      "SIZE_FMT"\n"
-	       "  quorum count:         %u\n",
+	       "  quorum slots:         ",
 		args->meta_device,
 	        args->data_device,
 		le64_to_cpu(super->hdr.fsid),
@@ -365,8 +354,22 @@ static int do_mkfs(struct mkfs_args *args)
 		SIZE_ARGS(le64_to_cpu(super->total_meta_blocks),
 			  SCOUTFS_BLOCK_LG_SIZE),
 		SIZE_ARGS(le64_to_cpu(super->total_data_blocks),
-			  SCOUTFS_BLOCK_SM_SIZE),
-		super->quorum_count);
+			  SCOUTFS_BLOCK_SM_SIZE));
+
+	indent = "";
+	for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
+		struct scoutfs_quorum_slot *sl = &super->qconf.slots[i];
+		struct in_addr in;
+
+		if (sl->addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4))
+			continue;
+
+		in.s_addr = htonl(le32_to_cpu(sl->addr.v4.addr));
+		printf("%s%u: %s:%u", indent,
+		       i, inet_ntoa(in), le16_to_cpu(sl->addr.v4.port));
+		indent = "\n                        ";
+	}
+	printf("\n");
 
 	ret = 0;
 out:
@@ -383,16 +386,61 @@ out:
 	return ret;
 }
 
+static bool valid_quorum_slots(struct scoutfs_quorum_slot *slots)
+{
+	struct in_addr in;
+	bool valid = true;
+	char *addr;
+	int i;
+	int j;
+
+	for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) {
+		if (slots[i].addr.v4.family == cpu_to_le16(SCOUTFS_AF_NONE))
+			continue;
+
+		if (slots[i].addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4)) {
+			fprintf(stderr, "quorum slot nr %u has invalid family %u\n",
+				i, le16_to_cpu(slots[i].addr.v4.family));
+			valid = false;
+		}
+
+		for (j = i + 1; j < SCOUTFS_QUORUM_MAX_SLOTS; j++) {
+			if (slots[i].addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4))
+				continue;
+
+			if (slots[i].addr.v4.addr == slots[j].addr.v4.addr &&
+			    slots[i].addr.v4.port == slots[j].addr.v4.port) {
+
+				in.s_addr =
+					htonl(le32_to_cpu(slots[i].addr.v4.addr));
+				addr = inet_ntoa(in);
+				fprintf(stderr, "quorum slot nr %u and %u have the same address %s:%u\n",
+					i, j, addr,
+					le16_to_cpu(slots[i].addr.v4.port));
+				valid = false;
+			}
+		}
+	}
+
+	return valid;
+}
+
 static int parse_opt(int key, char *arg, struct argp_state *state)
 {
 	struct mkfs_args *args = state->input;
+	struct scoutfs_quorum_slot slot;
 	int ret;
 
 	switch (key) {
 	case 'Q':
-		ret = parse_u64(arg, &args->quorum_count);
-		if (ret)
+		ret = parse_quorum_slot(&slot, arg);
+		if (ret < 0)
 			return ret;
+		if (args->slots[ret].addr.v4.family != cpu_to_le16(SCOUTFS_AF_NONE))
+			argp_error(state, "Quorum slot %u already specified before slot '%s'\n",
+				   ret, arg);
+		args->slots[ret] = slot;
+		args->nr_slots++;
 		break;
 	case 'f':
 		args->force = true;
@@ -432,12 +480,14 @@ static int parse_opt(int key, char *arg, struct argp_state *state)
 			argp_error(state, "more than two arguments given");
 		break;
 	case ARGP_KEY_FINI:
-		if (!args->quorum_count)
-			argp_error(state, "must provide nonzero quorum count with --quorum-count|-Q option");
+		if (!args->nr_slots)
+			argp_error(state, "must specify at least one quorum slot with --quorum-count|-Q");
 		if (!args->meta_device)
 			argp_error(state, "no metadata device argument given");
 		if (!args->data_device)
 			argp_error(state, "no data device argument given");
+		if (!valid_quorum_slots(args->slots))
+			argp_error(state, "invalid quorum slot configuration");
 		break;
 	default:
 		break;
@@ -447,7 +497,7 @@ static int parse_opt(int key, char *arg, struct argp_state *state)
 }
 
 static struct argp_option options[] = {
-	{ "quorum-count", 'Q', "NUM", 0, "Number of voters required to use the filesystem [Required]"},
+	{ "quorum-slot", 'Q', "NR,ADDR,PORT", 0, "Specify quorum slot addresses [Required]"},
 	{ "force", 'f', NULL, 0, "Overwrite existing data on block devices"},
 	{ "max-meta-size", 'm', "SIZE", 0, "Use a size less than the base metadata device size (bytes or KMGTP units)"},
 	{ "max-data-size", 'd', "SIZE", 0, "Use a size less than the base data device size (bytes or KMGTP units)"},
@@ -463,7 +513,7 @@ static struct argp argp = {
 
 static int mkfs_cmd(int argc, char *argv[])
 {
-	struct mkfs_args mkfs_args = {0};
+	struct mkfs_args mkfs_args = {NULL,};
 	int ret;
 
 	ret = argp_parse(&argp, argc, argv, 0, NULL, &mkfs_args);

utils/src/move_blocks.c

@@ -32,7 +32,7 @@ struct move_blocks_args {
 
 static int do_move_blocks(struct move_blocks_args *args)
 {
-	struct scoutfs_ioctl_move_blocks mb;
+	struct scoutfs_ioctl_move_blocks mb = {0};
 	int from_fd = -1;
 	int to_fd = -1;
 	int ret;

utils/src/parse.c

@@ -3,6 +3,9 @@
 #include <stdlib.h>
 #include <limits.h>
 #include <stdio.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
 
 #include "sparse.h"
 #include "util.h"
@@ -152,3 +155,66 @@ int parse_timespec(char *str, struct timespec *ts)
 
 	return 0;
 }
+
+/*
+ * Parse a quorum slot specification string "NR,ADDR,PORT" into its
+ * component parts.  We use sscanf to both parse the leading NR and
+ * trailing PORT integers, and to pull out the inner ADDR string which
+ * is then parsed to make sure that it's a valid unicast ipv4 address.
+ * We require that all components be specified, and sccanf will check
+ * this by the number of matches it returns.
+ */
+int parse_quorum_slot(struct scoutfs_quorum_slot *slot, char *arg)
+{
+#define ADDR_CHARS 45 /* max ipv6 */
+	char addr[ADDR_CHARS + 1] = {'\0',};
+	struct in_addr in;
+	int port;
+	int parsed;
+	int nr;
+	int ret;
+
+	/* leading and trailing ints, an inner sized string without ,, all separated by , */
+	ret = sscanf(arg, "%u,%"__stringify(ADDR_CHARS)"[^,],%u%n",
+		     &nr, addr, &port, &parsed);
+	if (ret == EOF) {
+		printf("error parsing quorum slot '%s': %s\n",
+			arg, strerror(errno));
+		return -EINVAL;
+	}
+
+	if (parsed != strlen(arg)) {
+		printf("extra unparsed trailing characters in quorum slot '%s'\n",
+			arg);
+		return -EINVAL;
+	}
+
+	if (ret != 3) {
+		printf("failed to parse all three NR,ADDR,PORT tokens in quorum slot '%s'\n", arg);
+		return -EINVAL;
+	}
+
+	if (nr < 0 || nr >= SCOUTFS_QUORUM_MAX_SLOTS) {
+		printf("invalid nr '%d' in quorum slot '%s', must be between 0 and %u\n",
+		       nr, arg, SCOUTFS_QUORUM_MAX_SLOTS - 1);
+		return -EINVAL;
+	}
+
+	if (port <= 0 || port > USHRT_MAX) {
+		printf("invalid ipv4 port '%u' in quorum slot '%s', must be between 1 and %u\n",
+		       port, arg, USHRT_MAX);
+		return -EINVAL;
+	}
+
+	if (inet_aton(addr, &in) == 0 || htonl(in.s_addr) == 0 ||
+	    htonl(in.s_addr) == UINT_MAX) {
+		printf("invalid ipv4 address '%s' in quorum slot '%s'\n",
+		       addr, arg);
+		return -EINVAL;
+	}
+
+	slot->addr.v4.family = cpu_to_le16(SCOUTFS_AF_IPV4);
+	slot->addr.v4.addr = cpu_to_le32(htonl(in.s_addr));
+	slot->addr.v4.port = cpu_to_le16(port);
+	return nr;
+}

utils/src/parse.h

@@ -4,11 +4,14 @@
 #include <sys/time.h>
 #include <argp.h>
 
+struct scoutfs_quorum_slot;
+
 int parse_human(char* str, u64 *val_ret);
 int parse_u64(char *str, u64 *val_ret);
 int parse_s64(char *str, s64 *val_ret);
 int parse_u32(char *str, u32 *val_ret);
 int parse_timespec(char *str, struct timespec *ts);
+int parse_quorum_slot(struct scoutfs_quorum_slot *slot, char *arg);
 
 static inline char* strdup_or_error(const struct argp_state *state, char *str)
 {

utils/src/print.c

@@ -339,14 +339,6 @@ static int print_srch_root_item(struct scoutfs_key *key, void *val,
 	return 0;
 }
 
-static int print_lock_clients_entry(struct scoutfs_key *key, void *val,
-				    unsigned val_len, void *arg)
-{
-	printf("    rid %016llx\n", le64_to_cpu(key->sklc_rid));
-
-	return 0;
-}
-
 static int print_trans_seqs_entry(struct scoutfs_key *key, void *val,
 				  unsigned val_len, void *arg)
 {
@@ -388,10 +380,10 @@ static int print_alloc_item(struct scoutfs_key *key, void *val,
 typedef int (*print_item_func)(struct scoutfs_key *key, void *val,
 			       unsigned val_len, void *arg);
 
-static int print_btree_ref(struct scoutfs_key *key, void *val,
+static int print_block_ref(struct scoutfs_key *key, void *val,
 			   unsigned val_len, print_item_func func, void *arg)
 {
-	struct scoutfs_btree_ref *ref = val;
+	struct scoutfs_block_ref *ref = val;
 
 	func(key, NULL, 0, arg);
 	printf("    ref blkno %llu seq %llu\n",
@@ -433,7 +425,7 @@ static void print_leaf_item_hash(struct scoutfs_btree_block *bt)
 }
 
 static int print_btree_block(int fd, struct scoutfs_super_block *super,
-			     char *which, struct scoutfs_btree_ref *ref,
+			     char *which, struct scoutfs_block_ref *ref,
 			     print_item_func func, void *arg, u8 level)
 {
 	struct scoutfs_btree_item *item;
@@ -500,7 +492,7 @@ static int print_btree_block(int fd, struct scoutfs_super_block *super,
 			val_len);
 
 		if (level)
-			print_btree_ref(key, val, val_len, func, arg);
+			print_block_ref(key, val, val_len, func, arg);
 		else
 			func(key, val, val_len, arg);
 	}
@@ -531,11 +523,10 @@ static int print_btree(int fd, struct scoutfs_super_block *super, char *which,
 	return ret;
 }
 
-static int print_alloc_list_block(int fd, char *str,
-				  struct scoutfs_alloc_list_ref *ref)
+static int print_alloc_list_block(int fd, char *str, struct scoutfs_block_ref *ref)
 {
 	struct scoutfs_alloc_list_block *lblk;
-	struct scoutfs_alloc_list_ref next;
+	struct scoutfs_block_ref next;
 	u64 blkno;
 	u64 start;
 	u64 len;
@@ -583,7 +574,7 @@ static int print_alloc_list_block(int fd, char *str,
 	return print_alloc_list_block(fd, str, &next);
 }
 
-static int print_srch_block(int fd, struct scoutfs_srch_ref *ref, int level)
+static int print_srch_block(int fd, struct scoutfs_block_ref *ref, int level)
 {
 	struct scoutfs_srch_parent *srp;
 	struct scoutfs_srch_block *srb;
@@ -729,7 +720,7 @@ static int print_srch_root_files(struct scoutfs_key *key, void *val,
 }
 
 static int print_btree_leaf_items(int fd, struct scoutfs_super_block *super,
-				  struct scoutfs_btree_ref *ref,
+				  struct scoutfs_block_ref *ref,
 				  print_item_func func, void *arg)
 {
 	struct scoutfs_btree_item *item;
@@ -770,7 +761,7 @@ static int print_btree_leaf_items(int fd, struct scoutfs_super_block *super,
 	return 0;
 }
 
-static char *alloc_addr_str(struct scoutfs_inet_addr *ia)
+static char *alloc_addr_str(union scoutfs_inet_addr *ia)
 {
 	struct in_addr addr;
 	char *quad;
@@ -778,12 +769,12 @@ static char *alloc_addr_str(struct scoutfs_inet_addr *ia)
 	int len;
 
 	memset(&addr, 0, sizeof(addr));
-	addr.s_addr = htonl(le32_to_cpu(ia->addr));
+	addr.s_addr = htonl(le32_to_cpu(ia->v4.addr));
 	quad = inet_ntoa(addr);
 	if (quad == NULL)
 		return NULL;
 
-	len = snprintf(NULL, 0, "%s:%u", quad, le16_to_cpu(ia->port));
+	len = snprintf(NULL, 0, "%s:%u", quad, le16_to_cpu(ia->v4.port));
 	if (len < 1 || len > 22)
 		return NULL;
 
@@ -792,18 +783,29 @@ static char *alloc_addr_str(struct scoutfs_inet_addr *ia)
 	if (!str)
 		return NULL;
 
-	snprintf(str, len, "%s:%u", quad, le16_to_cpu(ia->port));
+	snprintf(str, len, "%s:%u", quad, le16_to_cpu(ia->v4.port));
 	return str;
 }
 
+#define OFF_NAME(x) \
+	{ offsetof(struct scoutfs_quorum_block, x), __stringify_1(x) }
+
 static int print_quorum_blocks(int fd, struct scoutfs_super_block *super)
 {
+	struct print_events {
+		size_t offset;
+		char *name;
+	} events[] = {
+		OFF_NAME(write), OFF_NAME(update_term), OFF_NAME(set_leader),
+		OFF_NAME(clear_leader), OFF_NAME(fenced),
+	};
 	struct scoutfs_quorum_block *blk = NULL;
+	struct scoutfs_quorum_block_event *ev;
 	char *log_addr = NULL;
 	u64 blkno;
 	int ret;
 	int i;
-	int j;
+	int e;
 
 	for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) {
 		blkno = SCOUTFS_QUORUM_BLKNO + i;
@@ -812,31 +814,21 @@ static int print_quorum_blocks(int fd, struct scoutfs_super_block *super)
 		if (ret)
 			goto out;
 
-		if (blk->voter_rid != 0) {
-			printf("quorum block blkno %llu\n"
-			       "  fsid %llx blkno %llu crc 0x%08x\n"
-			       "  term %llu write_nr %llu voter_rid %016llx "
-			       "vote_for_rid %016llx\n"
-			       "  log_nr %u\n",
-			       blkno, le64_to_cpu(blk->fsid),
-			       le64_to_cpu(blk->blkno), le32_to_cpu(blk->crc),
-			       le64_to_cpu(blk->term),
-			       le64_to_cpu(blk->write_nr),
-			       le64_to_cpu(blk->voter_rid),
-			       le64_to_cpu(blk->vote_for_rid),
-			       blk->log_nr);
-			for (j = 0; j < blk->log_nr; j++) {
-				free(log_addr);
-				log_addr = alloc_addr_str(&blk->log[j].addr);
-				if (!log_addr) {
-					ret = -ENOMEM;
-					goto out;
-				}
-				printf("  [%u]: term %llu rid %llu addr %s\n",
-					j, le64_to_cpu(blk->log[j].term),
-					le64_to_cpu(blk->log[j].rid),
-					log_addr);
-			}
+		printf("quorum blkno %llu (slot %llu)\n",
+		       blkno, blkno - SCOUTFS_QUORUM_BLKNO);
+		print_block_header(&blk->hdr, SCOUTFS_BLOCK_SM_SIZE);
+		printf("  term %llu random_write_mark 0x%llx flags 0x%llx\n",
+		       le64_to_cpu(blk->term),
+		       le64_to_cpu(blk->random_write_mark),
+		       le64_to_cpu(blk->flags));
+
+		for (e = 0; e < array_size(events); e++) {
+			ev = (void *)blk + events[e].offset;
+
+			printf("  %12s: rid %016llx ts %llu.%08u\n",
+			       events[e].name, le64_to_cpu(ev->rid),
+			       le64_to_cpu(ev->ts.sec),
+			       le32_to_cpu(ev->ts.nsec));
 		}
 	}
 
@@ -850,7 +842,8 @@ out:
 static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
 {
 	char uuid_str[37];
-	char *server_addr;
+	char *addr;
+	int i;
 
 	uuid_unparse(super->uuid, uuid_str);
 
@@ -864,16 +857,10 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
 	       le64_to_cpu(super->version), uuid_str);
 	printf("  flags: 0x%016llx\n", le64_to_cpu(super->flags));
 
-	server_addr = alloc_addr_str(&super->server_addr);
-	if (!server_addr)
-		return;
-
 	/* XXX these are all in a crazy order */
 	printf("  next_ino %llu next_trans_seq %llu\n"
 	       "  total_meta_blocks %llu first_meta_blkno %llu last_meta_blkno %llu\n"
 	       "  total_data_blocks %llu first_data_blkno %llu last_data_blkno %llu\n"
-	       "  quorum_fenced_term %llu quorum_server_term %llu unmount_barrier %llu\n"
-	       "  quorum_count %u server_addr %s\n"
 	       "  meta_alloc[0]: "ALCROOT_F"\n"
 	       "  meta_alloc[1]: "ALCROOT_F"\n"
 	       "  data_alloc: "ALCROOT_F"\n"
@@ -881,7 +868,6 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
 	       "  server_meta_avail[1]: "AL_HEAD_F"\n"
 	       "  server_meta_freed[0]: "AL_HEAD_F"\n"
 	       "  server_meta_freed[1]: "AL_HEAD_F"\n"
-	       "  lock_clients root: height %u blkno %llu seq %llu\n"
 	       "  mounted_clients root: height %u blkno %llu seq %llu\n"
 	       "  srch_root root: height %u blkno %llu seq %llu\n"
 	       "  trans_seqs root: height %u blkno %llu seq %llu\n"
@@ -894,11 +880,6 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
 		le64_to_cpu(super->total_data_blocks),
 		le64_to_cpu(super->first_data_blkno),
 		le64_to_cpu(super->last_data_blkno),
-		le64_to_cpu(super->quorum_fenced_term),
-		le64_to_cpu(super->quorum_server_term),
-		le64_to_cpu(super->unmount_barrier),
-		super->quorum_count,
-		server_addr,
 		ALCROOT_A(&super->meta_alloc[0]),
 		ALCROOT_A(&super->meta_alloc[1]),
 		ALCROOT_A(&super->data_alloc),
@@ -906,9 +887,6 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
 		AL_HEAD_A(&super->server_meta_avail[1]),
 		AL_HEAD_A(&super->server_meta_freed[0]),
 		AL_HEAD_A(&super->server_meta_freed[1]),
-		super->lock_clients.height,
-		le64_to_cpu(super->lock_clients.ref.blkno),
-		le64_to_cpu(super->lock_clients.ref.seq),
 		super->mounted_clients.height,
 		le64_to_cpu(super->mounted_clients.ref.blkno),
 		le64_to_cpu(super->mounted_clients.ref.seq),
@@ -922,7 +900,18 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
 		le64_to_cpu(super->fs_root.ref.blkno),
 		le64_to_cpu(super->fs_root.ref.seq));
 
-	free(server_addr);
+	printf("  quorum config version %llu\n",
+		le64_to_cpu(super->qconf.version));
+	for (i = 0; i < array_size(super->qconf.slots); i++) {
+		if (super->qconf.slots[i].addr.v4.family != cpu_to_le16(SCOUTFS_AF_IPV4))
+			continue;
+
+		addr = alloc_addr_str(&super->qconf.slots[i].addr);
+		if (addr) {
+			printf("    quorum slot %2u: %s\n", i, addr);
+			free(addr);
+		}
+	}
 }
 
 struct print_args {
@@ -946,11 +935,6 @@ static int print_volume(int fd)
 
 	ret = print_quorum_blocks(fd, super);
 
-	err = print_btree(fd, super, "lock_clients", &super->lock_clients,
-			  print_lock_clients_entry, NULL);
-	if (err && !ret)
-		ret = err;
-
 	err = print_btree(fd, super, "mounted_clients", &super->mounted_clients,
 			  print_mounted_client_entry, NULL);
 	if (err && !ret)

utils/src/util.h

@@ -114,4 +114,7 @@ static inline int memcmp_lens(const void *a, int a_len,
 int get_path(char *path, int flags);
 int read_block(int fd, u64 blkno, int shift, void **ret_val);
 
+#define __stringify_1(x) #x
+#define __stringify(x) __stringify_1(x)
+
 #endif