Use full radix for buddy and record first set

The first pass of the buddy allocator had a fixed indirect block so it
couldn't address large devices.  It didn't index set bits or slots for
each order so we spent a lot of cpu searching for free space.  And it
didn't precisely account for stable free space so it could spend a lot
of cpu time discovering that free space can't be used because it wasn't
stable.

This fixes these initial critical flaws in the buddy allocator.  Before
it could only address a few hundred megs and now it can address 2^64
blocks.  Before it limited bulk inode creation searching for slots and
leaf bits and now other components are much higher in the profiles with
greater create rates.

First we remove the special case single indirect block.  The root now
references a block that can be at any height.  The root records the
height and each block records its level.  We descend until we hit the
leaf.  We add a stack of the blocks traversed so that we can ascend and
fix up parent indexing after we modify a leaf.

Now that we can have quite a lot of parent indirect blocks we can no
longer have a static bitmap for allocating buddy blocks.  We instead
precisely preallocate two blocks for every buddy block that will be used
to address all the device blocks.  The blkno offset of these pairs of
buddy blocks can be calculated for a given position in the tree.
Allocating a blkno xors the low bit of the blkno and freeing is a nop.
This happily gets rid of the specific allocation of buddy blocks with
its regions and worrying about stable free blocks itself.

Then we index the first set index in a block for each order.  In parent
blocks this tells you the slot you can traverse to find a free region of
that order.  In leaf blocks it tells you the specific block offset of
the first free extent.  This is kept up to date as we set and clear
buddy bits in leaves and free_order bits in parent slots.  Allocation
now is a simple matter of block reads and array dereferencing.

And we now precisely account for frees that should not satisfy
allocation until after a transaction commit.  We record frees of stable
data in extent nodes in an rbtree after their buddy blocks have been
dirtied.  Because their blocks are dirtied we can free them as the
transaction commits without errors.  Similarly, we can also revert them
if the transaction commit fails so that they don't satisfy allocation.
This prevents us from having to hang or go read-only if a transaction
commit fails.

The two changes visible to callers are easy argument changes:
scoutfs_buddy_free() now takes a seq to specify when the allocation was
first allocated, and scoutfs_buddy_alloc_same() has its arguments match
that it only makes sense for single block allocations.

Unfortunately all these changes are interrelated so the resulting patch
amounts to a rewrite.  The core buddy bitmap helper functions and loops
are the same but the surrounding block container code changes
significnatly.

Signed-off-by: Zach Brown <zab@versity.com>

kmod/src/block.c

@@ -372,7 +372,7 @@ struct buffer_head *scoutfs_block_dirty_ref(struct super_block *sb,
 	if (IS_ERR(bh) || ref->seq == sbi->super.hdr.seq)
 		return bh;
 
-	ret = scoutfs_buddy_alloc_same(sb, &blkno, 0, le64_to_cpu(ref->blkno));
+	ret = scoutfs_buddy_alloc_same(sb, &blkno, le64_to_cpu(ref->blkno));
 	if (ret < 0)
 		goto out;
 
@@ -382,7 +382,7 @@ struct buffer_head *scoutfs_block_dirty_ref(struct super_block *sb,
 		goto out;
 	}
 
-	ret = scoutfs_buddy_free(sb, bh->b_blocknr, 0);
+	ret = scoutfs_buddy_free(sb, ref->seq, bh->b_blocknr, 0);
 	if (ret)
 		goto out;
 
@@ -399,7 +399,8 @@ out:
 	scoutfs_block_put(bh);
 	if (ret) {
 		if (!IS_ERR_OR_NULL(copy_bh)) {
-			err = scoutfs_buddy_free(sb, copy_bh->b_blocknr, 0);
+			err = scoutfs_buddy_free(sb, sbi->super.hdr.seq,
+						 copy_bh->b_blocknr, 0);
 			WARN_ON_ONCE(err); /* freeing dirty must work */
 		}
 		scoutfs_block_put(copy_bh);
@@ -452,6 +453,8 @@ out:
  */
 struct buffer_head *scoutfs_block_dirty_alloc(struct super_block *sb)
 {
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	struct scoutfs_super_block *super = &sbi->stable_super;
 	struct buffer_head *bh;
 	u64 blkno;
 	int ret;
@@ -463,7 +466,7 @@ struct buffer_head *scoutfs_block_dirty_alloc(struct super_block *sb)
 
 	bh = scoutfs_block_dirty(sb, blkno);
 	if (IS_ERR(bh)) {
-		err = scoutfs_buddy_free(sb, blkno, 0);
+		err = scoutfs_buddy_free(sb, super->hdr.seq, blkno, 0);
 		WARN_ON_ONCE(err); /* freeing dirty must work */
 	}
 	return bh;

kmod/src/btree.c

@@ -550,7 +550,10 @@ static struct buffer_head *alloc_tree_block(struct super_block *sb)
 /* the caller has ensured that the free must succeed */
 static void free_tree_block(struct super_block *sb, __le64 blkno)
 {
-	int err = scoutfs_buddy_free(sb, le64_to_cpu(blkno), 0);
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+
+	int err = scoutfs_buddy_free(sb, sbi->super.hdr.seq,
+				     le64_to_cpu(blkno), 0);
 	WARN_ON_ONCE(err);
 }
 

kmod/src/buddy.h

@@ -2,15 +2,19 @@
 #define _SCOUTFS_BUDDY_H_
 
 int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order);
-int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, int order,
-			     u64 existing);
-int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order);
+int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, u64 existing);
+int scoutfs_buddy_free(struct super_block *sb, __le64 seq, u64 blkno,
+		       int order);
 void scoutfs_buddy_free_extent(struct super_block *sb, u64 blkno, u64 count);
 
 int scoutfs_buddy_was_free(struct super_block *sb, u64 blkno, int order);
-int scoutfs_buddy_bfree(struct super_block *sb, u64 *bfree);
+u64 scoutfs_buddy_bfree(struct super_block *sb);
 
 unsigned int scoutfs_buddy_alloc_count(struct super_block *sb);
-void scoutfs_buddy_reset_count(struct super_block *sb);
+int scoutfs_buddy_apply_pending(struct super_block *sb, bool alloc);
+void scoutfs_buddy_committed(struct super_block *sb);
+
+int scoutfs_buddy_setup(struct super_block *sb);
+void scoutfs_buddy_destroy(struct super_block *sb);
 
 #endif

kmod/src/filerw.c

@@ -147,7 +147,7 @@ static int alloc_file_block(struct super_block *sb, u64 *blkno)
 	spin_unlock(&sbi->file_alloc_lock);
 
 	if (order > 0)
-		scoutfs_buddy_free(sb, alloc_blkno, order);
+		scoutfs_buddy_free(sb, sbi->super.hdr.seq, alloc_blkno, order);
 
 out:
 	trace_printk("allocated blkno %llu ret %d\n", *blkno, ret);
@@ -246,7 +246,7 @@ int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size)
 			if (blkno == 0)
 				continue;
 
-			ret = scoutfs_buddy_free(sb, blkno, 0);
+			ret = scoutfs_buddy_free(sb, bmap.seq[i], blkno, 0);
 			if (ret)
 				break;
 
@@ -357,6 +357,8 @@ static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno)
 static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret)
 {
 	struct super_block *sb = inode->i_sb;
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	struct scoutfs_super_block *super = &sbi->stable_super;
 	struct scoutfs_btree_root *meta = SCOUTFS_META(sb);
 	struct scoutfs_block_map bmap;
 	struct scoutfs_btree_val val;
@@ -406,7 +408,7 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret)
 		goto out;
 
 	if (old_blkno) {
-		ret = scoutfs_buddy_free(sb, old_blkno, 0);
+		ret = scoutfs_buddy_free(sb, bmap.seq[i], old_blkno, 0);
 		if (ret)
 			goto out;
 	}

kmod/src/format.h

@@ -19,8 +19,7 @@
  */
 #define SCOUTFS_SUPER_BLKNO ((64 * 1024) >> SCOUTFS_BLOCK_SHIFT)
 #define SCOUTFS_SUPER_NR 2
-#define SCOUTFS_BUDDY_BM_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR)
-#define SCOUTFS_BUDDY_BM_NR 2
+#define SCOUTFS_BUDDY_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR)
 
 #define SCOUTFS_MAX_TRANS_BLOCKS  (128 * 1024 * 1024 / SCOUTFS_BLOCK_SIZE)
 
@@ -48,42 +47,49 @@ struct scoutfs_block_ref {
 	__le64 seq;
 } __packed;
 
-struct scoutfs_bitmap_block {
-	struct scoutfs_block_header hdr;
-	__le64 bits[0];
-} __packed;
-
 /*
- * Track allocations from BLOCK_SIZE to (BLOCK_SIZE << ..._ORDERS).
+ * If the block was full of bits the largest possible order would be
+ * the block size shift + 3 (BITS_PER_BYTE).  But the header uses
+ * up some space and then the buddy bits mean two bits per block.
+ * Then +1 for this being the number, not the greatest order.
  */
-#define SCOUTFS_BUDDY_ORDERS 8
+#define SCOUTFS_BUDDY_ORDERS (SCOUTFS_BLOCK_SHIFT + 3 - 2 + 1)
 
 struct scoutfs_buddy_block {
 	struct scoutfs_block_header hdr;
-	__le32 order_counts[SCOUTFS_BUDDY_ORDERS];
-	__le64 bits[0];
+	__le16 first_set[SCOUTFS_BUDDY_ORDERS];
+	__u8 level;
+	__u8 __pad[3]; /* naturally align bits */
+	union {
+		struct scoutfs_buddy_slot {
+			__le64 seq;
+			__le16 free_orders;
+			/* XXX seems like we could hide a bit somewhere */
+			__u8 blkno_off;
+		} __packed slots[0];
+		__le64 bits[0];
+	} __packed;
 } __packed;
 
 /*
- * If we had log2(raw bits) orders we'd fully use all of the raw bits in
- * the block.  We're close enough that the amount of space wasted at the
- * end (~1/256th of the block, ~64 bytes) isn't worth worrying about.
+ * Each buddy leaf block references order 0 blocks with half of its
+ * bitmap.  The other half of the bits are used for the higher order
+ * bits.
  */
 #define SCOUTFS_BUDDY_ORDER0_BITS \
 	(((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_block)) * 8) / 2)
 
-struct scoutfs_buddy_indirect {
-	struct scoutfs_block_header hdr;
-	__le64 order_totals[SCOUTFS_BUDDY_ORDERS];
-	struct scoutfs_buddy_slot {
-		__u8 free_orders;
-		struct scoutfs_block_ref ref;
-	} slots[0];
+#define SCOUTFS_BUDDY_SLOTS						\
+	((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_block)) /	\
+		sizeof(struct scoutfs_buddy_slot))
+
+struct scoutfs_buddy_root {
+	struct scoutfs_buddy_slot slot;
+	__u8 height;
 } __packed;
 
-#define SCOUTFS_BUDDY_SLOTS						\
-	((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_indirect)) /	\
-		sizeof(struct scoutfs_buddy_slot))
+/* ((SCOUTFS_BUDDY_SLOTS^5) * SCOUTFS_BUDDY_ORDER0_BITS) > 2^52 */
+#define SCOUTFS_BUDDY_MAX_HEIGHT 6
 
 /*
  * We should be able to make the offset smaller if neither dirents nor
@@ -180,10 +186,10 @@ struct scoutfs_super_block {
 	__u8 uuid[SCOUTFS_UUID_BYTES];
 	__le64 next_ino;
 	__le64 total_blocks;
-	__le32 buddy_blocks;
+	__le64 free_blocks;
+	__le64 buddy_blocks;
+        struct scoutfs_buddy_root buddy_root;
         struct scoutfs_btree_root btree_root;
-        struct scoutfs_block_ref buddy_ind_ref;
-        struct scoutfs_block_ref buddy_bm_ref;
 } __packed;
 
 #define SCOUTFS_ROOT_INO 1

kmod/src/super.c

@@ -48,12 +48,8 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst)
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
 	struct scoutfs_super_block *super = &sbi->super;
 	__le32 * __packed uuid = (void *)super->uuid;
-	int ret;
-
-	ret = scoutfs_buddy_bfree(sb, &kst->f_bfree);
-	if (ret)
-		return ret;
 
+	kst->f_bfree =  scoutfs_buddy_bfree(sb);
 	kst->f_type = SCOUTFS_SUPER_MAGIC;
 	kst->f_bsize = SCOUTFS_BLOCK_SIZE;
 	kst->f_blocks = le64_to_cpu(super->total_blocks);
@@ -198,8 +194,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 	sbi->block_dirty_tree = RB_ROOT;
 	init_waitqueue_head(&sbi->block_wq);
 	atomic_set(&sbi->block_writes, 0);
-	mutex_init(&sbi->buddy_mutex);
-	atomic_set(&sbi->buddy_count, 0);
 	init_rwsem(&sbi->btree_rwsem);
 	atomic_set(&sbi->trans_holds, 0);
 	init_waitqueue_head(&sbi->trans_hold_wq);
@@ -220,6 +214,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 
 	ret = scoutfs_setup_counters(sb) ?:
 	      read_supers(sb) ?:
+	      scoutfs_buddy_setup(sb) ?:
 	      scoutfs_setup_trans(sb);
 	if (ret)
 		return ret;
@@ -252,6 +247,7 @@ static void scoutfs_kill_sb(struct super_block *sb)
 	kill_block_super(sb);
 	if (sbi) {
 		scoutfs_shutdown_trans(sb);
+		scoutfs_buddy_destroy(sb);
 		scoutfs_destroy_counters(sb);
 		if (sbi->kset)
 			kset_unregister(sbi->kset);

kmod/src/super.h

@@ -8,7 +8,7 @@
 #include "buddy.h"
 
 struct scoutfs_counters;
-struct buddy_alloc;
+struct buddy_info;
 
 struct scoutfs_sb_info {
 	struct super_block *sb;
@@ -24,8 +24,7 @@ struct scoutfs_sb_info {
 	atomic_t block_writes;
 	int block_write_err;
 
-	struct mutex buddy_mutex;
-	atomic_t buddy_count;
+	struct buddy_info *buddy_info;
 
 	struct rw_semaphore btree_rwsem;
 

kmod/src/trans.c

@@ -90,17 +90,20 @@ void scoutfs_trans_write_func(struct work_struct *work)
 
 		scoutfs_filerw_free_alloc(sb);
 
-		ret = scoutfs_block_write_dirty(sb) ?:
+		ret = scoutfs_buddy_apply_pending(sb, false) ?:
+		      scoutfs_block_write_dirty(sb) ?:
 		      scoutfs_write_dirty_super(sb);
-		if (!ret)
+		if (ret) {
+			scoutfs_buddy_apply_pending(sb, true);
+		} else {
+			scoutfs_buddy_committed(sb);
 			advance = 1;
+		}
 	}
 
 	spin_lock(&sbi->trans_write_lock);
-	if (advance) {
+	if (advance)
 		scoutfs_advance_dirty_super(sb);
-		scoutfs_buddy_reset_count(sb);
-	}
 	sbi->trans_write_count++;
 	sbi->trans_write_ret = ret;
 	spin_unlock(&sbi->trans_write_lock);