scoutfs: store the buddy allocator in a radix

The current implementation of the allocator was built for for a world
where blocks were much, much, larger.  It could get away with keeping
the entire bitmap resident and having to read it its entirety before
being able to use it for the first time.

That will not work in the current architecture that's built around a
smaller metadata block size.  The raw size of the allocator gets large
enough that all of those behaviours become problematic at scale.

This shifts the buddy allocator to be stored in a radix of blocks
instead of in a ring log.  This brings it more in line with the
structure of the btree item indexes.  It can be initially read, cached,
and invalidated at block granularity.

In addition, it cleverly uses the cow block structures to solve the
unreferenced space allocation constraint that the previous allocator
hadn't.  It can compare the dirty and stable blocks to discover free
blocks that aren't referenced by the old stable state.  The old
allocator would have grown a bunch of extra special complexity to
address this.

There's still work to be done but this is a solid start.

Signed-off-by: Zach Brown <zab@versity.com>

kmod/src/block.c

@@ -434,7 +434,7 @@ static struct scoutfs_block *dirty_ref(struct super_block *sb,
 	struct scoutfs_block *found;
 	struct scoutfs_block *bl;
 	unsigned long flags;
-	u64 blkno;
+	u64 blkno = 0;
 	int ret;
 	int err;
 
@@ -442,7 +442,7 @@ static struct scoutfs_block *dirty_ref(struct super_block *sb,
 	if (IS_ERR(bl) || ref->seq == sbi->super.hdr.seq)
 		return bl;
 
-	ret = scoutfs_buddy_alloc(sb, &blkno, 0);
+	ret = scoutfs_buddy_alloc_same(sb, &blkno, 0, le64_to_cpu(ref->blkno));
 	if (ret < 0)
 		goto out;
 

kmod/src/buddy.h

@@ -2,10 +2,8 @@
 #define _SCOUTFS_BUDDY_H_
 
 int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order);
+int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, int order,
+			     u64 existing);
 int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order);
 
-int scoutfs_read_buddy_chunks(struct super_block *sb);
-void scoutfs_reset_buddy_chunks(struct super_block *sb);
-int scoutfs_dirty_buddy_chunks(struct super_block *sb);
-
 #endif

kmod/src/format.h

@@ -19,7 +19,8 @@
  */
 #define SCOUTFS_SUPER_BLKNO ((64 * 1024) >> SCOUTFS_BLOCK_SHIFT)
 #define SCOUTFS_SUPER_NR 2
-#define SCOUTFS_BUDDY_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR)
+#define SCOUTFS_BUDDY_BM_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR)
+#define SCOUTFS_BUDDY_BM_NR 2
 
 /*
  * This header is found at the start of every block so that we can
@@ -35,6 +36,52 @@ struct scoutfs_block_header {
 	__le64 blkno;
 } __packed;
 
+/*
+ * Block references include the sequence number so that we can detect
+ * readers racing with writers and so that we can tell that we don't
+ * need to follow a reference when traversing based on seqs.
+ */
+struct scoutfs_block_ref {
+	__le64 blkno;
+	__le64 seq;
+} __packed;
+
+struct scoutfs_bitmap_block {
+	struct scoutfs_block_header hdr;
+	__le64 bits[0];
+} __packed;
+
+/*
+ * Track allocations from BLOCK_SIZE to (BLOCK_SIZE << ..._ORDERS).
+ */
+#define SCOUTFS_BUDDY_ORDERS 8
+
+struct scoutfs_buddy_block {
+	struct scoutfs_block_header hdr;
+	__le32 order_counts[SCOUTFS_BUDDY_ORDERS];
+	__le64 bits[0];
+} __packed;
+
+/*
+ * If we had log2(raw bits) orders we'd fully use all of the raw bits in
+ * the block.  We're close enough that the amount of space wasted at the
+ * end (~1/256th of the block, ~64 bytes) isn't worth worrying about.
+ */
+#define SCOUTFS_BUDDY_ORDER0_BITS \
+	(((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_block)) * 8) / 2)
+
+struct scoutfs_buddy_indirect {
+	struct scoutfs_block_header hdr;
+	struct scoutfs_buddy_slot {
+		__u8 free_orders;
+		struct scoutfs_block_ref ref;
+	} slots[0];
+} __packed;
+
+#define SCOUTFS_BUDDY_SLOTS						\
+	((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_block)) /	\
+		sizeof(struct scoutfs_buddy_slot))
+
 /*
  * We should be able to make the offset smaller if neither dirents nor
  * data items use the full 64 bits.
@@ -57,16 +104,6 @@ struct scoutfs_key {
 
 #define SCOUTFS_MAX_ITEM_LEN 2048
 
-/*
- * Block references include the sequence number so that we can detect
- * readers racing with writers and so that we can tell that we don't
- * need to follow a reference when traversing based on seqs.
- */
-struct scoutfs_block_ref {
-	__le64 blkno;
-	__le64 seq;
-} __packed;
-
 struct scoutfs_treap_root {
 	__le16 off;
 } __packed;
@@ -109,48 +146,6 @@ struct scoutfs_btree_item {
 
 #define SCOUTFS_UUID_BYTES 16
 
-/*
- * Arbitrarily choose a reasonably fine grained 64byte chunk.  This is a
- * balance between write amplification of writing chunks with a single
- * modified bit, storage overhead of partial blocks losing a chunk to
- * make room for the block header and having a pos field per chunk, and
- * runtime memory overhead of a bit per chunk.
- */
-#define SCOUTFS_BUDDY_CHUNK_LE64S 8
-#define SCOUTFS_BUDDY_CHUNK_BYTES (SCOUTFS_BUDDY_CHUNK_LE64S * 8)
-#define SCOUTFS_BUDDY_CHUNK_BITS (SCOUTFS_BUDDY_CHUNK_BYTES * 8)
-
-/*
- * After the pair of super blocks are a preallocated ring of blocks
- * which record modified regions of the buddy bitmap allocator.
- *
- * The seq's header needs to match the unwrapped ring index of the
- * block.
- */
-struct scoutfs_buddy_block {
-	struct scoutfs_block_header hdr;
-	u8 nr_chunks;
-	struct scoutfs_buddy_chunk {
-		__le32 pos;
-		__le64 bits[SCOUTFS_BUDDY_CHUNK_LE64S];
-	} __packed chunks[0];
-} __packed;
-
-#define SCOUTFS_BUDDY_CHUNKS_PER_BLOCK \
-	((SCOUTFS_BLOCK_SIZE - offsetof(struct scoutfs_buddy_block, chunks)) /\
-	 SCOUTFS_BUDDY_CHUNK_BYTES)
-
-
-/*
- * The super is stored in a pair of blocks in the first chunk on the
- * device.
- *
- * The ring map blocks describe the chunks that make up the ring.
- *
- * The rest of the ring fields describe the state of the ring blocks
- * that are stored in their chunks.  The active portion of the ring
- * describes the current state of the system and is replayed on mount.
- */
 struct scoutfs_super_block {
 	struct scoutfs_block_header hdr;
 	__le64 id;
@@ -158,10 +153,9 @@ struct scoutfs_super_block {
 	__le64 next_ino;
 	__le64 total_blocks;
 	__le32 buddy_blocks;
-	__le32 buddy_sweep_bit;
-	__le64 buddy_head;
-	__le64 buddy_tail;
         struct scoutfs_btree_root btree_root;
+        struct scoutfs_block_ref buddy_ind_ref;
+        struct scoutfs_block_ref buddy_bm_ref;
 } __packed;
 
 #define SCOUTFS_ROOT_INO 1

kmod/src/scoutfs_trace.h

@@ -109,6 +109,53 @@ TRACE_EVENT(scoutfs_update_inode,
 		__entry->ino, __entry->size)
 );
 
+TRACE_EVENT(scoutfs_buddy_alloc,
+	TP_PROTO(u64 blkno, int order, int region, int ret),
+
+	TP_ARGS(blkno, order, region, ret),
+
+	TP_STRUCT__entry(
+		__field(u64, blkno)
+		__field(int, order)
+		__field(int, region)
+		__field(int, ret)
+	),
+
+	TP_fast_assign(
+		__entry->blkno = blkno;
+		__entry->order = order;
+		__entry->region = region;
+		__entry->ret = ret;
+	),
+
+	TP_printk("blkno %llu order %d region %d ret %d",
+		__entry->blkno, __entry->order, __entry->region, __entry->ret)
+);
+
+
+TRACE_EVENT(scoutfs_buddy_free,
+	TP_PROTO(u64 blkno, int order, int region, int ret),
+
+	TP_ARGS(blkno, order, region, ret),
+
+	TP_STRUCT__entry(
+		__field(u64, blkno)
+		__field(int, order)
+		__field(int, region)
+		__field(int, ret)
+	),
+
+	TP_fast_assign(
+		__entry->blkno = blkno;
+		__entry->order = order;
+		__entry->region = region;
+		__entry->ret = ret;
+	),
+
+	TP_printk("blkno %llu order %d region %d ret %d",
+		__entry->blkno, __entry->order, __entry->region, __entry->ret)
+);
+
 #endif /* _TRACE_SCOUTFS_H */
 
 /* This part must be outside protection */

kmod/src/super.c

@@ -47,6 +47,8 @@ void scoutfs_advance_dirty_super(struct super_block *sb)
 	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
 	struct scoutfs_super_block *super = &sbi->super;
 
+	sbi->stable_super = sbi->super;
+
 	le64_add_cpu(&super->hdr.blkno, 1);
 	if (le64_to_cpu(super->hdr.blkno) == (SCOUTFS_SUPER_BLKNO +
 					      SCOUTFS_SUPER_NR))
@@ -107,8 +109,7 @@ static int read_supers(struct super_block *sb)
 
 		if (found < 0 || (le64_to_cpu(super->hdr.seq) >
 				le64_to_cpu(sbi->super.hdr.seq))) {
-			memcpy(&sbi->super, super,
-			       sizeof(struct scoutfs_super_block));
+			sbi->super = *super;
 			found = i;
 		}
 	}
@@ -123,6 +124,8 @@ static int read_supers(struct super_block *sb)
 	scoutfs_info(sb, "using super %u with seq %llu",
 		     found, le64_to_cpu(sbi->super.hdr.seq));
 
+	sbi->stable_super = sbi->super;
+
 	return 0;
 }
 
@@ -162,13 +165,11 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 
 	ret = scoutfs_setup_counters(sb) ?:
 	      read_supers(sb) ?:
-	      scoutfs_setup_trans(sb) ?:
-	      scoutfs_read_buddy_chunks(sb);
+	      scoutfs_setup_trans(sb);
 	if (ret)
 		return ret;
 
 	scoutfs_advance_dirty_super(sb);
-	scoutfs_reset_buddy_chunks(sb);
 
 	inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO);
 	if (IS_ERR(inode))

kmod/src/super.h

@@ -14,6 +14,7 @@ struct scoutfs_sb_info {
 	struct super_block *sb;
 
 	struct scoutfs_super_block super;
+	struct scoutfs_super_block stable_super;
 
 	spinlock_t next_ino_lock;
 
@@ -24,7 +25,6 @@ struct scoutfs_sb_info {
 	int block_write_err;
 
 	struct mutex buddy_mutex;
-	struct buddy_alloc *bud;
 
 	/* XXX there will be a lot more of these :) */
 	struct rw_semaphore btree_rwsem;

kmod/src/trans.c

@@ -64,8 +64,7 @@ void scoutfs_trans_write_func(struct work_struct *work)
 	/* XXX probably want to write out dirty pages in inodes */
 
 	if (scoutfs_has_dirty_blocks(sb)) {
-		ret = scoutfs_dirty_buddy_chunks(sb) ?:
-		      scoutfs_write_dirty_blocks(sb) ?:
+		ret = scoutfs_write_dirty_blocks(sb) ?:
 		      scoutfs_write_dirty_super(sb);
 		if (!ret)
 			advance = 1;
@@ -73,10 +72,8 @@ void scoutfs_trans_write_func(struct work_struct *work)
 
 
 	spin_lock(&sbi->trans_write_lock);
-	if (advance) {
+	if (advance)
 		scoutfs_advance_dirty_super(sb);
-		scoutfs_reset_buddy_chunks(sb);
-	}
 	sbi->trans_write_count++;
 	sbi->trans_write_ret = ret;
 	spin_unlock(&sbi->trans_write_lock);