From c65b70f2aac731364623b59c63652b9cb20a9376 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 27 Oct 2016 15:11:49 -0700 Subject: [PATCH] Use full radix for buddy and record first set The first pass of the buddy allocator had a fixed indirect block so it couldn't address large devices. It didn't index set bits or slots for each order so we spent a lot of cpu searching for free space. And it didn't precisely account for stable free space so it could spend a lot of cpu time discovering that free space can't be used because it wasn't stable. This fixes these initial critical flaws in the buddy allocator. Before it could only address a few hundred megs and now it can address 2^64 blocks. Before it limited bulk inode creation searching for slots and leaf bits and now other components are much higher in the profiles with greater create rates. First we remove the special case single indirect block. The root now references a block that can be at any height. The root records the height and each block records its level. We descend until we hit the leaf. We add a stack of the blocks traversed so that we can ascend and fix up parent indexing after we modify a leaf. Now that we can have quite a lot of parent indirect blocks we can no longer have a static bitmap for allocating buddy blocks. We instead precisely preallocate two blocks for every buddy block that will be used to address all the device blocks. The blkno offset of these pairs of buddy blocks can be calculated for a given position in the tree. Allocating a blkno xors the low bit of the blkno and freeing is a nop. This happily gets rid of the specific allocation of buddy blocks with its regions and worrying about stable free blocks itself. Then we index the first set index in a block for each order. In parent blocks this tells you the slot you can traverse to find a free region of that order. In leaf blocks it tells you the specific block offset of the first free extent. This is kept up to date as we set and clear buddy bits in leaves and free_order bits in parent slots. Allocation now is a simple matter of block reads and array dereferencing. And we now precisely account for frees that should not satisfy allocation until after a transaction commit. We record frees of stable data in extent nodes in an rbtree after their buddy blocks have been dirtied. Because their blocks are dirtied we can free them as the transaction commits without errors. Similarly, we can also revert them if the transaction commit fails so that they don't satisfy allocation. This prevents us from having to hang or go read-only if a transaction commit fails. The two changes visible to callers are easy argument changes: scoutfs_buddy_free() now takes a seq to specify when the allocation was first allocated, and scoutfs_buddy_alloc_same() has its arguments match that it only makes sense for single block allocations. Unfortunately all these changes are interrelated so the resulting patch amounts to a rewrite. The core buddy bitmap helper functions and loops are the same but the surrounding block container code changes significnatly. Signed-off-by: Zach Brown --- kmod/src/block.c | 11 +- kmod/src/btree.c | 5 +- kmod/src/buddy.c | 1523 +++++++++++++++++++++++++-------------------- kmod/src/buddy.h | 14 +- kmod/src/filerw.c | 8 +- kmod/src/format.h | 60 +- kmod/src/super.c | 10 +- kmod/src/super.h | 5 +- kmod/src/trans.c | 13 +- 9 files changed, 924 insertions(+), 725 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index b8b3ccb8..b3f99f09 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -372,7 +372,7 @@ struct buffer_head *scoutfs_block_dirty_ref(struct super_block *sb, if (IS_ERR(bh) || ref->seq == sbi->super.hdr.seq) return bh; - ret = scoutfs_buddy_alloc_same(sb, &blkno, 0, le64_to_cpu(ref->blkno)); + ret = scoutfs_buddy_alloc_same(sb, &blkno, le64_to_cpu(ref->blkno)); if (ret < 0) goto out; @@ -382,7 +382,7 @@ struct buffer_head *scoutfs_block_dirty_ref(struct super_block *sb, goto out; } - ret = scoutfs_buddy_free(sb, bh->b_blocknr, 0); + ret = scoutfs_buddy_free(sb, ref->seq, bh->b_blocknr, 0); if (ret) goto out; @@ -399,7 +399,8 @@ out: scoutfs_block_put(bh); if (ret) { if (!IS_ERR_OR_NULL(copy_bh)) { - err = scoutfs_buddy_free(sb, copy_bh->b_blocknr, 0); + err = scoutfs_buddy_free(sb, sbi->super.hdr.seq, + copy_bh->b_blocknr, 0); WARN_ON_ONCE(err); /* freeing dirty must work */ } scoutfs_block_put(copy_bh); @@ -452,6 +453,8 @@ out: */ struct buffer_head *scoutfs_block_dirty_alloc(struct super_block *sb) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->stable_super; struct buffer_head *bh; u64 blkno; int ret; @@ -463,7 +466,7 @@ struct buffer_head *scoutfs_block_dirty_alloc(struct super_block *sb) bh = scoutfs_block_dirty(sb, blkno); if (IS_ERR(bh)) { - err = scoutfs_buddy_free(sb, blkno, 0); + err = scoutfs_buddy_free(sb, super->hdr.seq, blkno, 0); WARN_ON_ONCE(err); /* freeing dirty must work */ } return bh; diff --git a/kmod/src/btree.c b/kmod/src/btree.c index cf45346e..9f6a0ed4 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -550,7 +550,10 @@ static struct buffer_head *alloc_tree_block(struct super_block *sb) /* the caller has ensured that the free must succeed */ static void free_tree_block(struct super_block *sb, __le64 blkno) { - int err = scoutfs_buddy_free(sb, le64_to_cpu(blkno), 0); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + int err = scoutfs_buddy_free(sb, sbi->super.hdr.seq, + le64_to_cpu(blkno), 0); WARN_ON_ONCE(err); } diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c index 96f3048f..40dc2e65 100644 --- a/kmod/src/buddy.c +++ b/kmod/src/buddy.c @@ -12,6 +12,7 @@ */ #include #include +#include #include "super.h" #include "format.h" @@ -20,117 +21,139 @@ #include "scoutfs_trace.h" /* - * scoutfs uses buddy bitmaps to allocate block regions. The buddy - * allocator is nice because it uses one index for allocating by size - * and freeing and merging by location. The index is dense and has a - * predictable worst case size that we can preallocate. As described - * below, it also makes it easy to find unions of free regions between - * two indexes. + * scoutfs uses buddy bitmaps in an augmented radix to index free space. * - * The buddy allocator is built from a hierarchy of bitmaps for each - * power of two order of blocks that we can allocate. If a high order - * buddy bit is set then all the lower order bits that it covers are - * clear. The bits are stored in blocks that are stored in a fixed - * depth radix with a single parent indirect block. The super block - * references the indirect block. The block references in the indirect - * block also include a bitmap of orders that are free in the referenced - * block. + * At the heart of the allocator are the buddy bitmaps in the radix + * leaves. For a given region of blocks there are bitmaps for each + * power of two order of blocks that can be allocated. N bits record + * whether each order 0 size block region is allocated or freed, then + * N/2 bits describe order 1 regions that span pairs of order 0 blocks, + * and so on. This ends up using two bits in the bitmaps for each + * device block that's managed. * - * The blknos for the buddy blocks themselves are allocated out of a - * single bitmap block that is referenced by the super. + * An order bit is set when it is free. All of its lower order bits + * will be clear. To allocate we clear a bit. A partial allocation + * clears the higher order bit and each buddy for each lower order until + * the allocated order. Freeing sets an order bit. Then if it's buddy + * order is also set we clear both and set their higher order bit. This + * proceeds to the highest order. * - * All the blocks are read and cowed with the usual block layer routines - * so that we reuse the same code to evict and retry stale cached - * blocks, cow, etc. The allocator in the block code gives us the - * source blkno for a cow operation so we can use the correct allocator - * (none for bitmap blocks, bitmap for buddy blocks, buddy for btree - * blocks and extents). + * Each buddy block records the first set bit in each order bitmap. As + * bits are set they update these first set records if they're before + * the previous value. As bits are cleared we find the next set if it + * was the first. * - * The trickiest part of the allocator is due to the cow nature of our - * consistent updates. We can't satisfy an allocation with a region - * that's been freed in this transaction and is still referenced by the - * old stable transaction. We solve this by only returning regions that - * are free in both the stable and currently dirty allocator structures. + * These buddy bitmap blocks that each fully describe a region of blocks + * are assembled into a radix tree. Each reference to a leaf block in + * parent blocks have a bitmap of the orders that are free in its leaf + * block. The parent blocks then also record the first slot that has + * each order bit set in its child references. This indexing holds all + * the way to the root. This lets us quickly determine an order that + * will satisfy an allocation and descend to the leaf that contains the + * first free region of that order. * - * The single indirect block in the radix limits the number of blocks - * that can be described by the radix to just under a TB. The device - * will be managed by multiple radix trees some day. + * These buddy blocks themselves are located in preallocated space. Each + * logical position in the tree occupies two blocks on the device. In + * each transaction we use the currently referenced block to cow into + * its partner. Since the block positions are calculated the block + * references only need a bit to specify which of the pair is being + * referenced. The number of blocks needed is precisely calculated by + * taking the number of leaf blocks needed to track the device blocks + * and dividing by the radix fanout until we have a single root block. * - * XXX: - * - verify blocks on read? - * - more rigorously test valid blkno/order inputs - * - detect corruption/errors when trying to free free extents - * - mkfs should initialize all the slots - * - shrink and grow - * - metadata and data regions - * - worry about testing for free buddies outside device during free? - * - we could track the first set in order bitmaps, dunno if it'd be worth it + * Each aligned block allocation order is stored in a path down the + * radix to a leaf that's a function of the block offset. This lets us + * ensure that we can allocate or free a given allocation order by + * dirtying those blocks. If we've allocated an order in a transaction + * it can always be freed (or re-allocated) while the transaction holds + * the dirty buddy blocks. + * + * We use that property to ensure that frees of stable data don't + * satisfy allocation until the next transaction. When we free stable + * data we dirty the path to its position in the radix and record the + * free in an rbtree. We can then apply these frees as we commit the + * transaction. If the transaction fails we can undo the frees and let + * the file system carry on. We'll try to reapply the frees before the + * next transaction commits. The allocator never introduces + * unrecoverable errors. + * + * The radix isn't fully populated when it's created. mkfs only + * initializes the two paths down the tree that have partially + * initialized parent slots and leaf bitmaps. The path down the left + * spine has the initial file system blocks allocated. The path down + * the right spine can have partial parent slots and bits set in the + * leaf when device sizes aren't multiples of the leaf block bit count + * and radix fanout. The kernel then only has to initialize the rest of + * the buddy blocks blocks which have fully populated parent slots and + * leaf bitmaps. + * + * XXX + * - resize is going to be a thing. figure out that thing. */ -enum { - REGION_PAIR, /* two bitmap blocks at known blknos */ - REGION_BM, /* buddy blocks in the bitmap block off the super */ - REGION_BUDDY, /* btree blocks and extents in the buddy bitmaps */ +struct buddy_info { + struct mutex mutex; + + atomic_t alloc_count; + struct rb_root pending_frees; + + /* max height given total blocks */ + u8 max_height; + /* the device blkno of the first block of a given level */ + u64 level_blkno[SCOUTFS_BUDDY_MAX_HEIGHT]; + /* blk divisor to find slot index at each level */ + u64 level_div[SCOUTFS_BUDDY_MAX_HEIGHT]; + + struct buddy_stack { + struct buffer_head *bh[SCOUTFS_BUDDY_MAX_HEIGHT]; + u16 sl[SCOUTFS_BUDDY_MAX_HEIGHT]; + int nr; + } stack; }; -static int blkno_region(struct scoutfs_super_block *super, u64 blkno) -{ - u64 end; - - end = SCOUTFS_BUDDY_BM_BLKNO + SCOUTFS_BUDDY_BM_NR; - if (blkno < end) - return REGION_PAIR; - - end += le32_to_cpu(super->buddy_blocks); - if (blkno < end) - return REGION_BM; - - return REGION_BUDDY; -} - /* the first device blkno covered by the buddy allocator */ static u64 first_blkno(struct scoutfs_super_block *super) { - return SCOUTFS_BUDDY_BM_BLKNO + SCOUTFS_BUDDY_BM_NR + - le32_to_cpu(super->buddy_blocks); + return SCOUTFS_BUDDY_BLKNO + le64_to_cpu(super->buddy_blocks); } -/* the slot in the indirect block of a given blkno */ -static int indirect_slot(struct scoutfs_super_block *super, u64 blkno) +/* the last device blkno covered by the buddy allocator */ +static u64 last_blkno(struct scoutfs_super_block *super) { - return (u32)(blkno - first_blkno(super)) / SCOUTFS_BUDDY_ORDER0_BITS; + return le64_to_cpu(super->total_blocks) - 1; } -/* device blkno of order bit in slot */ -static u64 slot_buddy_blkno(struct scoutfs_super_block *super, int sl, - int order, int nr) +/* the last relative blkno covered by the buddy allocator */ +static u64 last_blk(struct scoutfs_super_block *super) { - return first_blkno(super) + ((u64)sl * SCOUTFS_BUDDY_ORDER0_BITS) + - ((u64)nr << order); + return last_blkno(super) - first_blkno(super); } -/* number of blocks managed by the buddy block referenced by the given slot */ -static int slot_count(struct scoutfs_super_block *super, int sl) +/* true when the device blkno is covered by the allocator */ +static bool device_blkno(struct scoutfs_super_block *super, u64 blkno) { - u64 first = first_blkno(super) + ((u64)sl * SCOUTFS_BUDDY_ORDER0_BITS); - - return min_t(int, le64_to_cpu(super->total_blocks) - first, - SCOUTFS_BUDDY_ORDER0_BITS); + return blkno >= first_blkno(super) && blkno <= last_blkno(super); } -/* the order 0 bit offset of blkno */ -static int buddy_bit(struct scoutfs_super_block *super, u64 blkno) +/* true when the device blkno is used for buddy blocks */ +static bool buddy_blkno(struct scoutfs_super_block *super, u64 blkno) { - return (u32)(blkno - first_blkno(super)) % SCOUTFS_BUDDY_ORDER0_BITS; + return blkno < first_blkno(super); } -/* true if the blkno could be the start of an allocation of the order */ -static bool valid_order(struct scoutfs_super_block *super, u64 blkno, int order) +/* the order 0 bit offset in a buddy block of a given relative blk */ +static int buddy_bit(u64 blk) { - return (buddy_bit(super, blkno) & ((1 << order) - 1)) == 0; + return do_div(blk, SCOUTFS_BUDDY_ORDER0_BITS); } -/* the starting bit offset in the block bitmap of an order's bitmap */ +/* true if the rel blk could be the start of an allocation of the order */ +static bool valid_order(u64 blk, int order) +{ + return (buddy_bit(blk) & ((1 << order) - 1)) == 0; +} + +/* the block bit offset of the first bit of the given order's bitmap */ static int order_off(int order) { if (order == 0) @@ -146,733 +169,893 @@ static int order_nr(int order, int nr) return order_off(order) + nr; } +static void stack_push(struct buddy_stack *sta, struct buffer_head *bh, u16 sl) +{ + sta->bh[sta->nr] = bh; + sta->sl[sta->nr++] = sl; +} + +/* sl isn't returned because callers peek the leaf where sl is meaningless */ +static struct buffer_head *stack_peek(struct buddy_stack *sta) +{ + if (sta->nr) + return sta->bh[sta->nr - 1]; + + return NULL; +} + +static struct buffer_head *stack_pop(struct buddy_stack *sta, u16 *sl) +{ + if (sta->nr) { + *sl = sta->sl[--sta->nr]; + return sta->bh[sta->nr]; + } + + return NULL; +} + +/* update first_set if the caller set an earlier nr for the given order */ +static void set_order_nr(struct scoutfs_buddy_block *bud, int order, u16 nr) +{ + u16 first = le16_to_cpu(bud->first_set[order]); + + trace_printk("set level %u order %d nr %u first %u\n", + bud->level, order, nr, first); + + if (nr <= first) + bud->first_set[order] = cpu_to_le16(nr); +} + +/* find the next first set if the caller just cleared the current first_set */ +static void clear_order_nr(struct scoutfs_buddy_block *bud, int order, u16 nr) +{ + u16 first = le16_to_cpu(bud->first_set[order]); + int size; + int i; + + trace_printk("cleared level %u order %d nr %u first %u\n", + bud->level, order, nr, first); + + if (nr != first) + return; + + if (bud->level) { + for (i = nr + 1; i < SCOUTFS_BUDDY_SLOTS; i++) { + if (le16_to_cpu(bud->slots[i].free_orders) & + (1 << order)) + break; + } + if (i == SCOUTFS_BUDDY_SLOTS) + i = U16_MAX; + + } else { + size = order_off(order + 1); + i = find_next_bit_le(bud->bits, size, + order_nr(order, first) + 1); + if (i >= size) + i = U16_MAX; + else + i -= order_off(order); + } + + bud->first_set[order] = cpu_to_le16(i); + +} + +#define for_each_changed_bit(nr, bit, old, new, tmp) \ + for (tmp = old ^ new; \ + tmp && (nr = ffs(tmp) - 1, bit = 1 << nr, 1); \ + tmp ^= bit) + +/* + * Set a slot's free_orders value and update first_set for each order + * that it changes. Returns true of the slot's free_orders was changed. + */ +static bool set_slot_free_orders(struct scoutfs_buddy_block *bud, u16 sl, + u16 free_orders) +{ + u16 old = le16_to_cpu(bud->slots[sl].free_orders); + int order; + int tmp; + int bit; + + if (old == free_orders) + return false; + + for_each_changed_bit(order, bit, old, free_orders, tmp) { + if (old & bit) + clear_order_nr(bud, order, sl); + else + set_order_nr(bud, order, sl); + } + + bud->slots[sl].free_orders = cpu_to_le16(free_orders); + return true; +} + +/* + * The block at the top of the stack has changed its bits or slots and + * updated its first set. We propagate those changes up through + * free_orders in parents slots and their first_set up through the tree + * to free_orders in the root. We can stop when a block's first_set + * values don't change free_orders in their parent's slot. + */ +static void stack_cleanup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; + struct buddy_stack *sta = &binf->stack; + struct scoutfs_buddy_root *root = &sbi->super.buddy_root; + struct scoutfs_buddy_block *bud; + struct buffer_head *bh; + u16 free_orders = 0; + bool parent; + u16 sl; + int i; + + parent = false; + while ((bh = stack_pop(sta, &sl))) { + + bud = bh_data(bh); + if (parent && !set_slot_free_orders(bud, sl, free_orders)) + break; + + free_orders = 0; + for (i = 0; i < ARRAY_SIZE(bud->first_set); i++) { + if (bud->first_set[i] != cpu_to_le16(U16_MAX)) + free_orders |= 1 << i; + } + + scoutfs_block_put(bh); + parent = true; + } + + /* set root if we got that far */ + if (bh == NULL) + root->slot.free_orders = cpu_to_le16(free_orders); + + /* put any remaining blocks */ + while ((bh = stack_pop(sta, &sl))) + scoutfs_block_put(bh); + +} + static int test_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) { return !!test_bit_le(order_nr(order, nr), bud->bits); } -static int test_buddy_bit_or_higher(struct scoutfs_buddy_block *bud, int order, - int nr) +static void set_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) { - int i; - - for (i = order; i < SCOUTFS_BUDDY_ORDERS; i++) { - if (test_buddy_bit(bud, i, nr)) - return true; - nr >>= 1; - } - - return false; + if (!test_and_set_bit_le(order_nr(order, nr), bud->bits)) + set_order_nr(bud, order, nr); } -static void set_buddy_bit(struct scoutfs_buddy_indirect *ind, - struct scoutfs_buddy_block *bud, int order, int nr) +static void clear_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) { - if (!test_and_set_bit_le(order_nr(order, nr), bud->bits)) { - le64_add_cpu(&ind->order_totals[order], 1); - le32_add_cpu(&bud->order_counts[order], 1); - } -} - -static void clear_buddy_bit(struct scoutfs_buddy_indirect *ind, - struct scoutfs_buddy_block *bud, int order, int nr) -{ - if (test_and_clear_bit_le(order_nr(order, nr), bud->bits)) { - le64_add_cpu(&ind->order_totals[order], -1); - le32_add_cpu(&bud->order_counts[order], -1); - } -} - -/* returns INT_MAX when there are no bits set */ -static int find_next_buddy_bit(struct scoutfs_buddy_block *bud, int order, - int nr) -{ - int size = order_off(order + 1); - - nr = find_next_bit_le(bud->bits, size, order_nr(order, nr)); - if (nr >= size) - return INT_MAX; - - return nr - order_off(order); -} - -static void update_free_orders(struct scoutfs_buddy_slot *slot, - struct scoutfs_buddy_block *bud) -{ - u8 free = 0; - int i; - - for (i = 0; i < SCOUTFS_BUDDY_ORDERS; i++) - free |= (!!bud->order_counts[i]) << i; - - slot->free_orders = free; + if (test_and_clear_bit_le(order_nr(order, nr), bud->bits)) + clear_order_nr(bud, order, nr); } /* - * Allocate a buddy block blkno from the super's dirty bitmap block. - * Stable buddy blocks are freed as they're cowed so we have to make - * sure that we only return blknos that were free in the previous stable - * bitmap block. + * mkfs always writes the paths down the sides of the radix that have + * partially populated blocks. We only have to initialize full blocks + * in the middle of the tree. */ -static int bitmap_alloc(struct super_block *sb, u64 *blkno) +static void init_buddy_block(struct buddy_info *binf, + struct scoutfs_super_block *super, + struct buffer_head *bh, int level) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_bitmap_block *st_bm; - struct scoutfs_bitmap_block *bm; - struct buffer_head *st_bh; - struct buffer_head *bm_bh; - int size; - int ret; - int d; - int s; - - /* mkfs should have ensured that there's bitmap blocks */ - /* XXX corruption */ - if (sbi->super.buddy_bm_ref.blkno == 0 || - sbi->stable_super.buddy_bm_ref.blkno == 0) - return -EIO; - - /* dirty the bitmap block */ - bm_bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_bm_ref); - if (IS_ERR(bm_bh)) - return PTR_ERR(bm_bh); - bm = bh_data(bm_bh); - - /* read the stable bitmap block */ - st_bh = scoutfs_block_read_ref(sb, &sbi->stable_super.buddy_bm_ref); - if (IS_ERR(st_bh)) { - ret = PTR_ERR(st_bh); - goto out; - } - st_bm = bh_data(st_bh); - - /* find the first bit that is set in both dirty and stable bitmaps */ - size = le32_to_cpu(sbi->super.buddy_blocks); - s = 0; - do { - d = find_next_bit_le(bm->bits, size, s); - s = find_next_bit_le(st_bm->bits, size, d); - } while (d != s); - if (d >= size) { - ret = -ENOSPC; - goto out; - } - - *blkno = SCOUTFS_BUDDY_BM_BLKNO + SCOUTFS_BUDDY_BM_NR + d; - clear_bit_le(d, &bm->bits); - ret = 0; -out: - scoutfs_block_put(st_bh); - scoutfs_block_put(bm_bh); - return ret; -} - -/* Free a buddy block blkno in the super's bitmap block. */ -static int bitmap_free(struct super_block *sb, u64 blkno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_bitmap_block *bm; - struct buffer_head *bh; + struct scoutfs_buddy_block *bud = bh_data(bh); + u16 count; int nr; + int i; - /* mkfs should have ensured that there's bitmap blocks */ - /* XXX corruption */ - if (sbi->super.buddy_bm_ref.blkno == 0) - return -EIO; + scoutfs_block_zero(bh, sizeof(bud->hdr)); - bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_bm_ref); - if (IS_ERR(bh)) - return PTR_ERR(bh); - bm = bh_data(bh); + for (i = 0; i < ARRAY_SIZE(bud->first_set); i++) + bud->first_set[i] = cpu_to_le16(U16_MAX); - nr = blkno - (SCOUTFS_BUDDY_BM_BLKNO + SCOUTFS_BUDDY_BM_NR); - set_bit_le(nr, bm->bits); - scoutfs_block_put(bh); + bud->level = level; - return 0; + if (level) { + for (i = 0; i < SCOUTFS_BUDDY_SLOTS; i++) + set_slot_free_orders(bud, i, SCOUTFS_BUDDY_ORDER0_BITS); + } else { + /* ensure that there aren't multiple highest orders */ + BUILD_BUG_ON((SCOUTFS_BUDDY_ORDER0_BITS / + (1 << (SCOUTFS_BUDDY_ORDERS - 1))) > 1); + + count = SCOUTFS_BUDDY_ORDER0_BITS; + nr = 0; + for (i = SCOUTFS_BUDDY_ORDERS - 1; i >= 0; i--) { + if (count & (1 << i)) { + set_buddy_bit(bud, i, nr); + nr = (nr + 1) << 1; + } else { + nr <<= 1; + } + } + } } /* - * Give the caller a dirty buddy block. If the slot hasn't been used - * yet then we need to allocate and initialize a new block. + * Give the caller the block referenced by the given slot. They've + * calculated the blkno of the pair of blocks while walking the tree. + * The slot describes which of the pair its referencing. The caller is + * always going to modify the block so we always try and cow it. We + * construct a fake ref so we can re-use the block ref cow code. When + * we initialize the first use of a block we use the first of the pair. */ -static struct buffer_head *dirty_buddy_block(struct super_block *sb, - struct scoutfs_buddy_indirect *ind, - int sl, - struct scoutfs_buddy_slot *slot) +static struct buffer_head *get_buddy_block(struct super_block *sb, + struct scoutfs_buddy_slot *slot, + u64 blkno, int level) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; + struct buddy_info *binf = sbi->buddy_info; struct scoutfs_buddy_block *bud; + struct scoutfs_block_ref ref; struct buffer_head *bh; - u64 blkno; - int count; - int order; - int size; - int ret; - int nr; - /* the fast path is to dirty an existing block */ - if (slot->ref.blkno) - return scoutfs_block_dirty_ref(sb, &slot->ref); + trace_printk("getting block level %d blkno %llu slot seq %llu off %u\n", + level, blkno, le64_to_cpu(slot->seq), slot->blkno_off); - ret = bitmap_alloc(sb, &blkno); - if (ret) - return ERR_PTR(ret); - - bh = scoutfs_block_dirty(sb, blkno); - if (IS_ERR(bh)) { - bitmap_free(sb, blkno); - return bh; - } - bud = bh_data(bh); - scoutfs_block_zero(bh, sizeof(bud->hdr)); - - /* mark the initial run of highest orders free */ - count = slot_count(super, sl); - order = SCOUTFS_BUDDY_ORDERS - 1; - size = 1 << order; - nr = 0; - while (count > size) { - set_buddy_bit(ind, bud, order, nr); - nr++; - count -= size; + /* init a new block for an unused slot */ + if (slot->seq == 0) { + bh = scoutfs_block_dirty(sb, blkno); + if (!IS_ERR(bh)) + init_buddy_block(binf, super, bh, level); + } else { + /* construct block ref from tree walk blkno and slot ref */ + ref.blkno = cpu_to_le64(blkno + slot->blkno_off); + ref.seq = slot->seq; + bh = scoutfs_block_dirty_ref(sb, &ref); } - /* set order bits for each of the bits set in the remaining count */ - do { - if (count & (1 << order)) { - set_buddy_bit(ind, bud, order, nr); - nr = (nr + 1) << 1; - } else { - nr <<= 1; + if (!IS_ERR(bh)) { + bud = bh_data(bh); + + trace_printk("got blkno %llu\n", (u64)bh->b_blocknr); + + /* rebuild slot ref to blkno */ + if (slot->seq != bud->hdr.seq) { + slot->blkno_off = le64_to_cpu(bud->hdr.blkno) - blkno; + /* alloc_same only xors low bit */ + BUG_ON(slot->blkno_off > 1); + slot->seq = bud->hdr.seq; } - } while (order--); - - slot->ref.blkno = bud->hdr.blkno; - slot->ref.seq = bud->hdr.seq; - - update_free_orders(slot, bud); + } return bh; } /* - * Return the order bitmap offset and order of the first allocation - * that fits the desired order. + * Walk the buddy block radix to the leaf that contains either the given + * relative blk or the first free given order. The radix is of a fixed + * depth and we initialize new blocks as we descend through + * uninitialized refs. * - * Returns INT_MAX if there are no free orders. + * If order is -1 then we search for the blk. + * + * As we descend we calculate the base blk offset of the path we're + * taking down the tree. This is used to find the blkno of the next + * block relative to the blkno of the given level. It's then used by + * the caller to calculate the total blk offset by adding the bit they + * find in the block. + * + * The path through the tree is recorded in the stack in the buddy info. + * The caller is responsible for cleaning up the stack and must do so + * even if we return an error. */ -static int find_first_fit(struct scoutfs_super_block *super, int sl, - struct scoutfs_buddy_block *bud, - struct scoutfs_buddy_block *st_bud, - int order, int *order_ret) +static int buddy_walk(struct super_block *sb, u64 blk, int order, u64 *base) { - int nrs[SCOUTFS_BUDDY_ORDERS] = {0,}; - u64 blkno = U64_MAX; - bool made_progress; - int ret = INT_MAX; - u64 bno; - int nr; - int i; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_info *binf = sbi->buddy_info; + struct buddy_stack *sta = &binf->stack; + struct scoutfs_buddy_root *root = &sbi->super.buddy_root; + struct scoutfs_buddy_block *bud; + struct scoutfs_buddy_slot *slot; + struct buffer_head *bh; + u64 blkno; + int level; + int ret = 0; + int sl = 0; - do { - made_progress = false; - for (i = order; i < SCOUTFS_BUDDY_ORDERS; i++) { - /* find the next bit in each order */ - nr = find_next_buddy_bit(bud, i, nrs[i]); - nrs[i] = nr; - if (nr == INT_MAX) { - continue; - } - made_progress = true; + /* XXX corruption? */ + if (blk > last_blk(super) || root->height == 0 || + root->height > SCOUTFS_BUDDY_MAX_HEIGHT) + return -EIO; - /* advance to next bit if it's not free in stable */ - if (st_bud && - !test_buddy_bit_or_higher(st_bud, i, nr)) { - nrs[i] = nr + 1; - continue; - } + slot = &root->slot; + level = root->height; + blkno = SCOUTFS_BUDDY_BLKNO; + *base = 0; - /* use the first lowest order blkno */ - bno = slot_buddy_blkno(super, sl, i, nr); - if (bno < blkno) { - blkno = bno; - *order_ret = i; - ret = nr; - } + while (level--) { + /* XXX do base and level make sense here? */ + bh = get_buddy_block(sb, slot, blkno, level); + if (IS_ERR(bh)) { + ret = PTR_ERR(bh); + break; } - } while (ret == INT_MAX && made_progress); + trace_printk("before blk %llu order %d level %d blkno %llu base %llu sl %d\n", + blk, order, level, blkno, *base, sl); + + bud = bh_data(bh); + + if (level) { + if (order >= 0) { + /* find first slot with order free */ + sl = le16_to_cpu(bud->first_set[order]); + /* XXX corruption */ + if (sl == U16_MAX) { + ret = -EIO; + break; + } + } else { + /* find slot based on blk */ + sl = div64_u64_rem(blk, binf->level_div[level], + &blk); + } + + /* shouldn't be sl * 2, right? */ + *base = (*base * SCOUTFS_BUDDY_SLOTS) + sl; + /* this is the only place we * 2 */ + blkno = binf->level_blkno[level - 1] + (*base * 2); + slot = &bud->slots[sl]; + } else { + *base *= SCOUTFS_BUDDY_ORDER0_BITS; + /* sl in stack is 0 for final leaf block */ + sl = 0; + } + + trace_printk("after blk %llu order %d level %d blkno %llu base %llu sl %d\n", + blk, order, level, blkno, *base, sl); + + + stack_push(sta, bh, sl); + } + + trace_printk("walking ret %d\n", ret); return ret; } /* - * Find the first free region that satisfies the given order that is - * also free in the stable buddy bitmaps. This can return an allocation - * that breaks up a larger order. Higher level callers iterate over - * smaller orders to provide partial allocations. + * Find the order to search for to allocate a requested order. We try + * to use the smallest greater or equal order and then the largest + * smaller order. */ -static int alloc_slot(struct super_block *sb, - struct scoutfs_buddy_indirect *ind, int sl, - struct scoutfs_buddy_slot *slot, - struct scoutfs_block_ref *stable_ref, - u64 *blkno, int order) +static int find_free_order(struct scoutfs_buddy_root *root, int order) +{ + u16 free = le16_to_cpu(root->slot.free_orders); + u16 smaller_mask = (1 << order) - 1; + u16 larger = free & ~smaller_mask; + u16 smaller = free & smaller_mask; + + if (larger) + return ffs(larger) - 1; + if (smaller) + return fls(smaller) - 1; + + return -ENOSPC; +} + +/* + * Walk to the leaf that contains the found order and allocate a region + * of the given order, returning the relative blk to the caller. + */ +static int buddy_alloc(struct super_block *sb, u64 *blk, int order, int found) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; + struct buddy_info *binf = sbi->buddy_info; + struct buddy_stack *sta = &binf->stack; struct scoutfs_buddy_block *bud; - struct scoutfs_buddy_block *st_bud; - struct buffer_head *st_bh; struct buffer_head *bh; - int found; + u64 base; int ret; int nr; int i; - /* initialize or dirty the slot's buddy block */ - bh = dirty_buddy_block(sb, ind, sl, slot); - if (IS_ERR(bh)) - return PTR_ERR(bh); + trace_printk("alloc order %d found %d\n", order, found); + + if (WARN_ON_ONCE(found >= 0 && order > found)) + return -EINVAL; + + ret = buddy_walk(sb, *blk, found, &base); + if (ret) + goto out; + + bh = stack_peek(sta); bud = bh_data(bh); - /* read stable slots's buddy block if there is one */ - if (stable_ref->blkno) { - st_bh = scoutfs_block_read_ref(sb, stable_ref); - if (IS_ERR(st_bh)) { - ret = PTR_ERR(st_bh); + if (found >= 0) { + nr = le16_to_cpu(bud->first_set[found]); + /* XXX corruption */ + if (nr == U16_MAX) { + ret = -EIO; goto out; } - st_bud = bh_data(st_bh); + + /* give caller the found blk for the order */ + *blk = base + (nr << found); } else { - st_bh = NULL; - st_bud = NULL; + nr = buddy_bit(*blk) >> found; } - nr = find_first_fit(super, sl, bud, st_bud, order, &found); - if (nr == INT_MAX) { - ret = -ENOSPC; - goto out; + /* always allocate the higher or equal found order */ + clear_buddy_bit(bud, found, nr); + + /* and maybe free our buddies between smaller order and larger found */ + nr = buddy_bit(*blk) >> order; + for (i = order; i < found; i++) { + set_buddy_bit(bud, i, nr ^ 1); + nr >>= 1; } - /* we'll succeed from this point on, use nr before mangling it */ - *blkno = slot_buddy_blkno(super, sl, found, nr); - - /* always clear the found order */ - clear_buddy_bit(ind, bud, found, nr); - - /* free right buddies if we're breaking up a larger order */ - for (nr <<= 1, i = found - 1; i >= order; i--, nr <<= 1) - set_buddy_bit(ind, bud, i, nr | 1); - - update_free_orders(slot, bud); ret = 0; out: - scoutfs_block_put(st_bh); - scoutfs_block_put(bh); + trace_printk("alloc order %d found %d blk %llu ret %d\n", + order, found, *blk, ret); + stack_cleanup(sb); return ret; } /* - * Try and find a free block extent of the given order. We can fail to - * find a free order when none of the slots have free orders as the - * volume fills or gets fragmented. + * Free a given order by setting its order bit. If the order's buddy + * isn't set then it isn't free and we can't merge so we set our order + * and are done. If the buddy is free then we can clear it and ascend + * up to try and set the next higher order. That performs the same + * buddy merging test. Eventually we make it to the highest order which + * doesn't have a buddy so we can always set it. * - * We also have to be careful to only return free extents that were free - * in the old stable buddy allocator so that we don't allocate and write - * over referenced data. This can cause us to skip otherwise available - * extents but it should be rare. There can only be a transaction's - * worth of difference between the dirty allocator and the stable - * allocator. This is one of the motivations to cap the size of - * transactions. + * As we're freeing orders in the final buddy bitmap that only partially + * covers the end of the device we might try to test buddies which are + * past the end of the device. The test will still fall within the leaf + * block bitmap and those bits past the device will never be set so we + * will fail the merge and correctly set the orders free. */ -static int alloc_order(struct super_block *sb, u64 *blkno, int order) +static int buddy_free(struct super_block *sb, u64 blk, int order) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_buddy_indirect *st_ind; - struct scoutfs_buddy_indirect *ind; - struct buffer_head *st_bh = NULL; - struct buffer_head *bh = NULL; - u8 mask; - int ret; - int i; - - /* mkfs should have ensured that there's indirect blocks */ - if (sbi->super.buddy_ind_ref.blkno == 0 || - sbi->stable_super.buddy_ind_ref.blkno == 0) { - ret = -EIO; - goto out; - } - - /* get the dirty indirect block */ - bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_ind_ref); - if (IS_ERR(bh)) { - ret = PTR_ERR(bh); - goto out; - } - ind = bh_data(bh); - - /* get the stable indirect block */ - st_bh = scoutfs_block_read_ref(sb, &sbi->stable_super.buddy_ind_ref); - if (IS_ERR(st_bh)) { - ret = PTR_ERR(st_bh); - goto out; - } - st_ind = bh_data(st_bh); - - mask = ~0U << order; - - /* - * try to alloc from each slot that has at least the order free - * in both the dirty and stable buddy blocks. - */ - for (i = 0; i < SCOUTFS_BUDDY_SLOTS; i++) { - if (!((mask & ind->slots[i].free_orders) && - (mask & st_ind->slots[i].free_orders))) { - ret = -ENOSPC; - continue; - } - - ret = alloc_slot(sb, ind, i, &ind->slots[i], - &st_ind->slots[i].ref, blkno, order); - if (ret != -ENOSPC) - break; - } - -out: - scoutfs_block_put(st_bh); - scoutfs_block_put(bh); - - return ret; -} - -/* - * The buddy allocator keeps trying smaller orders until it finds an - * allocation. - * - * The order of the allocation is returned. - */ -static int buddy_alloc(struct super_block *sb, u64 *blkno, int order) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - int ret; - - if (WARN_ON_ONCE(order < 0 || order >= SCOUTFS_BUDDY_ORDERS)) - return -EINVAL; - - mutex_lock(&sbi->buddy_mutex); - - do { - ret = alloc_order(sb, blkno, order); - } while (ret == -ENOSPC && order--); - - mutex_unlock(&sbi->buddy_mutex); - - return ret ?: order; -} - -/* - * Allocate a block from the given region. The caller has the buddy - * mutex if we're called for either of the pair or bitmap internal - * regions. - */ -static int alloc_region(struct super_block *sb, u64 *blkno, int order, - u64 existing, int region) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - int ret; - - switch(region) { - case REGION_PAIR: - *blkno = existing ^ 1; - ret = 0; - break; - case REGION_BM: - ret = bitmap_alloc(sb, blkno); - break; - case REGION_BUDDY: - ret = buddy_alloc(sb, blkno, order); - break; - default: - WARN_ON_ONCE(1); - ret = -EINVAL; - } - - /* this misses other direct calls to bitmap_alloc, but that's minor */ - if (ret >= 0) - atomic_add(1 << ret, &sbi->buddy_count); - - trace_scoutfs_buddy_alloc(*blkno, order, region, ret); - return ret; -} - -int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order) -{ - return alloc_region(sb, blkno, order, 0, REGION_BUDDY); -} - -/* - * The block layer allocates from the same region as an existing blkno - * when it's allocating for cow. - */ -int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, int order, - u64 existing) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - - return alloc_region(sb, blkno, order, existing, - blkno_region(super, existing)); -} - -/* - * Free the aligned allocation of the given order at the given blkno to - * the allocator. We merge it into adjoining free space by looking for - * free buddies as we increase the order. - */ -static int buddy_free(struct super_block *sb, u64 blkno, int order) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_buddy_indirect *ind; + struct buddy_info *binf = sbi->buddy_info; + struct buddy_stack *sta = &binf->stack; struct scoutfs_buddy_block *bud; - struct buffer_head *ind_bh = NULL; - struct buffer_head *bh = NULL; + struct buffer_head *bh; + u64 unused; int ret; - int sl; int nr; int i; - if (WARN_ON_ONCE(order < 0 || order >= SCOUTFS_BUDDY_ORDERS) || - WARN_ON_ONCE(!valid_order(super, blkno, order))) - return -EINVAL; - - mutex_lock(&sbi->buddy_mutex); - - /* mkfs should have ensured that there's indirect blocks */ - if (sbi->super.buddy_ind_ref.blkno == 0) { - ret = -EIO; + ret = buddy_walk(sb, blk, -1, &unused); + if (ret) goto out; - } - ind_bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_ind_ref); - if (IS_ERR(ind_bh)) { - ret = PTR_ERR(ind_bh); - goto out; - } - ind = bh_data(ind_bh); - - sl = indirect_slot(super, blkno); - bh = scoutfs_block_dirty_ref(sb, &ind->slots[sl].ref); - if (IS_ERR(bh)) { - ret = PTR_ERR(bh); - goto out; - } + bh = stack_peek(sta); bud = bh_data(bh); - /* - * Merge our region with its free buddy and then try to merge - * that higher order region with its buddy, and so on, until the - * highest order. The highest order doesn't have buddies. - */ - nr = buddy_bit(super, blkno) >> order; - for (i = order; i < SCOUTFS_BUDDY_ORDERS - 1; i++) { + nr = buddy_bit(blk) >> order; + for (i = order; i < SCOUTFS_BUDDY_ORDERS - 2; i++) { if (!test_buddy_bit(bud, i, nr ^ 1)) break; - clear_buddy_bit(ind, bud, i, nr ^ 1); + clear_buddy_bit(bud, i, nr ^ 1); nr >>= 1; } - set_buddy_bit(ind, bud, i, nr); + set_buddy_bit(bud, i, nr); - update_free_orders(&ind->slots[sl], bud); - scoutfs_block_put(bh); ret = 0; out: - mutex_unlock(&sbi->buddy_mutex); - scoutfs_block_put(ind_bh); - + stack_cleanup(sb); return ret; } -int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order) +/* + * Try to allocate an extent with the size number of blocks. blkno is + * set to the start of the extent and the order of the block count is + * returned. + */ +int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - int region; + struct buddy_info *binf = sbi->buddy_info; + int found; + u64 blk; int ret; - region = blkno_region(super, blkno); - switch(blkno_region(super, blkno)) { - case REGION_PAIR: - ret = 0; - break; - case REGION_BM: - ret = bitmap_free(sb, blkno); - break; - case REGION_BUDDY: - ret = buddy_free(sb, blkno, order); - break; + trace_printk("order %d\n", order); + + mutex_lock(&binf->mutex); + + found = find_free_order(&super->buddy_root, order); + if (found < 0) { + ret = found; + goto out; } - trace_scoutfs_buddy_free(blkno, order, region, ret); + if (found < order) + order = found; + + blk = 0; + ret = buddy_alloc(sb, &blk, order, found); + if (ret) + goto out; + + *blkno = first_blkno(super) + blk; + le64_add_cpu(&super->free_blocks, -(1ULL << order)); + atomic_add((1ULL << order), &binf->alloc_count); + ret = order; + +out: + trace_printk("blkno %llu order %d ret %d\n", *blkno, order, ret); + mutex_unlock(&binf->mutex); return ret; } +/* + * We use the block _ref() routines to dirty existing blocks to reuse + * all the block verification and cow machinery. During cow this is + * called to allocate a new blkno to cow an existing buddy block. We + * use the existing blkno to see if we have to return the other mirrored + * buddy blkno or do a real allocation for every other kind of block + * being cowed. + */ +int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, u64 existing) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + + if (buddy_blkno(super, existing)) { + *blkno = existing ^ 1; + trace_printk("existing %llu ret blkno %llu\n", + existing, *blkno); + return 0; + } + + return scoutfs_buddy_alloc(sb, blkno, 0); +} + +struct extent_node { + struct rb_node node; + u64 start; + u64 len; +}; + +static int add_enode_extent(struct rb_root *root, u64 start, u64 len) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct extent_node *left = NULL; + struct extent_node *right = NULL; + struct extent_node *enode; + + trace_printk("adding enode [%llu,%llu]\n", start, len); + + while (*node && !(left && right)) { + parent = *node; + enode = container_of(*node, struct extent_node, node); + + if (start < enode->start) { + if (!right && start + len == enode->start) + right = enode; + node = &(*node)->rb_left; + } else { + if (!left && enode->start + enode->len == start) + left = enode; + node = &(*node)->rb_right; + } + } + + if (right) { + right->start = start; + right->len += len; + trace_printk("right now [%llu, %llu]\n", + right->start, right->len); + } + + if (left) { + if (right) { + left->len += right->len; + rb_erase(&right->node, root); + kfree(right); + } else { + left->len += len; + } + trace_printk("left now [%llu, %llu]\n", left->start, left->len); + } + + if (left || right) + return 0; + + enode = kmalloc(sizeof(struct extent_node), GFP_NOFS); + if (!enode) + return -ENOMEM; + + enode->start = start; + enode->len = len; + + trace_printk("inserted new [%llu, %llu]\n", enode->start, enode->len); + + rb_link_node(&enode->node, parent, node); + rb_insert_color(&enode->node, root); + + return 0; +} + +static void destroy_pending_frees(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; + struct extent_node *enode; + struct rb_node *node; + + for (node = rb_first(&binf->pending_frees); node;) { + enode = rb_entry(node, struct extent_node, node); + node = rb_next(node); + + rb_erase(&enode->node, &binf->pending_frees); + kfree(enode); + } +} + /* XXX this should be generic */ #define min3_t(t, a, b, c) min3((t)(a), (t)(b), (t)(c)) /* - * Free all the order allocations that make up the given unaligned block - * extent. Think of it as figuring out the largest aligned allocation - * that starts at the blkno and then clamping it by the count. + * Allocate or free all the orders that make up a given arbitrary block + * extent. Today this is used by callers who know that the blocks for + * the extent have already been pinned so we BUG on error. + */ +static void apply_extent(struct super_block *sb, bool alloc, u64 blk, u64 len) +{ + unsigned int blk_order; + unsigned int blk_bit; + unsigned int size; + int order; + int ret; + + trace_printk("applying extent blk %llu len %llu\n", blk, len); + + while (len) { + /* buddy bit might be 0, len always has a bit set */ + blk_bit = buddy_bit(blk); + blk_order = blk_bit ? ffs(blk_bit) - 1 : 0; + order = min3_t(int, blk_order, fls64(len) - 1, + SCOUTFS_BUDDY_ORDERS - 1); + size = 1 << order; + + trace_printk("applying blk %llu order %d\n", blk, order); + + if (alloc) + ret = buddy_alloc(sb, &blk, order, -1); + else + ret = buddy_free(sb, blk, order); + BUG_ON(ret); + + blk += size; + len -= size; + } +} + +/* + * The pending rbtree has recorded frees of stable data that we had to + * wait until transaction commit to record. Once these are tracked in + * the allocator we can't use the allocator until the commit succeeds. + * This is called by transaction commit to get these pending frees into + * the current commit. If it fails they pull them back out. + */ +int scoutfs_buddy_apply_pending(struct super_block *sb, bool alloc) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; + struct extent_node *enode; + struct rb_node *node; + + for (node = rb_first(&binf->pending_frees); node;) { + enode = rb_entry(node, struct extent_node, node); + node = rb_next(node); + + apply_extent(sb, alloc, enode->start, enode->len); + } + + return 0; +} + +/* + * Free a given allocated extent. The seq tells us which transaction + * first allocated the extent. If it was allocated in this transaction + * then we can return it to the free buddy and that must succeed. * - * For now this is only used by callers who have pinned the blocks that - * provided the allocation that they're now freeing from. It can't - * fail. If it could we would ensure that we re-alloc partial frees - * before returning an error. + * If it was allocated in a previous transaction then we dirty the + * blocks it will take to free it then record it in an rbtree. The + * rbtree entries are replayed into the dirty blocks as the transaction + * commits. + * + * Buddy block numbers are preallocated and calculated from the radix + * tree structure so we can ignore the block layer's calls to free buddy + * blocks during cow. + */ +int scoutfs_buddy_free(struct super_block *sb, __le64 seq, u64 blkno, int order) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_info *binf = sbi->buddy_info; + u64 unused; + u64 blk; + int ret; + + trace_printk("seq %llu blkno %llu order %d rsv %u\n", + le64_to_cpu(seq), blkno, order, buddy_blkno(super, blkno)); + + /* no specific free tracking for buddy blocks */ + if (buddy_blkno(super, blkno)) + return 0; + + /* XXX corruption? */ + if (!device_blkno(super, blkno)) + return -EINVAL; + + blk = blkno - first_blkno(super); + + if (!valid_order(blk, order)) + return -EINVAL; + + mutex_lock(&binf->mutex); + + if (seq == super->hdr.seq) { + ret = buddy_free(sb, blk, order); + /* + * If this order was allocated in this transaction then its + * blocks should be pinned and we should always be able + * to free it. + */ + BUG_ON(ret); + } else { + ret = buddy_walk(sb, blk, -1, &unused) ?: + add_enode_extent(&binf->pending_frees, blk, 1 << order); + if (ret == 0) + trace_printk("added blk %llu order %d\n", blk, order); + stack_cleanup(sb); + } + + if (ret == 0) + le64_add_cpu(&super->free_blocks, 1ULL << order); + + mutex_unlock(&binf->mutex); + + return ret; +} + +/* + * This is current only used to return partial extents from larger + * allocations in this transaction. */ void scoutfs_buddy_free_extent(struct super_block *sb, u64 blkno, u64 count) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; struct scoutfs_super_block *super = &sbi->stable_super; - int order; - int size; - int ret; + u64 blk; - while (count) { - /* both blkno and count have to have bits set */ - order = min3_t(int, __ffs64(buddy_bit(super, blkno)), - fls64(count) - 1, - SCOUTFS_BUDDY_ORDERS - 1); - size = 1 << order; + BUG_ON(!device_blkno(super, blkno)); - ret = scoutfs_buddy_free(sb, blkno, order); - BUG_ON(ret); + blk = blkno - first_blkno(super); - blkno += size; - count -= size; - } -} + mutex_lock(&binf->mutex); -/* - * Return > 1 if the given order allocation was free in the old stable - * transaction, 0 if it wasn't, and -errno if errors prevented us from - * finding out. - * - * XXX I bet we could get away without using the buddy mutex - */ -int scoutfs_buddy_was_free(struct super_block *sb, u64 blkno, int order) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->stable_super; - struct buffer_head *ind_bh = NULL; - struct buffer_head *bh = NULL; - struct scoutfs_buddy_indirect *ind; - struct scoutfs_buddy_block *bud; - struct scoutfs_block_ref *ref; - int ret; - int nr; - int sl; - - /* mkfs should have ensured that there's bitmap blocks */ - /* XXX corruption */ - if (sbi->super.buddy_bm_ref.blkno == 0 || - sbi->stable_super.buddy_bm_ref.blkno == 0) - return -EIO; - - mutex_lock(&sbi->buddy_mutex); - - /* get the stable indirect block */ - ind_bh = scoutfs_block_read_ref(sb, &super->buddy_ind_ref); - if (IS_ERR(ind_bh)) { - ret = PTR_ERR(ind_bh); - goto out; - } - ind = bh_data(ind_bh); - - /* allocation was free if it's slot wasn't populated */ - sl = indirect_slot(super, blkno); - ref = &ind->slots[sl].ref; - if (!ref->blkno) { - ret = 1; - goto out; - } - - /* check the allocation bit in the old stable bitmap block */ - bh = scoutfs_block_read_ref(sb, ref); - if (IS_ERR(bh)) { - ret = PTR_ERR(bh); - goto out; - } - bud = bh_data(bh); - - nr = buddy_bit(super, blkno) >> order; - ret = !!test_buddy_bit_or_higher(bud, order, nr); - -out: - mutex_unlock(&sbi->buddy_mutex); - scoutfs_block_put(ind_bh); - scoutfs_block_put(bh); - - trace_printk("blkno %llu order %d ret %d\n", blkno, order, ret); - return ret; -} - -/* - * For now we only have one indirect block off the super. When we grow - * multiple commit block pairs that reference root and indirect blocks - * then we'll need to iterate over those. These results will only ever - * be approximate so we can simply use racey valid ref reads to be able - * to sample while others are writing. - */ -int scoutfs_buddy_bfree(struct super_block *sb, u64 *bfree) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_buddy_indirect *ind; - struct buffer_head *bh; - int ret; - int i; - - *bfree = 0; - - bh = scoutfs_block_read_ref(sb, &super->buddy_ind_ref); - if (IS_ERR(bh)) { - ret = PTR_ERR(bh); - goto out; - } - ind = bh_data(bh); - - for (i = 0; i < SCOUTFS_BUDDY_ORDERS; i++) - *bfree += le64_to_cpu(ind->order_totals[i]) << i; - - scoutfs_block_put(bh); - ret = 0; -out: - return ret; + apply_extent(sb, false, blkno - first_blkno(super), count); + le64_add_cpu(&super->free_blocks, count); + mutex_unlock(&binf->mutex); } /* * Return the number of block allocations since the last time the - * counter was reset. This count doesn't include some internal bitmap - * block allocations but that should be a small fraction of the main - * allocations. + * counter was reset. This count doesn't include dirty buddy blocks. */ unsigned int scoutfs_buddy_alloc_count(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; - return atomic_read(&sbi->buddy_count); + return atomic_read(&binf->alloc_count); } -void scoutfs_buddy_reset_count(struct super_block *sb) +u64 scoutfs_buddy_bfree(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; + struct scoutfs_super_block *super = &sbi->super; + u64 ret; - return atomic_set(&sbi->buddy_count, 0); + mutex_lock(&binf->mutex); + ret = le64_to_cpu(super->free_blocks); + mutex_unlock(&binf->mutex); + + return ret; } + +void scoutfs_buddy_committed(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; + + atomic_set(&binf->alloc_count, 0); + destroy_pending_frees(sb); +} + +int scoutfs_buddy_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_info *binf = sbi->buddy_info; + u64 level_blocks[SCOUTFS_BUDDY_MAX_HEIGHT]; + u64 blocks; + int i; + + /* first bit offsets in blocks are __le16 */ + BUILD_BUG_ON(SCOUTFS_BUDDY_ORDER0_BITS >= U16_MAX); + + /* bits need to be naturally aligned to long for _le bitops */ + BUILD_BUG_ON(offsetof(struct scoutfs_buddy_block, bits) & + (sizeof(long) - 1)); + + binf = kzalloc(sizeof(struct buddy_info), GFP_KERNEL); + if (!binf) + return -ENOMEM; + sbi->buddy_info = binf; + + mutex_init(&binf->mutex); + atomic_set(&binf->alloc_count, 0); + binf->pending_frees = RB_ROOT; + + /* calculate blocks at each level */ + blocks = DIV_ROUND_UP_ULL(last_blk(super) + 1, + SCOUTFS_BUDDY_ORDER0_BITS); + for (i = 0; i < SCOUTFS_BUDDY_MAX_HEIGHT; i++) { + level_blocks[i] = (blocks * 2); + if (blocks == 1) { + binf->max_height = i + 1; + break; + } + blocks = DIV_ROUND_UP_ULL(blocks, SCOUTFS_BUDDY_SLOTS); + } + + /* calculate device blkno of first block in each level */ + binf->level_blkno[binf->max_height - 1] = SCOUTFS_BUDDY_BLKNO; + for (i = (binf->max_height - 2); i >= 0; i--) { + binf->level_blkno[i] = binf->level_blkno[i + 1] + + level_blocks[i + 1]; + } + + /* calculate blk divisor to find slot at a given level */ + binf->level_div[1] = SCOUTFS_BUDDY_ORDER0_BITS; + for (i = 2; i < binf->max_height; i++) { + binf->level_div[i] = binf->level_div[i - 1] * + SCOUTFS_BUDDY_SLOTS; + } + + for (i = 0; i < binf->max_height; i++) + trace_printk("level %d div %llu blkno %llu blocks %llu\n", + i, binf->level_div[i], binf->level_blkno[i], + level_blocks[i]); + + return 0; +} + +void scoutfs_buddy_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; + + if (binf) + WARN_ON_ONCE(!RB_EMPTY_ROOT(&binf->pending_frees)); + kfree(binf); +} + diff --git a/kmod/src/buddy.h b/kmod/src/buddy.h index 059d586a..24c0ed0c 100644 --- a/kmod/src/buddy.h +++ b/kmod/src/buddy.h @@ -2,15 +2,19 @@ #define _SCOUTFS_BUDDY_H_ int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order); -int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, int order, - u64 existing); -int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order); +int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, u64 existing); +int scoutfs_buddy_free(struct super_block *sb, __le64 seq, u64 blkno, + int order); void scoutfs_buddy_free_extent(struct super_block *sb, u64 blkno, u64 count); int scoutfs_buddy_was_free(struct super_block *sb, u64 blkno, int order); -int scoutfs_buddy_bfree(struct super_block *sb, u64 *bfree); +u64 scoutfs_buddy_bfree(struct super_block *sb); unsigned int scoutfs_buddy_alloc_count(struct super_block *sb); -void scoutfs_buddy_reset_count(struct super_block *sb); +int scoutfs_buddy_apply_pending(struct super_block *sb, bool alloc); +void scoutfs_buddy_committed(struct super_block *sb); + +int scoutfs_buddy_setup(struct super_block *sb); +void scoutfs_buddy_destroy(struct super_block *sb); #endif diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 597ce144..abd754a7 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -147,7 +147,7 @@ static int alloc_file_block(struct super_block *sb, u64 *blkno) spin_unlock(&sbi->file_alloc_lock); if (order > 0) - scoutfs_buddy_free(sb, alloc_blkno, order); + scoutfs_buddy_free(sb, sbi->super.hdr.seq, alloc_blkno, order); out: trace_printk("allocated blkno %llu ret %d\n", *blkno, ret); @@ -246,7 +246,7 @@ int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size) if (blkno == 0) continue; - ret = scoutfs_buddy_free(sb, blkno, 0); + ret = scoutfs_buddy_free(sb, bmap.seq[i], blkno, 0); if (ret) break; @@ -357,6 +357,8 @@ static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno) static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) { struct super_block *sb = inode->i_sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->stable_super; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_block_map bmap; struct scoutfs_btree_val val; @@ -406,7 +408,7 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) goto out; if (old_blkno) { - ret = scoutfs_buddy_free(sb, old_blkno, 0); + ret = scoutfs_buddy_free(sb, bmap.seq[i], old_blkno, 0); if (ret) goto out; } diff --git a/kmod/src/format.h b/kmod/src/format.h index 58fea85f..fe83a346 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -19,8 +19,7 @@ */ #define SCOUTFS_SUPER_BLKNO ((64 * 1024) >> SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_SUPER_NR 2 -#define SCOUTFS_BUDDY_BM_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR) -#define SCOUTFS_BUDDY_BM_NR 2 +#define SCOUTFS_BUDDY_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR) #define SCOUTFS_MAX_TRANS_BLOCKS (128 * 1024 * 1024 / SCOUTFS_BLOCK_SIZE) @@ -48,42 +47,49 @@ struct scoutfs_block_ref { __le64 seq; } __packed; -struct scoutfs_bitmap_block { - struct scoutfs_block_header hdr; - __le64 bits[0]; -} __packed; - /* - * Track allocations from BLOCK_SIZE to (BLOCK_SIZE << ..._ORDERS). + * If the block was full of bits the largest possible order would be + * the block size shift + 3 (BITS_PER_BYTE). But the header uses + * up some space and then the buddy bits mean two bits per block. + * Then +1 for this being the number, not the greatest order. */ -#define SCOUTFS_BUDDY_ORDERS 8 +#define SCOUTFS_BUDDY_ORDERS (SCOUTFS_BLOCK_SHIFT + 3 - 2 + 1) struct scoutfs_buddy_block { struct scoutfs_block_header hdr; - __le32 order_counts[SCOUTFS_BUDDY_ORDERS]; - __le64 bits[0]; + __le16 first_set[SCOUTFS_BUDDY_ORDERS]; + __u8 level; + __u8 __pad[3]; /* naturally align bits */ + union { + struct scoutfs_buddy_slot { + __le64 seq; + __le16 free_orders; + /* XXX seems like we could hide a bit somewhere */ + __u8 blkno_off; + } __packed slots[0]; + __le64 bits[0]; + } __packed; } __packed; /* - * If we had log2(raw bits) orders we'd fully use all of the raw bits in - * the block. We're close enough that the amount of space wasted at the - * end (~1/256th of the block, ~64 bytes) isn't worth worrying about. + * Each buddy leaf block references order 0 blocks with half of its + * bitmap. The other half of the bits are used for the higher order + * bits. */ #define SCOUTFS_BUDDY_ORDER0_BITS \ (((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_block)) * 8) / 2) -struct scoutfs_buddy_indirect { - struct scoutfs_block_header hdr; - __le64 order_totals[SCOUTFS_BUDDY_ORDERS]; - struct scoutfs_buddy_slot { - __u8 free_orders; - struct scoutfs_block_ref ref; - } slots[0]; +#define SCOUTFS_BUDDY_SLOTS \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_block)) / \ + sizeof(struct scoutfs_buddy_slot)) + +struct scoutfs_buddy_root { + struct scoutfs_buddy_slot slot; + __u8 height; } __packed; -#define SCOUTFS_BUDDY_SLOTS \ - ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_indirect)) / \ - sizeof(struct scoutfs_buddy_slot)) +/* ((SCOUTFS_BUDDY_SLOTS^5) * SCOUTFS_BUDDY_ORDER0_BITS) > 2^52 */ +#define SCOUTFS_BUDDY_MAX_HEIGHT 6 /* * We should be able to make the offset smaller if neither dirents nor @@ -180,10 +186,10 @@ struct scoutfs_super_block { __u8 uuid[SCOUTFS_UUID_BYTES]; __le64 next_ino; __le64 total_blocks; - __le32 buddy_blocks; + __le64 free_blocks; + __le64 buddy_blocks; + struct scoutfs_buddy_root buddy_root; struct scoutfs_btree_root btree_root; - struct scoutfs_block_ref buddy_ind_ref; - struct scoutfs_block_ref buddy_bm_ref; } __packed; #define SCOUTFS_ROOT_INO 1 diff --git a/kmod/src/super.c b/kmod/src/super.c index a668f8a2..b6d94a42 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -48,12 +48,8 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; __le32 * __packed uuid = (void *)super->uuid; - int ret; - - ret = scoutfs_buddy_bfree(sb, &kst->f_bfree); - if (ret) - return ret; + kst->f_bfree = scoutfs_buddy_bfree(sb); kst->f_type = SCOUTFS_SUPER_MAGIC; kst->f_bsize = SCOUTFS_BLOCK_SIZE; kst->f_blocks = le64_to_cpu(super->total_blocks); @@ -198,8 +194,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) sbi->block_dirty_tree = RB_ROOT; init_waitqueue_head(&sbi->block_wq); atomic_set(&sbi->block_writes, 0); - mutex_init(&sbi->buddy_mutex); - atomic_set(&sbi->buddy_count, 0); init_rwsem(&sbi->btree_rwsem); atomic_set(&sbi->trans_holds, 0); init_waitqueue_head(&sbi->trans_hold_wq); @@ -220,6 +214,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) ret = scoutfs_setup_counters(sb) ?: read_supers(sb) ?: + scoutfs_buddy_setup(sb) ?: scoutfs_setup_trans(sb); if (ret) return ret; @@ -252,6 +247,7 @@ static void scoutfs_kill_sb(struct super_block *sb) kill_block_super(sb); if (sbi) { scoutfs_shutdown_trans(sb); + scoutfs_buddy_destroy(sb); scoutfs_destroy_counters(sb); if (sbi->kset) kset_unregister(sbi->kset); diff --git a/kmod/src/super.h b/kmod/src/super.h index 604e0231..9247f917 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -8,7 +8,7 @@ #include "buddy.h" struct scoutfs_counters; -struct buddy_alloc; +struct buddy_info; struct scoutfs_sb_info { struct super_block *sb; @@ -24,8 +24,7 @@ struct scoutfs_sb_info { atomic_t block_writes; int block_write_err; - struct mutex buddy_mutex; - atomic_t buddy_count; + struct buddy_info *buddy_info; struct rw_semaphore btree_rwsem; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index e2679dd7..d089a1d4 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -90,17 +90,20 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_filerw_free_alloc(sb); - ret = scoutfs_block_write_dirty(sb) ?: + ret = scoutfs_buddy_apply_pending(sb, false) ?: + scoutfs_block_write_dirty(sb) ?: scoutfs_write_dirty_super(sb); - if (!ret) + if (ret) { + scoutfs_buddy_apply_pending(sb, true); + } else { + scoutfs_buddy_committed(sb); advance = 1; + } } spin_lock(&sbi->trans_write_lock); - if (advance) { + if (advance) scoutfs_advance_dirty_super(sb); - scoutfs_buddy_reset_count(sb); - } sbi->trans_write_count++; sbi->trans_write_ret = ret; spin_unlock(&sbi->trans_write_lock);