diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 95f35039..6749f25b 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -2,5 +2,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += block.o btree.o counters.o crc.o dir.o filerw.o inode.o msg.o \ - scoutfs_trace.o super.o trans.o treap.o +scoutfs-y += block.o btree.o buddy.o counters.o crc.o dir.o filerw.o \ + inode.o msg.o scoutfs_trace.o super.o trans.o treap.o diff --git a/kmod/src/block.c b/kmod/src/block.c index efaff13c..70ec7a57 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -21,6 +21,7 @@ #include "block.h" #include "crc.h" #include "counters.h" +#include "buddy.h" #define DIRTY_RADIX_TAG 0 @@ -428,20 +429,19 @@ struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, unsigned long flags; u64 blkno; int ret; + int err; bl = scoutfs_read_block(sb, le64_to_cpu(ref->blkno)); if (IS_ERR(bl) || ref->seq == sbi->super.hdr.seq) return bl; - ret = radix_tree_preload(GFP_NOFS); - if (ret) { - scoutfs_put_block(bl); - return ERR_PTR(ret); - } + ret = scoutfs_buddy_alloc(sb, &blkno, 0); + if (ret < 0) + goto out; - /* XXX cheesy */ - blkno = atomic64_inc_return(&sbi->next_blkno); - hdr = bl->data; + ret = radix_tree_preload(GFP_NOFS); + if (ret) + goto out; spin_lock_irqsave(&sbi->block_lock, flags); @@ -453,8 +453,10 @@ struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, } bl->blkno = blkno; + hdr = bl->data; hdr->blkno = cpu_to_le64(blkno); hdr->seq = sbi->super.hdr.seq; + radix_tree_insert(&sbi->block_radix, blkno, bl); radix_tree_tag_set(&sbi->block_radix, blkno, DIRTY_RADIX_TAG); atomic_inc(&bl->refcount); @@ -464,6 +466,16 @@ struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, ref->blkno = hdr->blkno; ref->seq = hdr->seq; + ret = 0; +out: + if (ret) { + if (blkno) { + err = scoutfs_buddy_free(sb, blkno, 0); + WARN_ON_ONCE(err); /* XXX hmm */ + } + scoutfs_put_block(bl); + bl = ERR_PTR(ret); + } return bl; } @@ -530,13 +542,21 @@ out: */ struct scoutfs_block *scoutfs_alloc_block(struct super_block *sb) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_block *bl; u64 blkno; + int ret; + int err; - /* XXX cheesy */ - blkno = atomic64_inc_return(&sbi->next_blkno); + ret = scoutfs_buddy_alloc(sb, &blkno, 0); + if (ret < 0) + return ERR_PTR(ret); - return scoutfs_new_block(sb, blkno); + bl = scoutfs_new_block(sb, blkno); + if (IS_ERR(bl)) { + err = scoutfs_buddy_free(sb, blkno, 0); + WARN_ON_ONCE(err); /* XXX hmm */ + } + return bl; } void scoutfs_calc_hdr_crc(struct scoutfs_block *bl) @@ -545,3 +565,12 @@ void scoutfs_calc_hdr_crc(struct scoutfs_block *bl) hdr->crc = cpu_to_le32(scoutfs_crc_block(hdr)); } + +void scoutfs_zero_block_tail(struct scoutfs_block *bl, size_t off) +{ + if (WARN_ON_ONCE(off > SCOUTFS_BLOCK_SIZE)) + return; + + if (off < SCOUTFS_BLOCK_SIZE) + memset(bl->data + off, 0, SCOUTFS_BLOCK_SIZE - off); +} diff --git a/kmod/src/block.h b/kmod/src/block.h index 724f4aca..91f55f81 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -37,5 +37,6 @@ int scoutfs_write_dirty_blocks(struct super_block *sb); void scoutfs_put_block(struct scoutfs_block *bl); void scoutfs_calc_hdr_crc(struct scoutfs_block *bl); +void scoutfs_zero_block_tail(struct scoutfs_block *bl, size_t off); #endif diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c new file mode 100644 index 00000000..47d89504 --- /dev/null +++ b/kmod/src/buddy.c @@ -0,0 +1,670 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include + +#include "super.h" +#include "format.h" +#include "block.h" +#include "buddy.h" +#include "msg.h" + +/* + * scoutfs uses buddy bitmaps to allocate block regions. It has a nice + * and simple implementation and reasonably small storage and memory + * overhead, particularly in the pathological fragmented case, but + * results in more rigid allocation constraints and fragmentation. + * + * The buddy allocator is build from a hierarchy of bitmaps for each + * power of two order of blocks that we can allocate. If a high order + * buddy bit is set then all the lower order bits that it covers are + * clear. + * + * At runtime all the bitmaps for all the orders are stored in a single + * packed bitmap in memory. We construct an array of pointers into the + * big bitmap for each individual order bitmap. This lets us easily + * track modifications of all the order bitmaps with a second bitmap + * which tracks fixed size chunks of the main bitmap. + * + * As a transaction is written the modified chunks of the main bitmap + * are written to the tail of a preallocated ring of buddy blocks. + * This turns noisy scattered bit modification operations into one large + * contiguous block IO. + * + * We always write to the tail of the ring so we need to ensure that the + * blocks at the tail don't contain live data. As we mark each chunk of + * the bitmap modified during a transaction we also sweep through the + * bitmap finding another chunk that has never been modified by the + * current sweep. Eventually enough chunks are modified by transactions + * to advance the sweep through the whole bitmap. At this point we're + * sure that all the blocks written to the tail during the sweep have to + * contain the full bitmap. By sizing the ring to 4x the bitmap size we + * ensure that we'll finish the sweep in each half, ensuring that the + * tail is always far enough behind the head to not overwrite live + * chunks. + * + * The entire ring is read the first time the allocator is needed. + * Today that's on mount for the entire system. As we layer on + * functionality we'll have multiple allocators and they'll be passed + * around the cluster as mounts are given access. As mounts get access + * they only need to read the newly written blocks in the ring to bring + * their stale allocator up to date with recent modifications written to + * the tail. The ring indices are full 64bits so that readers can + * recognize when they need to read the whole ring. + * + * The allocator only covers the blocks after the ring blocks to the end + * of the device. When we move to multiple allocators each will cover a + * fixed set of blocks excluding their ring blocks. Resizing will + * change the number of allocators needed to cover the device and will + * modify the bits in a final allocator. The bitmap modifications for + * resizing would be written to ring blocks as usual. Care will be + * taken to recognize device sizes whose final blocks land in the ring + * blocks. + */ + +struct buddy_alloc { + + /* + * addr: pointer to le64 that contains the start of the bitmap + * addr_bit: full bit nr of lsb at addr + * addr_off: bit offset from addr to first order bit + * addr_size: bit count from addr of the order's bits + * first_set: first logical order bit offset that might be set + */ + struct buddy_order { + __le64 *addr; + long addr_bit; + long addr_off; + long addr_size; + long first_set; + } orders[64]; + + int max_order; + + u64 orig_tail; + long *modified; + long modified_size; + + long reserved_chunks; + + __le64 *bitmap; +}; + +/* return the first device blkno covered by the allocator */ +static u64 first_blkno(struct scoutfs_super_block *super) +{ + return SCOUTFS_BUDDY_BLKNO + le32_to_cpu(super->buddy_blocks); +} + +/* return the number of blocks addressible by the allocator. */ +static u64 covered_blocks(struct scoutfs_super_block *super) +{ + return le64_to_cpu(super->total_blocks) - first_blkno(super); +} + +/* return the device block number of a ring index */ +static u64 ring_blkno(struct scoutfs_super_block *super, u64 index) +{ + return SCOUTFS_BUDDY_BLKNO + + do_div(index, le32_to_cpu(super->buddy_blocks)); +} + +/* + * Find and mark the next chunk in the bitmap that has never been + * written to the current half of the block ring. + * + * If we finish the sweep through the bitmap then we know that the most + * current half of the ring contain the full bitmap and reading at the + * head no longer has to start from the previous half. + */ +static bool modify_sweep_bit(struct scoutfs_super_block *super, + struct buddy_alloc *bud) +{ + bool did_set; + long bit; + + bit = le32_to_cpu(super->buddy_sweep_bit); + if (bit >= bud->modified_size) + return false; + + bit = find_next_zero_bit(bud->modified, bud->modified_size, bit); + if (bit < bud->modified_size) { + set_bit(bit, bud->modified); + bud->reserved_chunks--; + bit++; + did_set = true; + } else { + bit = bud->modified_size; + did_set = false; + } + + super->buddy_sweep_bit = cpu_to_le32(bit); + + /* advance head once we finish the sweep */ + if (bit == bud->modified_size) { + u64 head = le64_to_cpu(super->buddy_head); + u64 tail = le64_to_cpu(super->buddy_tail); + u32 half = le32_to_cpu(super->buddy_blocks) / 2; + + if ((tail - head) > half) + le64_add_cpu(&super->buddy_head, half); + } + + return did_set; +} + +/* + * The caller has modified the given bit in the full buddy bitmap. We + * try to mark its chunk modified and advance the sweep through older + * chunks. + */ +static void modified_bit(struct scoutfs_super_block *super, + struct buddy_alloc *bud, int order, long bit) +{ + struct buddy_order *ord = &bud->orders[order]; + + bit = (ord->addr_bit + ord->addr_off + bit) / SCOUTFS_BUDDY_CHUNK_BITS; + + if (!test_and_set_bit(bit, bud->modified)) { + bud->reserved_chunks--; + modify_sweep_bit(super, bud); + } +} + +static int test_buddy_bit(struct buddy_alloc *bud, int order, long bit) +{ + struct buddy_order *ord = &bud->orders[order]; + + return !!test_bit_le(ord->addr_off + bit, ord->addr); +} + +static void set_buddy_bit(struct scoutfs_super_block *super, + struct buddy_alloc *bud, int order, long bit) +{ + struct buddy_order *ord = &bud->orders[order]; + + set_bit_le(ord->addr_off + bit, ord->addr); + ord->first_set = min(bit, ord->first_set); + + modified_bit(super, bud, order, bit); +} + +static void clear_buddy_bit(struct scoutfs_super_block *super, + struct buddy_alloc *bud, int order, long bit) +{ + struct buddy_order *ord = &bud->orders[order]; + + clear_bit_le(ord->addr_off + bit, ord->addr); + if (ord->first_set == bit) + ord->first_set++; + + modified_bit(super, bud, order, bit); +} + +/* returns LONG_MAX when there are no bits set */ +static long find_first_buddy_bit(struct buddy_alloc *bud, int order) +{ + struct buddy_order *ord = &bud->orders[order]; + long ret; + + ret = find_next_bit_le(ord->addr, ord->addr_size, + ord->addr_off + ord->first_set); + if (ret >= ord->addr_size) { + ret = LONG_MAX; + ord->first_set = ord->addr_size - ord->addr_off; + } else { + ret -= ord->addr_off; + ord->first_set = ret; + } + + return ret; +} + +/* test if the index is at the first block in either half of the ring */ +static bool start_of_half(struct scoutfs_super_block *super, u64 index) +{ + u32 half = le32_to_cpu(super->buddy_blocks) / 2; + + return do_div(index, half) == 0; +} + +/* + * A buddy operation can modify bits at every order in the worst case. + * (This is a bit overly conservative because high orders will + * eventually share a chunk.) We'll also try to mark old chunks + * modified for each new chunk we modify. + * + * Before we modify the buddy bits we pin dirty blocks to make sure that + * we have enough chunks to store the modified chunks. + * + * As we advance the tail to store new blocks we might wander into the + * next half of the ring. When that happens we reset the sweep bit so + * that we'll start migrating chunks into this new half of the ring. + * + * This is called with the buddy mutex held. It's the only thing that + * does blocking work under the mutex so we could be more clever and + * make the allocation fast path locking more efficient. + */ +static int reserve_block_chunks(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_alloc *bud = sbi->bud; + struct scoutfs_block *bl; + u64 blkno; + + if (bud->reserved_chunks >= (bud->max_order * 2)) + return 0; + + blkno = ring_blkno(super, le64_to_cpu(super->buddy_tail)); + bl = scoutfs_new_block(sb, blkno); + if (IS_ERR(bl)) + return PTR_ERR(bl); + + scoutfs_put_block(bl); + bud->reserved_chunks += SCOUTFS_BUDDY_CHUNKS_PER_BLOCK; + le64_add_cpu(&super->buddy_tail, 1); + if (start_of_half(super, le64_to_cpu(super->buddy_tail))) + super->buddy_sweep_bit = 0; + + return 0; +} + +/* + * Return the block number of an allocation of at least the requested + * order. If an allocation at the given order isn't free then first try + * to satisfy the allocation with a part of a larger order, then return + * a smaller allocation. + * + * The order of the allocation is returned. + */ +int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_alloc *bud = sbi->bud; + int found; + long bit; + int ret; + int i; + + if (WARN_ON_ONCE(order < 0 || order > bud->max_order)) + return -EINVAL; + + mutex_lock(&sbi->buddy_mutex); + + ret = reserve_block_chunks(sb); + if (ret) + goto out; + + /* search for larger and smaller orders */ + i = order; + while (i >= 0) { + bit = find_first_buddy_bit(bud, i); + if (bit < LONG_MAX) + break; + + if (i >= order && i < bud->max_order) + i++; + else if (i == bud->max_order) + i = order - 1; + else + i--; + } + if (i < 0) { + ret = -ENOSPC; + goto out; + } + found = i; + + /* we'll succeed from this point on, use bit before mangling it */ + *blkno = first_blkno(super) + ((u64)bit << found); + ret = min(found, order); + + /* always clear the found order */ + clear_buddy_bit(super, bud, found, bit); + + /* free right buddies if we're breaking up a larger order */ + for (bit <<= 1, i = found - 1; i >= order; i--, bit <<= 1) + set_buddy_bit(super, bud, i, bit | 1); + +out: + mutex_unlock(&sbi->buddy_mutex); + if (WARN_ON_ONCE(ret < 0)) + *blkno = 0; + return ret; +} + +/* + * Free the aligned allocation of the given order at the given blkno to + * the allocator. We merge it into adjoining free space by looking for + * free buddies as we increase the order. + */ +int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_alloc *bud = sbi->bud; + long bit; + int ret; + int i; + + if (WARN_ON_ONCE(order < 0 || order > bud->max_order) || + WARN_ON_ONCE(((blkno + 1) << order) >= covered_blocks(super))) + return -EINVAL; + + mutex_lock(&sbi->buddy_mutex); + + ret = reserve_block_chunks(sb); + if (ret) + goto out; + + bit = (blkno - first_blkno(super)) >> order; + for (i = order; i <= bud->max_order; i++) { + + /* set bit free and finish when buddy isn't free */ + if (!test_buddy_bit(bud, i, bit ^ 1)) { + set_buddy_bit(super, bud, i, bit); + break; + } + + /* otherwise clear buddy and try to set higher parent */ + clear_buddy_bit(super, bud, i, bit ^ 1); + bit >>= 1; + } + +out: + mutex_unlock(&sbi->buddy_mutex); + return ret; +} + +/* + * We're writing a transaction. The buddy allocator records chunks of + * the main bitmap which have been modified during the transaction. We + * copy them to the pinned dirty blocks which will be written as part of + * the transaction. The bitmap of modified chunks and the old ring tail + * are only reset when the transaction is successfully written. + */ +int scoutfs_dirty_buddy_chunks(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_alloc *bud = sbi->bud; + struct scoutfs_buddy_chunk *chunk; + struct scoutfs_buddy_block *bb; + struct scoutfs_block *bl; + long bit; + long ind; + u64 tail; + int i; + + /* short circuit a transaction with no modified chunks */ + if (bud->orig_tail == le64_to_cpu(super->buddy_tail)) + return 0; + + while (bud->reserved_chunks && modify_sweep_bit(super, bud)) + ; + + for (tail = bud->orig_tail, bit = 0; + tail < le64_to_cpu(super->buddy_tail) && bit < bud->modified_size; + tail++) { + + bl = scoutfs_read_block(sb, ring_blkno(super, tail)); + if (WARN_ON_ONCE(IS_ERR(bl))) + return PTR_ERR(bl); + + bb = bl->data; + bb->hdr.seq = cpu_to_le64(tail); + bb->nr_chunks = 0; + + for (i = 0; i < SCOUTFS_BUDDY_CHUNKS_PER_BLOCK; i++) { + bit = find_next_bit(bud->modified, bud->modified_size, + bit); + if (bit >= bud->modified_size) + break; + + chunk = &bb->chunks[i]; + chunk->pos = cpu_to_le32(bit); + ind = bit * SCOUTFS_BUDDY_CHUNK_LE64S; + memcpy(chunk->bits, &bud->bitmap[ind], + SCOUTFS_BUDDY_CHUNK_BYTES); + bit++; + } + + bb->nr_chunks = i; + scoutfs_zero_block_tail(bl, offsetof(struct scoutfs_buddy_block, + chunks[bb->nr_chunks])); + scoutfs_put_block(bl); + } + + /* + * Chunk reservation should have ensured that there's always room + * in the tail blocks for the modified chunks. + */ + if (WARN_ON_ONCE(bit < bud->modified_size)) + return -EIO; + + return 0; +} + +void scoutfs_reset_buddy_chunks(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_alloc *bud = sbi->bud; + + bud->orig_tail = le64_to_cpu(super->buddy_tail); + memset(bud->modified, 0, DIV_ROUND_UP(bud->modified_size, 8)); +} + +static int check_buddy_fields(struct super_block *sb, + struct scoutfs_super_block *super) +{ + u32 blocks = le32_to_cpu(super->buddy_blocks); + u32 half = blocks / 2; + u64 head = le64_to_cpu(super->buddy_head); + u64 tail = le64_to_cpu(super->buddy_tail); + u64 buddy_bits; + u64 chunk_bits; + + /* have to at least have two halves */ + if (blocks < 2) { + scoutfs_info(sb, "buddy_blocks %lu must be at least 2", blocks); + return -EIO; + } + + /* + * insist that blocks be a multiple of two so that we don't have + * scary fencepost off by ones around the half calculations. + */ + if (blocks & 1) { + scoutfs_info(sb, "buddy_blocks %lu isn't even", blocks); + return -EIO; + } + + /* shouldn't fill the device with buddy blocks */ + if (first_blkno(super) >= le64_to_cpu(super->total_blocks)) { + scoutfs_info(sb, "buddy_blocks %lu must be at least 2", blocks); + return -EIO; + } + + /* can only reference a 32bit long's worth of buddy bits */ + buddy_bits = covered_blocks(super) * 2; + if (buddy_bits >= INT_MAX) { + scoutfs_info(sb, "device needs %llu > INT_MAX buddy bits", + buddy_bits); + return -EIO; + } + + /* need enough ring blocks for 4 full buddy copies */ + chunk_bits = blocks * SCOUTFS_BUDDY_CHUNKS_PER_BLOCK * + SCOUTFS_BUDDY_CHUNK_BITS; + if (buddy_bits * 4 > chunk_bits) { + scoutfs_info(sb, "only room for %llu bits in chunks, need %llu", + chunk_bits, buddy_bits * 4); + return -EIO; + } + + if (head > tail) { + scoutfs_info(sb, "buddy_head %llu > buddy_tail %llu", + head, tail); + return -EIO; + } + + /* tail can't wrap around into head */ + if ((tail - head) >= blocks) { + scoutfs_info(sb, "buddy_tail %llu overlaps buddy_head %llu", + tail, head); + return -EIO; + } + + /* head always has to start one of the halves */ + if (!start_of_half(super, head)) { + scoutfs_info(sb, "buddy_head %llu isn't multiple of half %u", + head, half); + return -EIO; + } + + return 0; +} + +/* + * Reconstruct the entire buddy bitmap by replaying the chunks that are + * contained in the buddy block ring. + * + * The allocator doesn't cover the super blocks and ring blocks and is + * initialized with all the device blocks marked free so that mkfs + * doesn't have to write any chunks to initialize free space. + * + * We go a little nuts with variables to make it easier to read. + */ +int scoutfs_read_buddy_chunks(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_buddy_chunk *chunk; + struct scoutfs_buddy_block *bb; + struct scoutfs_block *bl; + struct buddy_alloc *bud; + struct buddy_order *ord; + u64 buddy_bits; + u64 dev_blocks; + u64 chunks; + u64 head; + u64 tail; + long bits; + long bit; + long ind; + int ret; + int i; + + ret = check_buddy_fields(sb, super); + if (ret) + return ret; + + dev_blocks = covered_blocks(super); + buddy_bits = dev_blocks * 2; + chunks = DIV_ROUND_UP(buddy_bits, SCOUTFS_BUDDY_CHUNK_BITS); + + bud = kzalloc(sizeof(struct buddy_alloc), GFP_KERNEL); + if (bud) { + bud->bitmap = vzalloc(round_up(buddy_bits, 64) / 8); + bud->modified = vzalloc(round_up(chunks, BITS_PER_LONG) / 8); + } + if (!bud || !bud->bitmap || !bud->modified) { + ret = -ENOMEM; + goto out; + } + sbi->bud = bud; + + bud->modified_size = chunks; + + /* + * Updating first_set across the orders would be tricky so we + * initialize it to 0 and suffer an initial expensive find_first + * call. + */ + bit = 0; + bits = dev_blocks; + for (i = 0; i < ARRAY_SIZE(bud->orders); i++) { + ord = &bud->orders[i]; + + ord->addr = &bud->bitmap[bit / 64]; + ord->addr_bit = bit & ~63ULL; + ord->addr_off = bit & 63; + ord->addr_size = ord->addr_off + bits; + ord->first_set = 0; + + bit += bits; + bits >>= 1; + if (!bits) + break; + } + bud->max_order = i; + + /* + * Initialize the allocator with the all the blocks covered by + * the fewest number of greatest order free allocations. Ring + * replay will overwrite this. + */ + bit = 0; + for (i = bud->max_order; i >= 0; i--) { + ord = &bud->orders[i]; + + if (ord->addr_off + bit == ord->addr_size) + break; + + set_bit_le(ord->addr_off + bit, ord->addr); + bit = (bit + 1) << 1; + } + + head = le64_to_cpu(super->buddy_head); + tail = le64_to_cpu(super->buddy_tail); + while (head < tail) { + bl = scoutfs_read_block(sb, ring_blkno(super, head)); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + goto out; + } + + bb = bl->data; + if (le64_to_cpu(bb->hdr.seq) != head) { + /* XXX corruption */ + ret = -EIO; + scoutfs_put_block(bl); + goto out; + } + + for (i = 0; i < bb->nr_chunks; i++) { + chunk = &bb->chunks[i]; + + /* XXX check */ + ind = le32_to_cpu(chunk->pos) * + SCOUTFS_BUDDY_CHUNK_LE64S; + + memcpy(&bud->bitmap[ind], chunk->bits, + SCOUTFS_BUDDY_CHUNK_BYTES); + } + scoutfs_put_block(bl); + head++; + } + ret = 0; +out: + if (ret) { + if (bud) { + vfree(bud->bitmap); + vfree(bud->modified); + } + } + return ret; +} diff --git a/kmod/src/buddy.h b/kmod/src/buddy.h new file mode 100644 index 00000000..8a411178 --- /dev/null +++ b/kmod/src/buddy.h @@ -0,0 +1,11 @@ +#ifndef _SCOUTFS_BUDDY_H_ +#define _SCOUTFS_BUDDY_H_ + +int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order); +int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order); + +int scoutfs_read_buddy_chunks(struct super_block *sb); +void scoutfs_reset_buddy_chunks(struct super_block *sb); +int scoutfs_dirty_buddy_chunks(struct super_block *sb); + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index a80e95b0..6f80059e 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -19,6 +19,7 @@ */ #define SCOUTFS_SUPER_BLKNO ((64 * 1024) >> SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_SUPER_NR 2 +#define SCOUTFS_BUDDY_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR) /* * This header is found at the start of every block so that we can @@ -102,6 +103,38 @@ struct scoutfs_btree_item { #define SCOUTFS_UUID_BYTES 16 +/* + * Arbitrarily choose a reasonably fine grained 64byte chunk. This is a + * balance between write amplification of writing chunks with a single + * modified bit, storage overhead of partial blocks losing a chunk to + * make room for the block header and having a pos field per chunk, and + * runtime memory overhead of a bit per chunk. + */ +#define SCOUTFS_BUDDY_CHUNK_LE64S 8 +#define SCOUTFS_BUDDY_CHUNK_BYTES (SCOUTFS_BUDDY_CHUNK_LE64S * 8) +#define SCOUTFS_BUDDY_CHUNK_BITS (SCOUTFS_BUDDY_CHUNK_BYTES * 8) + +/* + * After the pair of super blocks are a preallocated ring of blocks + * which record modified regions of the buddy bitmap allocator. + * + * The seq's header needs to match the unwrapped ring index of the + * block. + */ +struct scoutfs_buddy_block { + struct scoutfs_block_header hdr; + u8 nr_chunks; + struct scoutfs_buddy_chunk { + __le32 pos; + __le64 bits[SCOUTFS_BUDDY_CHUNK_LE64S]; + } __packed chunks[0]; +} __packed; + +#define SCOUTFS_BUDDY_CHUNKS_PER_BLOCK \ + ((SCOUTFS_BLOCK_SIZE - offsetof(struct scoutfs_buddy_block, chunks)) /\ + SCOUTFS_BUDDY_CHUNK_BYTES) + + /* * The super is stored in a pair of blocks in the first chunk on the * device. @@ -116,6 +149,11 @@ struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; __u8 uuid[SCOUTFS_UUID_BYTES]; + __le64 total_blocks; + __le32 buddy_blocks; + __le32 buddy_sweep_bit; + __le64 buddy_head; + __le64 buddy_tail; struct scoutfs_btree_root btree_root; } __packed; diff --git a/kmod/src/super.c b/kmod/src/super.c index b79c6b8f..e8354d5e 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -126,7 +126,6 @@ static int read_supers(struct super_block *sb) * XXX These don't exist in the super yet. They should soon. */ atomic64_set(&sbi->next_ino, SCOUTFS_ROOT_INO + 1); - atomic64_set(&sbi->next_blkno, 6); return 0; } @@ -151,6 +150,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) INIT_RADIX_TREE(&sbi->block_radix, GFP_NOFS); init_waitqueue_head(&sbi->block_wq); atomic_set(&sbi->block_writes, 0); + mutex_init(&sbi->buddy_mutex); init_rwsem(&sbi->btree_rwsem); atomic_set(&sbi->trans_holds, 0); init_waitqueue_head(&sbi->trans_hold_wq); @@ -165,11 +165,13 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) ret = scoutfs_setup_counters(sb) ?: read_supers(sb) ?: - scoutfs_setup_trans(sb); + scoutfs_setup_trans(sb) ?: + scoutfs_read_buddy_chunks(sb); if (ret) return ret; scoutfs_advance_dirty_super(sb); + scoutfs_reset_buddy_chunks(sb); inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO); if (IS_ERR(inode)) diff --git a/kmod/src/super.h b/kmod/src/super.h index df2dd30f..18c9be86 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -5,8 +5,10 @@ #include #include "format.h" +#include "buddy.h" struct scoutfs_counters; +struct buddy_alloc; struct scoutfs_sb_info { struct super_block *sb; @@ -20,7 +22,9 @@ struct scoutfs_sb_info { int block_write_err; atomic64_t next_ino; - atomic64_t next_blkno; + + struct mutex buddy_mutex; + struct buddy_alloc *bud; /* XXX there will be a lot more of these :) */ struct rw_semaphore btree_rwsem; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 7194e9af..4dcfdb99 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -19,6 +19,7 @@ #include "super.h" #include "block.h" #include "trans.h" +#include "buddy.h" #include "scoutfs_trace.h" /* @@ -63,7 +64,8 @@ void scoutfs_trans_write_func(struct work_struct *work) /* XXX probably want to write out dirty pages in inodes */ if (scoutfs_has_dirty_blocks(sb)) { - ret = scoutfs_write_dirty_blocks(sb) ?: + ret = scoutfs_dirty_buddy_chunks(sb) ?: + scoutfs_write_dirty_blocks(sb) ?: scoutfs_write_dirty_super(sb); if (!ret) advance = 1; @@ -71,8 +73,10 @@ void scoutfs_trans_write_func(struct work_struct *work) spin_lock(&sbi->trans_write_lock); - if (advance) + if (advance) { scoutfs_advance_dirty_super(sb); + scoutfs_reset_buddy_chunks(sb); + } sbi->trans_write_count++; sbi->trans_write_ret = ret; spin_unlock(&sbi->trans_write_lock);