diff --git a/kmod/src/balloc.c b/kmod/src/balloc.c deleted file mode 100644 index bcd03bd6..00000000 --- a/kmod/src/balloc.c +++ /dev/null @@ -1,630 +0,0 @@ -/* - * Copyright (C) 2019 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "key.h" -#include "counters.h" -#include "msg.h" -#include "block.h" -#include "btree.h" -#include "per_task.h" -#include "balloc.h" - -#include "scoutfs_trace.h" - -/* - * scoutfs tracks free metadata blocks in bitmap items in allocation - * btrees. Most of the free metadata is operated on by the server and - * tracked in large core trees rooted in the super block. The server - * moves free items from the core trees to private trees for mounts. - * - * Allocation is performed by btrees which are performing cow updates. - * We can't write to stable blocks during a transaction, we can only - * write into free space in the previous stable fs image. This means - * that we can't satisfy dirty block allocations with frees of - * previously stable blocks in this transaction. We implement this by - * allocating from one tree and freeing into another. They're merged as - * the free blocks are committed and can be safely written to in the - * next transaction. - * - * We're allocating and freeing blocks on behalf of btree ops by calling - * btree ops. This would deadlock if we always called btree ops from - * the allocator directly, but instead we recognize recursion and have - * the called allocator hand blknos back to its calling allocator to - * store into btrees on its behalf. - * - * We use explicit allocation and writing contexts because both the - * client and server are working on independent allocation and item - * trees. - */ - -struct item_modification { - struct list_head entry; - u64 blkno; - u64 count; - int op; - struct scoutfs_balloc_root *root; - struct scoutfs_balloc_root *src; -}; - -static bool add_item_mod(struct list_head *list, int op, u64 blkno, u64 count, - struct scoutfs_balloc_root *root, - struct scoutfs_balloc_root *src) -{ - struct item_modification *im = kmalloc(sizeof(struct item_modification), - GFP_NOFS); - if (im) { - im->blkno = blkno; - im->count = count; - im->op = op; - im->root = root; - im->src = src; - list_add_tail(&im->entry, list); - return true; - } - - return false; -} - -/* make room to dirty two trees of an absurdly large height */ -#define MAX_BLKNOS (2 * ((32 * 2) + 1)) - -struct blkno_fifo { - int first; - int nr; - u64 blknos[MAX_BLKNOS]; -}; - -static inline void blkno_fifo_init(struct blkno_fifo *bf) -{ - bf->first = 0; - bf->nr = 0; -} - -static inline int blkno_fifo_nr(struct blkno_fifo *bf) -{ - BUG_ON(bf->nr < 0 || bf->nr > MAX_BLKNOS); - return bf->nr; -} - -static inline u64 blkno_fifo_out(struct blkno_fifo *bf) -{ - BUG_ON(blkno_fifo_nr(bf) == 0); - bf->nr--; - return bf->blknos[bf->first++]; -} - -static inline void blkno_fifo_in(struct blkno_fifo *bf, u64 blkno) -{ - unsigned int end = (bf->first + bf->nr) % MAX_BLKNOS; - - BUG_ON(blkno_fifo_nr(bf) == MAX_BLKNOS); - bf->blknos[end] = blkno; - bf->nr++; -} - -struct caller_blknos { - struct blkno_fifo free; - struct blkno_fifo alloced; - struct blkno_fifo freed; -}; - -/* - * Find a number of next free blknos from a starting point. We can land - * in the end of an empty item. If this returns 0 then nr_found have - * been found. - */ -static int find_next_free(struct super_block *sb, - struct scoutfs_balloc_root *root, u64 from, - u64 *found, unsigned int nr_found) -{ - struct scoutfs_balloc_item_key bik; - struct scoutfs_balloc_item_val biv; - SCOUTFS_BTREE_ITEM_REF(iref); - unsigned int f = 0; - unsigned int bit; - u64 base; - int ret = 0; - - while (f < nr_found) { - base = from >> SCOUTFS_BALLOC_ITEM_BASE_SHIFT; - bit = from & SCOUTFS_BALLOC_ITEM_BIT_MASK; - bik.base = cpu_to_be64(base); - - ret = scoutfs_btree_next(sb, &root->root, - &bik, sizeof(bik), &iref); - if (ret < 0) /* including ENOENT */ - break; - - if (iref.key_len == sizeof(bik) && - iref.val_len == sizeof(biv)) { - memcpy(&bik, iref.key, iref.key_len); - memcpy(&biv, iref.val, iref.val_len); - - /* start from first bit in next whole item */ - if (be64_to_cpu(bik.base) != base) - bit = 0; - - while (f < nr_found) { - bit = find_next_bit_le(biv.bits, - SCOUTFS_BALLOC_ITEM_BITS, bit); - if (bit >= SCOUTFS_BALLOC_ITEM_BITS) - break; - - found[f++] = (be64_to_cpu(bik.base) << - SCOUTFS_BALLOC_ITEM_BASE_SHIFT) + - bit; - bit++; - } - - from = (be64_to_cpu(bik.base) << - SCOUTFS_BALLOC_ITEM_BASE_SHIFT) + bit; - ret = 0; - - } else { - ret = -EIO; - } - scoutfs_btree_put_iref(&iref); - if (ret < 0) - break; - } - - return ret; -} - -/* - * Return the first blkno in the next item. Because from can land in an - * item we can return a blkno that is less than from. - */ -static int find_next_item(struct super_block *sb, - struct scoutfs_balloc_root *root, u64 from, - u64 *found) -{ - struct scoutfs_balloc_item_key bik; - SCOUTFS_BTREE_ITEM_REF(iref); - u64 base; - int ret; - - base = from >> SCOUTFS_BALLOC_ITEM_BASE_SHIFT; - bik.base = cpu_to_be64(base); - - ret = scoutfs_btree_next(sb, &root->root, &bik, sizeof(bik), &iref); - if (ret < 0) /* including ENOENT */ - goto out; - - if (iref.key_len == sizeof(struct scoutfs_balloc_item_key) && - iref.val_len == sizeof(struct scoutfs_balloc_item_val)) { - memcpy(&bik, iref.key, iref.key_len); - *found = be64_to_cpu(bik.base) << - SCOUTFS_BALLOC_ITEM_BASE_SHIFT; - ret = 0; - } else { - ret = -EIO; - } - scoutfs_btree_put_iref(&iref); -out: - return ret; -} - -enum { - IM_OP_SET, - IM_OP_SET_BULK, - IM_OP_CLEAR, - IM_OP_MOVE, -}; - -static int copy_item_bits(struct scoutfs_balloc_item_val *biv, - struct scoutfs_btree_item_ref *iref, int ret, - bool *existed) -{ - if (ret < 0) { - if (ret == -ENOENT) { - memset(biv, 0, sizeof(struct scoutfs_balloc_item_val)); - if (existed) - *existed = false; - ret = 0; - } - } else { - if (iref->key_len == sizeof(struct scoutfs_balloc_item_key) && - iref->val_len == sizeof(struct scoutfs_balloc_item_val)) { - memcpy(biv, iref->val, iref->val_len); - if (existed) - *existed = true; - } else { - ret = -EIO; - } - scoutfs_btree_put_iref(iref); - } - - return ret; -} - -/* - * We can use native longs to set aligned 64bit regions, but have to use - * individual _le calls on leading and trailing partial regions. - */ -static void bitmap_set_le(__le64 *map, int start, int nr) -{ - unsigned int full; - - while (start & 63 && nr-- > 0) - set_bit_le(start++, map); - - if (nr > 64) { - full = round_down(nr, 64); - bitmap_set((long *)map, start, full); - start += full; - nr -= full; - - } - - while (nr-- > 0) - set_bit_le(start++, map); -} - -/* - * Modify allocation item bits in service of the caller's operation. - * This has to be done very carefully so that we don't deadlock in - * recursion as btree dirtying calls back in to block allocation. - * - * A given btree operation can need to allocate blknos for dirty blocks - * and free the old clean blknos. The btree code will attempt to call - * balloc again. We add a per_task record of allocated and freed blknos - * which those allocation calls use instead of calling more btree ops. - * They then return to us and we perform the btree ops to satisfy those - * allocations and frees that were recorded. - * - * Each op that cows btree blocks generates more ops to records those - * allocations and frees. Eventually the ops hit existing dirty blocks - * and we can return. - */ -static int modify_items(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - int op, u64 blkno, u64 count, - struct scoutfs_balloc_root *root, - struct scoutfs_balloc_root *src, u64 next_free) -{ - SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); - struct scoutfs_balloc_item_key bik; - struct scoutfs_balloc_item_val biv; - struct scoutfs_balloc_item_val tmp; - struct item_modification *im; - SCOUTFS_BTREE_ITEM_REF(iref); - struct caller_blknos *cb; - unsigned int need_free; - unsigned int nr; - LIST_HEAD(mods); - u64 nexts[16]; - bool existed; - u64 base; - int bit; - int ret; - int i; - - /* doing native long ops on stack bits */ - BUILD_BUG_ON(offsetof(struct scoutfs_balloc_item_val, bits) % - (BITS_PER_LONG / 8)); - - cb = kmalloc(sizeof(struct caller_blknos), GFP_NOFS); - if (!cb) { - ret = -ENOMEM; - goto out; - } - - blkno_fifo_init(&cb->free); - blkno_fifo_init(&cb->alloced); - blkno_fifo_init(&cb->freed); - - scoutfs_per_task_add(&alloc->pt_caller_blknos, &pt_ent, cb); - - if (!add_item_mod(&mods, op, blkno, count, root, src)) { - ret = -ENOENT; - goto out; - } - - while ((im = list_first_entry_or_null(&mods, struct item_modification, - entry))) { - - base = im->blkno >> SCOUTFS_BALLOC_ITEM_BASE_SHIFT; - bik.base = cpu_to_be64(base); - existed = false; - - if (im->op != IM_OP_SET_BULK) { - /* get the current item to modify */ - ret = scoutfs_btree_lookup(sb, &im->root->root, - &bik, sizeof(bik), &iref); - ret = copy_item_bits(&biv, &iref, ret, &existed); - if (ret < 0) - goto out; - /* XXX corruption */ - BUG_ON(im->op == IM_OP_CLEAR && !existed); - } - - /* modify the item's bit */ - bit = im->blkno & SCOUTFS_BALLOC_ITEM_BIT_MASK; - if (im->op == IM_OP_SET) { - set_bit_le(bit, &biv.bits); - } else if (im->op == IM_OP_SET_BULK) { - memset(&biv, 0, sizeof(biv)); - bitmap_set_le(biv.bits, 0, im->count); - } else if (im->op == IM_OP_CLEAR) { - clear_bit_le(bit, &biv.bits); - } - - /* move just read the destination item, or in src item bits */ - if (im->op == IM_OP_MOVE) { - ret = scoutfs_btree_lookup(sb, &im->src->root, &bik, - sizeof(bik), &iref); - ret = copy_item_bits(&tmp, &iref, ret, NULL); - if (ret < 0) - goto out; - - /* shouldn't have free in both places */ - if (bitmap_intersects((long *)biv.bits, - (long *)tmp.bits, - SCOUTFS_BALLOC_ITEM_BITS)) { - ret = -EIO; - goto out; - } - bitmap_or((long *)biv.bits, (long *)biv.bits, - (long *)tmp.bits, SCOUTFS_BALLOC_ITEM_BITS); - } - - /* make sure we have enough free blocks for btree dirtying */ - need_free = (im->root->root.height * 2) + 1; - if (im->op == IM_OP_MOVE) - need_free += (im->src->root.height * 2) + 1; - - /* fill free fifo for potential dirtying */ - while (blkno_fifo_nr(&cb->free) < need_free) { - nr = min_t(int, need_free - blkno_fifo_nr(&cb->free), - ARRAY_SIZE(nexts)); - ret = find_next_free(sb, &alloc->alloc_root, next_free, - nexts, nr); - if (ret < 0) - goto out; - - next_free = nexts[nr - 1] + 1; - for (i = 0; i < nr; i++) - blkno_fifo_in(&cb->free, nexts[i]); - } - - /* - * Perform the op's item modifications, we go to do the - * trouble of differentiating between update and - * insertion instead of just using force so that we - * don't split when we don't need to. - */ - if (im->op == IM_OP_CLEAR && - bitmap_empty((long *)biv.bits, SCOUTFS_BALLOC_ITEM_BITS)) - ret = scoutfs_btree_delete(sb, alloc, wri, - &im->root->root, - &bik, sizeof(bik)); - else if (im->op == IM_OP_SET_BULK || - (im->op == IM_OP_SET && !existed)) - ret = scoutfs_btree_insert(sb, alloc, wri, - &im->root->root, - &bik, sizeof(bik), - &biv, sizeof(biv)); - else if (im->op == IM_OP_MOVE && existed) - ret = scoutfs_btree_delete(sb, alloc, wri, - &im->src->root, - &bik, sizeof(bik)) ?: - scoutfs_btree_update(sb, alloc, wri, - &im->root->root, - &bik, sizeof(bik), - &biv, sizeof(biv)); - else if (im->op == IM_OP_MOVE && !existed) - ret = scoutfs_btree_delete(sb, alloc, wri, - &im->src->root, - &bik, sizeof(bik)) ?: - scoutfs_btree_insert(sb, alloc, wri, - &im->root->root, - &bik, sizeof(bik), - &biv, sizeof(biv)); - else - ret = scoutfs_btree_update(sb, alloc, wri, - &im->root->root, - &bik, sizeof(bik), - &biv, sizeof(biv)); - if (ret < 0) - goto out; - - /* update bit counts to reflect op */ - if (im->op == IM_OP_SET) { - le64_add_cpu(&root->total_free, 1); - } else if (im->op == IM_OP_SET_BULK) { - le64_add_cpu(&root->total_free, im->count); - } else if (im->op == IM_OP_CLEAR) { - le64_add_cpu(&root->total_free, -1); - } else if (im->op == IM_OP_MOVE) { - nr = bitmap_weight((long *)biv.bits, - SCOUTFS_BALLOC_ITEM_BITS); - le64_add_cpu(&root->total_free, nr); - le64_add_cpu(&src->total_free, -nr); - } - - list_del(&im->entry); - kfree(im); - - /* and queue new modifications needed from btree ops */ - - while (blkno_fifo_nr(&cb->alloced)) { - if (!add_item_mod(&mods, IM_OP_CLEAR, - blkno_fifo_out(&cb->alloced), 0, - &alloc->alloc_root, NULL)) { - ret = -ENOENT; - goto out; - } - } - - while (blkno_fifo_nr(&cb->freed)) { - if (!add_item_mod(&mods, IM_OP_SET, - blkno_fifo_out(&cb->freed), 0, - &alloc->free_root, NULL)) { - ret = -ENOENT; - goto out; - } - } - } - - ret = 0; -out: - scoutfs_per_task_del(&alloc->pt_caller_blknos, &pt_ent); - BUG_ON(ret < 0); /* dirty block refs and bits are inconsistent */ - BUG_ON(!list_empty(&mods)); /* reminder to clean up */ - kfree(cb); - return ret; -} - -void scoutfs_balloc_init(struct scoutfs_balloc_allocator *alloc, - struct scoutfs_balloc_root *alloc_root, - struct scoutfs_balloc_root *free_root) -{ - mutex_init(&alloc->mutex); - scoutfs_per_task_init(&alloc->pt_caller_blknos); - alloc->alloc_root = *alloc_root; - alloc->free_root = *free_root; -} - -/* - * Add alloc items for a contiugous regions of blknos. The starting - * blkno must be aligned to the start of a bitmap item. Once these are - * added they can be used by the current transaction so the caller must - * be very careful that they're free. - */ -int scoutfs_balloc_add_alloc_bulk(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - u64 blkno, u64 count) -{ - u64 nr; - int ret = 0; - - mutex_lock(&alloc->mutex); - while (count > 0) { - nr = min_t(u64, count, SCOUTFS_BALLOC_ITEM_BITS), - ret = modify_items(sb, alloc, wri, IM_OP_SET_BULK, blkno, nr, - &alloc->alloc_root, NULL, 0); - if (ret < 0) - break; - blkno += nr; - count -= nr; - } - mutex_unlock(&alloc->mutex); - - return ret; -} - -int scoutfs_balloc_alloc(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, u64 *blkno_ret) -{ - struct caller_blknos *cb; - u64 next; - int ret; - - /* if we're called by balloc then the caller works for us */ - cb = scoutfs_per_task_get(&alloc->pt_caller_blknos); - if (cb) { - *blkno_ret = blkno_fifo_out(&cb->free); - blkno_fifo_in(&cb->alloced, *blkno_ret); - return 0; - } - - mutex_lock(&alloc->mutex); - ret = find_next_free(sb, &alloc->alloc_root, 0, &next, 1) ?: - modify_items(sb, alloc, wri, IM_OP_CLEAR, next, 0, - &alloc->alloc_root, NULL, next + 1); - mutex_unlock(&alloc->mutex); - - if (ret == 0) - *blkno_ret = next; - - return ret; -} - -int scoutfs_balloc_free(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - u64 blkno) -{ - struct caller_blknos *cb; - int ret; - - /* if we're called by balloc then the caller works for us */ - cb = scoutfs_per_task_get(&alloc->pt_caller_blknos); - if (cb) { - blkno_fifo_in(&cb->freed, blkno); - return 0; - } - - mutex_lock(&alloc->mutex); - ret = modify_items(sb, alloc, wri, IM_OP_SET, blkno, 0, - &alloc->free_root, NULL, 0); - mutex_unlock(&alloc->mutex); - - return ret; -} - -/* - * Move full items from the source to destination tree, moving at least - * the given number of blocks but likely more. - * - * This has to be done very carefully because we don't want to allocate - * dirty btree blocks from blknos in the source item that is moving. We - * find the first blkno in the next free item in the source tree so that - * we can start allocating dirty btree blocks after that item. - * - * This will not wrap the starting from blkno if it doesn't start at 0 - * and runs out of items. The caller is expected to deal with this. - */ -int scoutfs_balloc_move(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_balloc_root *dst, - struct scoutfs_balloc_root *src, - u64 from, u64 at_least, u64 *next_past) -{ - u64 target; - u64 next; - int ret = 0; - - mutex_lock(&alloc->mutex); - - target = le64_to_cpu(dst->total_free) + at_least; - - while (le64_to_cpu(dst->total_free) < target && - le64_to_cpu(src->total_free) > 0) { - ret = find_next_item(sb, src, from, &next) ?: - modify_items(sb, alloc, wri, IM_OP_MOVE, next, 0, - dst, src, - next + SCOUTFS_BALLOC_ITEM_BITS); - if (ret < 0) - break; - - from = next + SCOUTFS_BALLOC_ITEM_BITS; - *next_past = from; - } - - mutex_unlock(&alloc->mutex); - - return ret; -} diff --git a/kmod/src/balloc.h b/kmod/src/balloc.h deleted file mode 100644 index 246a42a6..00000000 --- a/kmod/src/balloc.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef _SCOUTFS_BALLOC_H_ -#define _SCOUTFS_BALLOC_H_ - -#include "per_task.h" - -struct scoutfs_block_writer; - -struct scoutfs_balloc_allocator { - struct mutex mutex; - struct scoutfs_per_task pt_caller_blknos; - struct scoutfs_balloc_root alloc_root; - struct scoutfs_balloc_root free_root; -}; - -void scoutfs_balloc_init(struct scoutfs_balloc_allocator *alloc, - struct scoutfs_balloc_root *alloc_root, - struct scoutfs_balloc_root *free_root); -int scoutfs_balloc_add_alloc_bulk(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - u64 blkno, u64 count); -int scoutfs_balloc_alloc(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, u64 *blkno_ret); -int scoutfs_balloc_free(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - u64 blkno); -int scoutfs_balloc_move(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_balloc_root *dst, - struct scoutfs_balloc_root *src, - u64 from, u64 at_least, u64 *next_past); - -#endif diff --git a/kmod/src/data.c b/kmod/src/data.c index 4eaf701e..0750b441 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -689,599 +689,6 @@ static int set_extent(struct super_block *sb, struct inode *inode, return 0; } -#if 0 -static bool block_bitmap_fits(u64 blkno, u64 count) -{ - return ((blkno & SCOUTFS_BLOCK_BITMAP_BIT_MASK) + count) <= - SCOUTFS_BLOCK_BITMAP_BITS; -} - -static void block_bitmap_bit(u64 *base, int *bit, u64 blkno, u8 type) -{ - if (type == SCOUTFS_BLOCK_BITMAP_BIG) - blkno >>= SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; - - *bit = blkno & SCOUTFS_BLOCK_BITMAP_BIT_MASK; - *base = blkno >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; -} - -static u64 block_bitmap_blkno(u64 base, int bit, u8 type) -{ - u64 blkno; - - blkno = (base << SCOUTFS_BLOCK_BITMAP_BASE_SHIFT) + bit; - - if (type == SCOUTFS_BLOCK_BITMAP_BIG) - blkno <<= SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; - - return blkno; -} - -struct block_bitmap { - u64 base; - u8 type; - bool exists; - unsigned long bits[DIV_ROUND_UP(SCOUTFS_BLOCK_BITMAP_BITS, - BITS_PER_LONG)]; -}; - -static inline __le64 long_bits_to_le64(unsigned long *bits, unsigned int i) -{ -#if BITS_PER_LONG == 64 - return cpu_to_le64(bits[i]); -#elif BITS_PER_LONG == 32 - i <<= 1; - return cpu_to_le64(bits[i] | ((u64)bits[i + 1] << 32)); -#else -#error "unexpected BITS_PER_LONG value?" -#endif -} - -static inline void u64_to_long_bits(unsigned long *bits, unsigned int i, u64 x) -{ -#if BITS_PER_LONG == 64 - bits[i] = x; -#else - i <<= 1; - bits[i] = x; - bits[i + 1] = x >> 32; -#endif -} - -/* - * Block bitmaps are unpacked into native long bitmaps in memory for use - * with the kernel's bitmap functions. This requires a bit of finesse - * to make sure that we translate the bits appropriately to - * architectures with different word size and endian. - */ -static int unpack_block_bitmap(struct block_bitmap *bb, - struct scoutfs_btree_item_ref *iref) -{ - struct scoutfs_block_bitmap_key *bbk; - struct scoutfs_packed_bitmap *pb; - unsigned int nr; - u64 present; - u64 set; - u64 b; - int ret; - int w; - int i; - - if (iref->key_len != sizeof(struct scoutfs_block_bitmap_key) || - iref->val_len < sizeof(struct scoutfs_packed_bitmap)) { - ret = -EIO; - goto out; - } - pb = iref->val; - - bbk = iref->key; - bb->type = bbk->type; - bb->base = be64_to_cpu(bbk->base); - - nr = hweight64(le64_to_cpu(pb->present)); - - if (iref->val_len != - offsetof(struct scoutfs_packed_bitmap, words[nr])) { - ret = -EIO; - goto out; - } - - present = le64_to_cpu(pb->present); - set = le64_to_cpu(pb->set); - w = 0; - for (i = 0, b = 1; - (present | set) && i < SCOUTFS_PACKED_BITMAP_WORDS; - i++, b <<= 1) { - if (set & b) - u64_to_long_bits(bb->bits, i, ~0ULL); - else if (present & b) - u64_to_long_bits(bb->bits, i, - le64_to_cpu(pb->words[w++])); - } - ret = 0; - -out: - return ret; -} - -static int load_block_bitmap(struct super_block *sb, - struct scoutfs_btree_root *root, - u64 blkno, u8 type, bool next, bool zero_enoent, - struct block_bitmap **bb_ret) -{ - struct scoutfs_block_bitmap_key bbk; - struct block_bitmap *bb = NULL; - SCOUTFS_BTREE_ITEM_REF(iref); - u64 base; - int bit; - int ret; - - bb = kzalloc(sizeof(struct block_bitmap), GFP_NOFS); - if (!bb) { - ret = -ENOMEM; - goto out; - } - - block_bitmap_bit(&base, &bit, blkno, type); - - bbk.type = type; - bbk.base = cpu_to_be64(base); - - if (next) - ret = scoutfs_btree_next(sb, root, &bbk, sizeof(bbk), &iref); - else - ret = scoutfs_btree_lookup(sb, root, &bbk, sizeof(bbk), &iref); - if (ret == 0) { - ret = unpack_block_bitmap(bb, &iref); - bb->exists = true; - scoutfs_btree_put_iref(&iref); - } - if (ret == -ENOENT && zero_enoent) { - bb->base = base; - bb->type = type; - ret = 0; - } - -out: - if (ret < 0) { - kfree(bb); - *bb_ret = NULL; - } else { - *bb_ret = bb; - } - return ret; -} - -/* - * Block bitmaps start with two flag words that indicate if logical - * words are all 0s, all 1s, or a mix of set and clear bits stored in - * the item payload. Typically the allocators will have long runs of - * set or clear bits so we don't store most of the bitmaps. Badly - * fragmented allocators will be (2/64 = ~3%) larger. - */ -static int pack_block_bitmap(struct scoutfs_packed_bitmap *pb, - struct block_bitmap *bb) -{ - __le64 word; - u64 present = 0; - u64 set = 0; - u64 b; - int w; - int i; - - w = 0; - for (i = 0, b = 1; i < SCOUTFS_PACKED_BITMAP_WORDS; i++, b <<= 1) { - word = long_bits_to_le64(bb->bits, i); - - if (word == cpu_to_le64(~0ULL)) { - set |= b; - } else if (word != 0) { - present |= b; - pb->words[w++] = word; - } - } - - pb->set = cpu_to_le64(set); - pb->present = cpu_to_le64(present); - - return offsetof(struct scoutfs_packed_bitmap, words[w]); -} - -static int store_block_bitmap(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_btree_root *root, - struct block_bitmap *bb) -{ - struct scoutfs_block_bitmap_key bbk; - struct scoutfs_packed_bitmap *pb; - int size; - int ret; - - bbk.type = bb->type; - bbk.base = cpu_to_be64(bb->base); - - if (bitmap_empty(bb->bits, SCOUTFS_BLOCK_BITMAP_BITS)) { - if (!bb->exists) { - ret = 0; - goto out; - } - - ret = scoutfs_btree_delete(sb, alloc, wri, root, - &bbk, sizeof(bbk)); - - } else { - pb = kmalloc(SCOUTFS_PACKED_BITMAP_MAX_BYTES, GFP_NOFS); - if (!pb) { - ret = -ENOMEM; - goto out; - } - - size = pack_block_bitmap(pb, bb); - - ret = scoutfs_btree_force(sb, alloc, wri, root, - &bbk, sizeof(bbk), pb, size); - kfree(pb); - if (ret == 0) - bb->exists = true; - } -out: - return ret; -} - -/* - * Set a region of bitmaps which must fit in one item. The caller's - * blkno is translated to an item base and then the number of bits are - * set. The caller is specifying a number of bits to set, not a block - * extent. - */ -static int set_block_bits(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_btree_root *root, u8 type, u64 blkno, - int nbits) -{ - struct block_bitmap *bb = NULL; - u64 base; - int bit; - int ret; - - if (WARN_ON_ONCE(!block_bitmap_fits(blkno, nbits))) - return -EINVAL; - - ret = load_block_bitmap(sb, root, blkno, type, false, true, &bb); - if (ret < 0) - goto out; - - block_bitmap_bit(&base, &bit, blkno, type); - - bitmap_set(bb->bits, bit, nbits); - - /* if a little bitmap is full, set it's big and delete it */ - if (type == SCOUTFS_BLOCK_BITMAP_LITTLE && - bitmap_full(bb->bits, SCOUTFS_PACKED_BITMAP_BITS)) { - ret = set_block_bits(sb, alloc, wri, root, - SCOUTFS_BLOCK_BITMAP_BIG, blkno, 1); - if (ret < 0) - goto out; - - bitmap_zero(bb->bits, SCOUTFS_PACKED_BITMAP_BITS); - } - - ret = store_block_bitmap(sb, alloc, wri, root, bb); - BUG_ON(ret < 0); /* cleared bit out of sync with existing littles */ - -out: - kfree(bb); - return ret; -} - -/* - * Find a region of free blocks for the caller. The caller can ask for - * an arbitrarily large extent but we'll only return at most a bitmap's - * worth of blocks from one allocation. - * - * Big bitmap items are stored before little items. This let's large - * allocations naturally fall back to being satisfied by little items - * when there are no more remaining big items. Small allocations first - * look for little items and then search again for big items that they - * can break up. - * - * We always simply look for the first free region. This is operating - * in the client on trees whose items are populated by the server - * between each transaction. The server is responsible for distributing - * the items such that the client tends to allocate across the device - * over time. - */ -static int alloc_blocks(struct super_block *sb, u64 count, u64 *blkno_ret, - u64 *count_ret) -{ - DECLARE_DATA_INFO(sb, datinf); - struct scoutfs_balloc_root *broot = &datinf->data_alloc; - struct block_bitmap *bb = NULL; - u64 blkno; - u8 type; - int bit; - int end; - int ret; - - if (WARN_ON_ONCE(count == 0)) - return -EINVAL; - - /* will only allocate from one block bitmap item at a time */ - count = min_t(u64, count, SCOUTFS_BLOCK_BITMAP_BITS); - - /* small allocations first look for little items, then check big */ - if (count < SCOUTFS_BLOCK_BITMAP_BITS) - type = SCOUTFS_BLOCK_BITMAP_LITTLE; - else - type = SCOUTFS_BLOCK_BITMAP_BIG; - - do { - ret = load_block_bitmap(sb, &broot->root, 0, type, - true, false, &bb); - } while ((ret == -ENOENT && type == SCOUTFS_BLOCK_BITMAP_LITTLE) && - (type = SCOUTFS_BLOCK_BITMAP_BIG, 1)); - if (ret < 0) { - if (ret == -ENOENT) - ret = -ENOSPC; - goto out; - } - - bit = find_first_bit(bb->bits, SCOUTFS_BLOCK_BITMAP_BITS); - if (WARN_ON_ONCE(bit >= SCOUTFS_BLOCK_BITMAP_BITS)) { - ret = -EIO; /* stored items should have bits set */ - goto out; - } - - blkno = block_bitmap_blkno(bb->base, bit, bb->type); - - if (bb->type == SCOUTFS_BLOCK_BITMAP_BIG) { - /* set remaining little bits if using big for partial small */ - if (count != SCOUTFS_BLOCK_BITMAP_BITS) { - ret = set_block_bits(sb, datinf->alloc, datinf->wri, - &broot->root, - SCOUTFS_BLOCK_BITMAP_LITTLE, - blkno + count, - SCOUTFS_BLOCK_BITMAP_BITS - count); - if (ret < 0) - goto out; - } - - clear_bit(bit, bb->bits); - - } else { - end = find_next_zero_bit(bb->bits, SCOUTFS_BLOCK_BITMAP_BITS, - bit + 1); - end = min(end, SCOUTFS_BLOCK_BITMAP_BITS); /* catch > size */ - count = min_t(u64, count, end - bit); - - bitmap_clear(bb->bits, bit, count); - } - - ret = store_block_bitmap(sb, datinf->alloc, datinf->wri, - &broot->root, bb); - BUG_ON(ret < 0); /* little partial out of sync with big */ - - le64_add_cpu(&broot->total_free, -count); - - *blkno_ret = blkno; - *count_ret = count; - - trace_scoutfs_data_alloc_blocks(sb, broot, bb->base, bb->type, bit, - blkno, count); - -out: - kfree(bb); - return ret; -} - -/* - * Set free block bits in the block bitmaps and update the root's - * total_free count. The caller can specifiy the root so that this can - * be used both to free used allocations as well as to return unused - * allocations in error paths. The caller must ensure that the block - * regions fit in a single block bitmap (by for the blocks in an - * extent). - */ -static int free_blocks(struct super_block *sb, - struct scoutfs_balloc_root *broot, u64 blkno, u64 count) -{ - DECLARE_DATA_INFO(sb, datinf); - int ret; - - if (count == SCOUTFS_BLOCK_BITMAP_BITS) - ret = set_block_bits(sb, datinf->alloc, datinf->wri, - &broot->root, SCOUTFS_BLOCK_BITMAP_BIG, - blkno, 1); - else - ret = set_block_bits(sb, datinf->alloc, datinf->wri, - &broot->root, SCOUTFS_BLOCK_BITMAP_LITTLE, - blkno, count); - - if (ret == 0) { - le64_add_cpu(&broot->total_free, count); - trace_scoutfs_data_free_blocks(sb, broot, blkno, count); - } - - return ret; -} - -/* - * Ensure that the destination free block bitmap tree has the minimum - * total free blocks by moving bits from the source tree. It will first - * try to find big bits starting at the cursor but will fall back to - * little bits after having wrapped the cursor. - * - * This will move all the items from the source to the destination if - * that's what it takes to reach the minimum. - * - * This is called by the server which provides its writer and metadata - * allocation contexts. It has locked the two allocation trees that - * will be modified. - */ -int scoutfs_data_move_alloc_bits(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_balloc_root *dst, - struct scoutfs_balloc_root *src, - __le64 *cursor, u64 min_dst_total) - -{ - struct block_bitmap *sbb = NULL; - struct block_bitmap *dbb = NULL; - u64 needed; - u64 blocks; - u64 moved; - u64 blkno; - u64 base; - u64 curs; - u8 type; - int nbits; - int bit; - int end; - int ret = 0; - - /* start moving big bitmap items */ - type = SCOUTFS_BLOCK_BITMAP_BIG; - curs = le64_to_cpup(cursor); - - while (le64_to_cpu(dst->total_free) < min_dst_total) { - - /* find the next source bitmap item with bits to move */ - kfree(sbb); - ret = load_block_bitmap(sb, &src->root, curs, type, - true, false, &sbb); - if (ret == 0 && sbb->type != type) - ret = -ENOENT; - if (ret < 0) { - if (ret == -ENOENT) { - if (curs > 0) { - curs = 0; - continue; - } - if (type == SCOUTFS_BLOCK_BITMAP_BIG) { - type = SCOUTFS_BLOCK_BITMAP_LITTLE; - curs = le64_to_cpup(cursor); - continue; - } - ret = -ENOSPC; - } - break; - } - - /* load the destination bitmap */ - blkno = block_bitmap_blkno(sbb->base, 0, type); - kfree(dbb); - ret = load_block_bitmap(sb, &dst->root, blkno, type, - false, true, &dbb); - if (ret < 0) - break; - - /* figure out how many bits to move, can overshoot */ - needed = min_dst_total - le64_to_cpu(dst->total_free); - if (type == SCOUTFS_BLOCK_BITMAP_BIG) { - needed = (needed + SCOUTFS_BLOCK_BITMAP_BITS - 1) - >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; - } - - /* start searching from the cursor if within item */ - if (curs > blkno) - blkno = curs; - block_bitmap_bit(&base, &bit, blkno, type); - - moved = 0; - while (moved < needed) { - bit = find_next_bit(sbb->bits, - SCOUTFS_BLOCK_BITMAP_BITS, bit); - if (bit >= SCOUTFS_BLOCK_BITMAP_BITS) - break; - - end = find_next_zero_bit(sbb->bits, - SCOUTFS_BLOCK_BITMAP_BITS, - bit + 1); - end = min(end, SCOUTFS_BLOCK_BITMAP_BITS); - nbits = min_t(u64, needed - moved, end - bit); - - bitmap_clear(sbb->bits, bit, nbits); - bitmap_set(dbb->bits, bit, nbits); - - curs = block_bitmap_blkno(dbb->base, bit + nbits, type); - moved += nbits; - } - - ret = store_block_bitmap(sb, alloc, wri, &dst->root, dbb); - if (ret < 0) - break; - - ret = store_block_bitmap(sb, alloc, wri, &src->root, sbb); - BUG_ON(ret); /* inconsistent src/dst, save orig src */ - - blocks = moved; - if (sbb->type == SCOUTFS_BLOCK_BITMAP_BIG) - blocks <<= SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; - - le64_add_cpu(&dst->total_free, blocks); - le64_add_cpu(&src->total_free, -blocks); - - *cursor = cpu_to_le64(curs); - } - - kfree(sbb); - kfree(dbb); - - return ret; -} - -/* - * The server caller is making their way through free data blocks - * initializing free block bitmap bits for the first time. This is the - * only mechanism that initializes free block bitmap items so we know - * that we never have to merge with existing items as long as we always - * write a full item. - * - * The caller gives us the fully extent of blknos that we could - * initialize and we figure out the size of the largest item and its - * bits which cover the start of the extent. We can set big bits if the - * extent is aligned to a small bitmap and is large enough. - */ -int scoutfs_data_add_free_blocks(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_balloc_root *broot, - u64 blkno, u64 count) - -{ - u64 base; - u8 type; - int nbits; - int bit; - int ret; - - type = SCOUTFS_BLOCK_BITMAP_LITTLE; - block_bitmap_bit(&base, &bit, blkno, type); - - if (bit == 0 && count >= SCOUTFS_BLOCK_BITMAP_BITS) { - type = SCOUTFS_BLOCK_BITMAP_BIG; - block_bitmap_bit(&base, &bit, blkno, type); - nbits = min_t(u64, count >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT, - SCOUTFS_BLOCK_BITMAP_BITS - bit); - count = (u64)nbits << SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; - } else { - nbits = min_t(u64, count, SCOUTFS_BLOCK_BITMAP_BITS - bit); - count = nbits; - } - - ret = set_block_bits(sb, alloc, wri, &broot->root, type, blkno, nbits); - if (ret == 0) { - le64_add_cpu(&broot->total_free, count); - ret = count; - } - - return ret; -} -#endif - /* * Find and remove or mark offline the block mappings that intersect * with the caller's range. The caller is responsible for transactions diff --git a/kmod/src/data.h b/kmod/src/data.h index 0da25fa1..895de61b 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -72,19 +72,6 @@ int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, struct scoutfs_ioctl_data_waiting_entry *dwe, unsigned int nr); -#if 0 -int scoutfs_data_move_alloc_bits(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_balloc_root *dst, - struct scoutfs_balloc_root *src, - __le64 *cursor, u64 min_dst_total); -int scoutfs_data_add_free_blocks(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_balloc_root *broot, - u64 blkno, u64 count); -#endif void scoutfs_data_init_btrees(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, diff --git a/kmod/src/format.h b/kmod/src/format.h index 34f55252..6cfb7322 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -246,57 +246,6 @@ struct scoutfs_btree_block { struct scoutfs_btree_item_header item_hdrs[0]; } __packed; -#if 0 -/* - * Free metadata blocks are tracked by block allocator items. - */ -struct scoutfs_balloc_root { - struct scoutfs_btree_root root; - __le64 total_free; -} __packed; -struct scoutfs_balloc_item_key { - __be64 base; -} __packed; - -#define SCOUTFS_BALLOC_ITEM_BYTES 256 -#define SCOUTFS_BALLOC_ITEM_U64S (SCOUTFS_BALLOC_ITEM_BYTES / \ - sizeof(__u64)) -#define SCOUTFS_BALLOC_ITEM_BITS (SCOUTFS_BALLOC_ITEM_BYTES * 8) -#define SCOUTFS_BALLOC_ITEM_BASE_SHIFT ilog2(SCOUTFS_BALLOC_ITEM_BITS) -#define SCOUTFS_BALLOC_ITEM_BIT_MASK (SCOUTFS_BALLOC_ITEM_BITS - 1) - -struct scoutfs_balloc_item_val { - __le64 bits[SCOUTFS_BALLOC_ITEM_U64S]; -} __packed; - -/* - * Free data blocks are tracked in bitmaps stored in btree items. - */ -struct scoutfs_block_bitmap_key { - __u8 type; - __be64 base; -} __packed; - -#define SCOUTFS_BLOCK_BITMAP_BIG 0 -#define SCOUTFS_BLOCK_BITMAP_LITTLE 1 - -#define SCOUTFS_PACKED_BITMAP_WORDS 32 -#define SCOUTFS_PACKED_BITMAP_BITS (SCOUTFS_PACKED_BITMAP_WORDS * 64) -#define SCOUTFS_PACKED_BITMAP_MAX_BYTES \ - offsetof(struct scoutfs_packed_bitmap, \ - words[SCOUTFS_PACKED_BITMAP_WORDS]) - -#define SCOUTFS_BLOCK_BITMAP_BITS SCOUTFS_PACKED_BITMAP_BITS -#define SCOUTFS_BLOCK_BITMAP_BIT_MASK (SCOUTFS_PACKED_BITMAP_BITS - 1) -#define SCOUTFS_BLOCK_BITMAP_BASE_SHIFT (ilog2(SCOUTFS_PACKED_BITMAP_BITS)) - -struct scoutfs_packed_bitmap { - __le64 present; - __le64 set; - __le64 words[0]; -}; -#endif - /* * The lock server keeps a persistent record of connected clients so that * server failover knows who to wait for before resuming operations. diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index c5b028fc..c4b4b48d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -118,73 +118,6 @@ TRACE_EVENT(scoutfs_complete_truncate, __entry->flags) ); -#if 0 -TRACE_EVENT(scoutfs_data_alloc_blocks, - TP_PROTO(struct super_block *sb, struct scoutfs_balloc_root *broot, - u64 base, u8 type, int bit, u64 blkno, u64 count), - - TP_ARGS(sb, broot, base, type, bit, blkno, count), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, root_blkno) - __field(__u64, root_seq) - __field(__u64, root_total_free) - __field(__u64, base) - __field(u8, type) - __field(int, bit) - __field(__u64, blkno) - __field(__u64, count) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->root_blkno = le64_to_cpu(broot->root.ref.blkno); - __entry->root_seq = le64_to_cpu(broot->root.ref.seq); - __entry->root_total_free = le64_to_cpu(broot->total_free); - __entry->base = base; - __entry->type = type; - __entry->bit = bit; - __entry->blkno = blkno; - __entry->count = count; - ), - - TP_printk(SCSBF" root_blkno %llu root_seq %llu root_total_free %llu base %llu type %u bit %d blkno %llu count %llu\n", - SCSB_TRACE_ARGS, __entry->root_blkno, __entry->root_seq, - __entry->root_total_free, __entry->base, __entry->type, - __entry->bit, __entry->blkno, __entry->count) -); - -TRACE_EVENT(scoutfs_data_free_blocks, - TP_PROTO(struct super_block *sb, struct scoutfs_balloc_root *broot, - u64 blkno, u64 count), - - TP_ARGS(sb, broot, blkno, count), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, root_blkno) - __field(__u64, root_seq) - __field(__u64, root_total_free) - __field(__u64, blkno) - __field(__u64, count) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->root_blkno = le64_to_cpu(broot->root.ref.blkno); - __entry->root_seq = le64_to_cpu(broot->root.ref.seq); - __entry->root_total_free = le64_to_cpu(broot->total_free); - __entry->blkno = blkno; - __entry->count = count; - ), - - TP_printk(SCSBF" root_blkno %llu root_seq %llu root_total_free %llu blkno %llu count %llu\n", - SCSB_TRACE_ARGS, __entry->root_blkno, __entry->root_seq, - __entry->root_total_free, __entry->blkno, __entry->count) -); -#endif - TRACE_EVENT(scoutfs_data_fallocate, TP_PROTO(struct super_block *sb, u64 ino, int mode, loff_t offset, loff_t len, int ret),