From 8f946aa478a412cd271aa39229d765f80d8c0ba6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 21 Sep 2020 11:39:55 -0700 Subject: [PATCH] scoutfs: add btree item extent allocator Add an allocator which uses btree items to store extents. Both the client and server will use this for btree blocks, the client will use it for srch blocks and data extents, and the server will move extents between the core fs allocator btree roots and the clients' roots. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/alloc.c | 1112 ++++++++++++++++++++++++++++++++++++++ kmod/src/alloc.h | 124 +++++ kmod/src/counters.h | 11 +- kmod/src/format.h | 48 ++ kmod/src/scoutfs_trace.h | 117 ++++ 6 files changed, 1412 insertions(+), 1 deletion(-) create mode 100644 kmod/src/alloc.c create mode 100644 kmod/src/alloc.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 9c299c5d..bfd8f38a 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -10,6 +10,7 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += \ avl.o \ + alloc.o \ block.o \ btree.o \ client.o \ diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c new file mode 100644 index 00000000..ca088c89 --- /dev/null +++ b/kmod/src/alloc.c @@ -0,0 +1,1112 @@ +/* + * Copyright (C) 2020 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "block.h" +#include "btree.h" +#include "trans.h" +#include "alloc.h" +#include "counters.h" +#include "scoutfs_trace.h" + +/* + * The core allocator uses extent items in btrees rooted in the super. + * Each free extent is stored in two items. The first item is indexed + * by block location and is used to merge adjacent extents when freeing. + * The second item is indexed by length and is used to find large + * extents to allocate from. + * + * Free extent always consumes the front of the largest extent. This + * attempts to discourage fragmentation by given smaller freed extents + * time for an adjacent free to merge before we attempt to re-use them. + * + * The metadata btrees that store extents are updated with cow. This + * requires allocation during extent item modification on behalf of + * allocation. Avoiding this recursion introduces the second structure, + * persistent singly linked lists of individual blknos. + * + * The alloc lists are used for metadata allocation during a + * transaction. Before each transaction lists of blknos are prepared + * for use during the transaction. This ensures a small predictable + * number of cows needed to fully dirty the metadata allocator + * structures during the transaction. As the transaction proceeds + * allocations are made from a list of available meta blknos, and frees + * are performed by adding blknos to another list of freed blknos. + * After transactions these lists are merged back in to extents. + * + * Data allocations are performed directly on a btree of extent items, + * with a bit of caching to stream small file data allocations from + * memory instead of performing multiple btree calls per block + * allocation. + * + * Every transaction has exclusive access to its metadata list blocks + * and data extent trees which are prepared by the server. For client + * metadata and srch transactions the server moved extents and blocks + * into persistent items that are communicated with the server. For + * server transactions metadata the server has to prepare structures for + * itself. To avoid modifying the same structure both explicitly + * (refilling an allocator) and implicitly (using the current allocator + * for cow allocations), it double buffers list blocks. It uses current + * blocks to modify the next blocks, and swaps them at each transaction. + */ + +/* + * Free extents don't have flags and are stored in two indexes sorted by + * block location and by length, largest first. The block location key + * is set to the final block in the extent so that we can find + * intersections by calling _next() iterators starting with the block + * we're searching for. + */ +static void init_ext_key(struct scoutfs_key *key, int type, u64 start, u64 len) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FREE_EXTENT_ZONE, + .sk_type = type, + }; + + if (type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { + key->skfb_end = cpu_to_le64(start + len - 1); + key->skfb_len = cpu_to_le64(len); + } else if (type == SCOUTFS_FREE_EXTENT_LEN_TYPE) { + key->skfl_neglen = cpu_to_le64(-len); + key->skfl_blkno = cpu_to_le64(start); + } else { + BUG(); + } +} + +static void ext_from_key(struct scoutfs_extent *ext, struct scoutfs_key *key) +{ + if (key->sk_type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { + ext->start = le64_to_cpu(key->skfb_end) - + le64_to_cpu(key->skfb_len) + 1; + ext->len = le64_to_cpu(key->skfb_len); + } else { + ext->start = le64_to_cpu(key->skfl_blkno); + ext->len = -le64_to_cpu(key->skfl_neglen); + } + ext->map = 0; + ext->flags = 0; +} + +struct alloc_ext_args { + struct scoutfs_alloc *alloc; + struct scoutfs_block_writer *wri; + struct scoutfs_alloc_root *root; + int type; +}; + +static int alloc_ext_next(struct super_block *sb, void *arg, + u64 start, u64 len, struct scoutfs_extent *ext) +{ + struct alloc_ext_args *args = arg; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + int ret; + + init_ext_key(&key, args->type, start, len); + + ret = scoutfs_btree_next(sb, &args->root->root, &key, &iref); + if (ret == 0) { + if (iref.val_len != 0) + ret = -EIO; + else if (iref.key->sk_type != args->type) + ret = -ENOENT; + else + ext_from_key(ext, iref.key); + scoutfs_btree_put_iref(&iref); + } + + if (ret < 0) + memset(ext, 0, sizeof(struct scoutfs_extent)); + + return ret; +} + +static int other_type(int type) +{ + if (type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) + return SCOUTFS_FREE_EXTENT_LEN_TYPE; + else if (type == SCOUTFS_FREE_EXTENT_LEN_TYPE) + return SCOUTFS_FREE_EXTENT_BLKNO_TYPE; + else + BUG(); +} + +/* + * Insert an extent along with its matching item which is indexed by + * opposite of its len or blkno. If we succeed we update the root's + * record of the total length of all the stored extents. + */ +static int alloc_ext_insert(struct super_block *sb, void *arg, + u64 start, u64 len, u64 map, u8 flags) +{ + struct alloc_ext_args *args = arg; + struct scoutfs_key other; + struct scoutfs_key key; + int ret; + int err; + + /* allocator extents don't have mappings or flags */ + if (WARN_ON_ONCE(map || flags)) + return -EINVAL; + + init_ext_key(&key, args->type, start, len); + init_ext_key(&other, other_type(args->type), start, len); + + ret = scoutfs_btree_insert(sb, args->alloc, args->wri, + &args->root->root, &key, NULL, 0); + if (ret == 0) { + ret = scoutfs_btree_insert(sb, args->alloc, args->wri, + &args->root->root, &other, NULL, 0); + if (ret < 0) { + err = scoutfs_btree_delete(sb, args->alloc, args->wri, + &args->root->root, &key); + BUG_ON(err); + } else { + le64_add_cpu(&args->root->total_len, len); + } + } + + return ret; +} + +static int alloc_ext_remove(struct super_block *sb, void *arg, + u64 start, u64 len, u64 map, u8 flags) +{ + struct alloc_ext_args *args = arg; + struct scoutfs_key other; + struct scoutfs_key key; + int ret; + int err; + + init_ext_key(&key, args->type, start, len); + init_ext_key(&other, other_type(args->type), start, len); + + ret = scoutfs_btree_delete(sb, args->alloc, args->wri, + &args->root->root, &key); + if (ret == 0) { + ret = scoutfs_btree_delete(sb, args->alloc, args->wri, + &args->root->root, &other); + if (ret < 0) { + err = scoutfs_btree_insert(sb, args->alloc, args->wri, + &args->root->root, &key, + NULL, 0); + BUG_ON(err); + } else { + le64_add_cpu(&args->root->total_len, -len); + } + } + + return ret; +} + +static struct scoutfs_ext_ops alloc_ext_ops = { + .next = alloc_ext_next, + .insert = alloc_ext_insert, + .remove = alloc_ext_remove, +}; + +static bool invalid_extent(u64 start, u64 end, u64 first, u64 last) +{ + return start > end || start < first || end > last; +} + +static bool invalid_meta_blkno(struct super_block *sb, u64 blkno) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + return invalid_extent(blkno, blkno, + le64_to_cpu(super->first_meta_blkno), + le64_to_cpu(super->last_meta_blkno)); +} + +static bool invalid_data_extent(struct super_block *sb, u64 start, u64 len) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + return invalid_extent(start, start + len - 1, + le64_to_cpu(super->first_data_blkno), + le64_to_cpu(super->last_data_blkno)); +} + +void scoutfs_alloc_init(struct scoutfs_alloc *alloc, + struct scoutfs_alloc_list_head *avail, + struct scoutfs_alloc_list_head *freed) +{ + memset(alloc, 0, sizeof(struct scoutfs_alloc)); + + spin_lock_init(&alloc->lock); + mutex_init(&alloc->mutex); + alloc->avail = *avail; + alloc->freed = *freed; +} + +/* + * We're about to commit the transaction that used this allocator, drop + * its block references. + */ +int scoutfs_alloc_prepare_commit(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri) +{ + scoutfs_block_put(sb, alloc->dirty_avail_bl); + alloc->dirty_avail_bl = NULL; + scoutfs_block_put(sb, alloc->dirty_freed_bl); + alloc->dirty_freed_bl = NULL; + + return 0; +} + +static u32 list_block_space(__le32 nr) +{ + return SCOUTFS_ALLOC_LIST_MAX_BLOCKS - le32_to_cpu(nr); +} + +static u64 list_block_peek(struct scoutfs_alloc_list_block *lblk, + unsigned int skip) +{ + BUG_ON(skip >= le32_to_cpu(lblk->nr)); + + return le64_to_cpu(lblk->blknos[le32_to_cpu(lblk->start) + skip]); +} + +/* + * Add a blkno to the array. Typically we append of the array. But we + * can also prepend once there's no more room at the end. Consumers of + * the blocks sort before removing them. + */ +static void list_block_add(struct scoutfs_alloc_list_head *lhead, + struct scoutfs_alloc_list_block *lblk, u64 blkno) +{ + u32 start = le32_to_cpu(lblk->start); + u32 nr = le32_to_cpu(lblk->nr); + + BUG_ON(lhead->ref.blkno != lblk->hdr.blkno); + BUG_ON(list_block_space(lblk->nr) == 0); + + if (start + nr < SCOUTFS_ALLOC_LIST_MAX_BLOCKS) { + lblk->blknos[start + nr] = cpu_to_le64(blkno); + } else { + start--; + lblk->blknos[start] = cpu_to_le64(blkno); + lblk->start = cpu_to_le32(start); + } + + le32_add_cpu(&lblk->nr, 1); + le64_add_cpu(&lhead->total_nr, 1); + le32_add_cpu(&lhead->first_nr, 1); +} + +/* + * Remove blknos from the start of the array. + */ +static void list_block_remove(struct scoutfs_alloc_list_head *lhead, + struct scoutfs_alloc_list_block *lblk, + unsigned int count) +{ + BUG_ON(lhead->ref.blkno != lblk->hdr.blkno); + BUG_ON(count > SCOUTFS_ALLOC_LIST_MAX_BLOCKS); + BUG_ON(le32_to_cpu(lblk->nr) < count); + + le32_add_cpu(&lblk->nr, -count); + if (lblk->nr == 0) + lblk->start = 0; + else + le32_add_cpu(&lblk->start, count); + le64_add_cpu(&lhead->total_nr, -(u64)count); + le32_add_cpu(&lhead->first_nr, -count); +} + +static int cmp_le64(const void *A, const void *B) +{ + const __le64 *a = A; + const __le64 *b = B; + + return scoutfs_cmp_u64s(le64_to_cpu(*a), le64_to_cpu(*b)); +} + +static void swap_le64(void *A, void *B, int size) +{ + __le64 *a = A; + __le64 *b = B; + + swap(*a, *b); +} + +static void list_block_sort(struct scoutfs_alloc_list_block *lblk) +{ + sort(&lblk->blknos[le32_to_cpu(lblk->start)], le32_to_cpu(lblk->nr), + sizeof(lblk->blknos[0]), cmp_le64, swap_le64); +} + +/* + * We're always reading blocks that we own, so we shouldn't see stale + * references. But the cached block can be stale and we can need to + * invalidate it. + */ +static int read_list_block(struct super_block *sb, + struct scoutfs_alloc_list_ref *ref, + struct scoutfs_block **bl_ret) +{ + struct scoutfs_block *bl = NULL; + + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (!IS_ERR_OR_NULL(bl) && + !scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno, + SCOUTFS_BLOCK_MAGIC_ALLOC_LIST)) { + scoutfs_inc_counter(sb, alloc_stale_cached_list_block); + scoutfs_block_invalidate(sb, bl); + scoutfs_block_put(sb, bl); + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + } + if (IS_ERR(bl)) { + *bl_ret = NULL; + return PTR_ERR(bl); + } + + *bl_ret = bl; + return 0; +} + +/* + * Give the caller a dirty list block, always allocating a new block if + * the ref is empty. + * + * If the caller gives us an allocated blkno for the cow then we know + * that they're taking care of allocating and freeing the blknos, if not + * we call meta alloc and free. + */ +static int dirty_list_block(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_ref *ref, + u64 dirty, u64 *old, + struct scoutfs_block **bl_ret) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_block *cow_bl = NULL; + struct scoutfs_block *bl = NULL; + struct scoutfs_alloc_list_block *lblk; + bool undo_alloc = false; + u64 blkno; + int ret; + int err; + + blkno = le64_to_cpu(ref->blkno); + if (blkno) { + ret = read_list_block(sb, ref, &bl); + if (ret < 0) + goto out; + + if (scoutfs_block_writer_is_dirty(sb, bl)) { + ret = 0; + goto out; + } + } + + if (dirty == 0) { + ret = scoutfs_alloc_meta(sb, alloc, wri, &dirty); + if (ret < 0) + goto out; + undo_alloc = true; + } + + cow_bl = scoutfs_block_create(sb, dirty); + if (IS_ERR(cow_bl)) { + ret = PTR_ERR(cow_bl); + goto out; + } + + if (old) { + *old = blkno; + } else if (blkno) { + ret = scoutfs_free_meta(sb, alloc, wri, blkno); + if (ret < 0) + goto out; + } + + if (bl) + memcpy(cow_bl->data, bl->data, SCOUTFS_BLOCK_LG_SIZE); + else + memset(cow_bl->data, 0, SCOUTFS_BLOCK_LG_SIZE); + scoutfs_block_put(sb, bl); + bl = cow_bl; + cow_bl = NULL; + + lblk = bl->data; + lblk->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_ALLOC_LIST); + lblk->hdr.fsid = super->hdr.fsid; + lblk->hdr.blkno = cpu_to_le64(bl->blkno); + prandom_bytes(&lblk->hdr.seq, sizeof(lblk->hdr.seq)); + + ref->blkno = lblk->hdr.blkno; + ref->seq = lblk->hdr.seq; + + scoutfs_block_writer_mark_dirty(sb, wri, bl); + ret = 0; + +out: + scoutfs_block_put(sb, cow_bl); + if (ret < 0 && undo_alloc) { + err = scoutfs_free_meta(sb, alloc, wri, dirty); + BUG_ON(err); /* inconsistent */ + } + + if (ret < 0) { + scoutfs_block_put(sb, bl); + bl = NULL; + } + *bl_ret = bl; + + return ret; +} + +/* Allocate a new dirty list block if we fill up more than 3/4 of the block. */ +#define EMPTY_FREED_THRESH (SCOUTFS_ALLOC_LIST_MAX_BLOCKS / 4) + +/* + * Get dirty avail and freed list blocks that will be used for meta + * allocations during our transaction. We peek at the next avail blknos + * for the cow allocations and manually record the cow frees rather than + * recursively calling into alloc_meta and free_meta. + * + * In the client the server will have emptied the freed list so it will + * always allocate a new first empty block for frees. But in the server + * it might have long lists of frees that it's trying to merge in to + * extents over multiple transactions. If the head of the freed list + * doesn't have room we add a new empty block. + */ +static int dirty_alloc_blocks(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri) +{ + struct scoutfs_alloc_list_ref orig_freed; + struct scoutfs_alloc_list_block *lblk; + struct scoutfs_block *av_bl = NULL; + struct scoutfs_block *fr_bl = NULL; + struct scoutfs_block *bl; + bool link_orig = false; + u64 av_peek; + u64 av_old; + u64 fr_peek; + u64 fr_old; + int ret; + + if (alloc->dirty_avail_bl != NULL) + return 0; + + mutex_lock(&alloc->mutex); + + /* undo dirty freed if we get an error after */ + orig_freed = alloc->freed.ref; + + if (alloc->dirty_avail_bl != NULL) { + ret = 0; + goto out; + } + + /* caller must ensure that transactions commit before running out */ + if (WARN_ON_ONCE(alloc->avail.ref.blkno == 0) || + WARN_ON_ONCE(le32_to_cpu(alloc->avail.first_nr) < 2)) { + ret = -ENOSPC; + goto out; + } + + ret = read_list_block(sb, &alloc->avail.ref, &bl); + if (ret < 0) + goto out; + + lblk = bl->data; + av_peek = list_block_peek(lblk, 0); + fr_peek = list_block_peek(lblk, 1); + scoutfs_block_put(sb, bl); + lblk = NULL; + + if (alloc->freed.ref.blkno && + list_block_space(alloc->freed.first_nr) < EMPTY_FREED_THRESH) { + /* zero ref to force alloc of new block... */ + memset(&alloc->freed.ref, 0, sizeof(alloc->freed.ref)); + alloc->freed.first_nr = 0; + link_orig = true; + } + + /* dirty the first free block */ + ret = dirty_list_block(sb, alloc, wri, &alloc->freed.ref, + fr_peek, &fr_old, &fr_bl); + if (ret < 0) + goto out; + + if (link_orig) { + /* .. and point the new block at the rest of the list */ + lblk = fr_bl->data; + lblk->next = orig_freed; + lblk = NULL; + } + + ret = dirty_list_block(sb, alloc, wri, &alloc->avail.ref, + av_peek, &av_old, &av_bl); + if (ret < 0) + goto out; + + list_block_remove(&alloc->avail, av_bl->data, 2); + /* sort dirty avail to encourage contiguous sorted meta blocks */ + list_block_sort(av_bl->data); + + if (av_old) + list_block_add(&alloc->freed, fr_bl->data, av_old); + if (fr_old) + list_block_add(&alloc->freed, fr_bl->data, fr_old); + + alloc->dirty_avail_bl = av_bl; + av_bl = NULL; + alloc->dirty_freed_bl = fr_bl; + fr_bl = NULL; + ret = 0; + +out: + if (ret < 0 && alloc->freed.ref.blkno != orig_freed.blkno) { + if (fr_bl) + scoutfs_block_writer_forget(sb, wri, fr_bl); + alloc->freed.ref = orig_freed; + } + + mutex_unlock(&alloc->mutex); + scoutfs_block_put(sb, av_bl); + scoutfs_block_put(sb, fr_bl); + return ret; +} + +/* + * Alloc a metadata block for a transaction in either the client or the + * server. The list block in the allocator was prepared for the transaction. + */ +int scoutfs_alloc_meta(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 *blkno) +{ + struct scoutfs_alloc_list_block *lblk; + int ret; + + ret = dirty_alloc_blocks(sb, alloc, wri); + if (ret < 0) + goto out; + + spin_lock(&alloc->lock); + lblk = alloc->dirty_avail_bl->data; + if (WARN_ON_ONCE(lblk->nr == 0)) { + /* shouldn't happen, transaction should commit first */ + ret = -ENOSPC; + } else { + *blkno = list_block_peek(lblk, 0); + list_block_remove(&alloc->avail, lblk, 1); + ret = 0; + } + spin_unlock(&alloc->lock); + +out: + if (ret < 0) + *blkno = 0; + scoutfs_inc_counter(sb, alloc_alloc_meta); + trace_scoutfs_alloc_alloc_meta(sb, *blkno, ret); + return ret; +} + +int scoutfs_free_meta(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 blkno) +{ + struct scoutfs_alloc_list_block *lblk; + int ret; + + if (WARN_ON_ONCE(invalid_meta_blkno(sb, blkno))) + return -EINVAL; + + ret = dirty_alloc_blocks(sb, alloc, wri); + if (ret < 0) + goto out; + + spin_lock(&alloc->lock); + lblk = alloc->dirty_freed_bl->data; + if (WARN_ON_ONCE(list_block_space(lblk->nr) == 0)) { + /* shouldn't happen, transaction should commit first */ + ret = -EIO; + } else { + list_block_add(&alloc->freed, lblk, blkno); + ret = 0; + } + spin_unlock(&alloc->lock); + +out: + scoutfs_inc_counter(sb, alloc_free_meta); + trace_scoutfs_alloc_free_meta(sb, blkno, ret); + return ret; +} + +/* + * Allocate a data extent. An extent that's smaller than the requested + * size can be returned. + * + * The caller can provide a cached extent that can satisfy allocations + * and will be refilled by allocations. The caller is responsible for + * freeing any remaining cached extent back into persistent items before + * committing. + * + * Unlike meta allocations, the caller is expected to serialize + * allocations from the root. + */ +int scoutfs_alloc_data(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, + struct scoutfs_extent *cached, u64 count, + u64 *blkno_ret, u64 *count_ret) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + .root = root, + .type = SCOUTFS_FREE_EXTENT_LEN_TYPE, + }; + struct scoutfs_extent ext; + u64 len; + int ret; + + /* large allocations come straight from the allocator */ + if (count >= SCOUTFS_ALLOC_DATA_LG_THRESH) { + ret = scoutfs_ext_alloc(sb, &alloc_ext_ops, &args, + 0, 0, count, &ext); + if (ret < 0) + goto out; + + *blkno_ret = ext.start; + *count_ret = ext.len; + ret = 0; + goto out; + } + + /* smaller allocations come from a cached extent */ + if (cached->len == 0) { + ret = scoutfs_ext_alloc(sb, &alloc_ext_ops, &args, 0, 0, + SCOUTFS_ALLOC_DATA_LG_THRESH, cached); + if (ret < 0) + goto out; + } + + len = min(count, cached->len); + + *blkno_ret = cached->start; + *count_ret = len; + + cached->start += len; + cached->len -= len; + ret = 0; +out: + if (ret < 0) { + if (ret == -ENOENT) + ret = -ENOSPC; + *blkno_ret = 0; + *count_ret = 0; + } + + scoutfs_inc_counter(sb, alloc_alloc_data); + trace_scoutfs_alloc_alloc_data(sb, count, *blkno_ret, *count_ret, ret); + return ret; +} + +/* + * Free data extents into the freed tree that will be reclaimed by the + * server and made available for future allocators only if our + * transaction succeeds. We don't want to overwrite existing data if + * our transaction fails. + * + * Unlike meta allocations, the caller is expected to serialize data + * allocations. + */ +int scoutfs_free_data(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, u64 blkno, u64 count) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + .root = root, + .type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE, + }; + int ret; + + if (WARN_ON_ONCE(invalid_data_extent(sb, blkno, count))) + return -EINVAL; + + ret = scoutfs_ext_insert(sb, &alloc_ext_ops, &args, blkno, count, 0, 0); + scoutfs_inc_counter(sb, alloc_free_data); + trace_scoutfs_alloc_free_data(sb, blkno, count, ret); + return ret; +} + + +/* + * Move extent items adding up to the requested total length from the + * src to the dst tree. The caller is responsible for locking the + * trees, usually because they're also looking at total_len to decide + * how much to move. + * + * -ENOENT is returned if we run out of extents in the source tree + * before moving the total. + * + * This first pass is not optimal because it performs full btree walks + * per extent. We could optimize this with more clever btree item + * manipulation functions which can iterate through src and dst blocks + * and let callbacks indicate how to change items. + */ +int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *dst, + struct scoutfs_alloc_root *src, u64 total) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + }; + struct scoutfs_extent ext; + u64 moved = 0; + int ret = 0; + int err; + + while (moved < total) { + args.root = src; + args.type = SCOUTFS_FREE_EXTENT_LEN_TYPE; + ret = scoutfs_ext_alloc(sb, &alloc_ext_ops, &args, + 0, 0, total - moved, &ext); + if (ret < 0) + break; + + args.root = dst; + args.type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; + ret = scoutfs_ext_insert(sb, &alloc_ext_ops, &args, ext.start, + ext.len, ext.map, ext.flags); + if (ret < 0) { + args.root = src; + args.type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; + err = scoutfs_ext_insert(sb, &alloc_ext_ops, &args, + ext.start, ext.len, ext.map, + ext.flags); + BUG_ON(err); /* inconsistent */ + break; + } + + moved += ext.len; + scoutfs_inc_counter(sb, alloc_moved_extent); + } + + scoutfs_inc_counter(sb, alloc_move); + trace_scoutfs_alloc_move(sb, total, moved, ret); + + return ret; +} + +/* + * We only trim one block, instead of looping trimming all, because the + * caller is assuming that we do a fixed amount of work when they check + * that their allocator has enough remaining free blocks for us. + */ +static int trim_empty_first_block(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *lhead) +{ + struct scoutfs_alloc_list_block *one = NULL; + struct scoutfs_alloc_list_block *two = NULL; + struct scoutfs_block *one_bl = NULL; + struct scoutfs_block *two_bl = NULL; + int ret; + + if (WARN_ON_ONCE(lhead->ref.blkno == 0) || + WARN_ON_ONCE(lhead->first_nr != 0)) + return 0; + + ret = read_list_block(sb, &lhead->ref, &one_bl); + if (ret < 0) + goto out; + one = one_bl->data; + + if (one->next.blkno) { + ret = read_list_block(sb, &one->next, &two_bl); + if (ret < 0) + goto out; + two = two_bl->data; + } + + ret = scoutfs_free_meta(sb, alloc, wri, le64_to_cpu(lhead->ref.blkno)); + if (ret < 0) + goto out; + + lhead->ref = one->next; + lhead->first_nr = two ? two->nr : 0; + ret = 0; +out: + scoutfs_block_put(sb, one_bl); + scoutfs_block_put(sb, two_bl); + return ret; +} + +/* + * True if the allocator has enough free blocks to cow (alloc and free) + * a list block and all the btree blocks that store extent items. + * + * At most, an extent operation can dirty down three paths of the tree + * to modify a blkno item and two distant len items. We can grow and + * split the root, and then those three paths could share blocks but each + * modify two leaf blocks. + */ +static bool list_can_cow(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_alloc_root *root) +{ + u32 most = 1 + (1 + 1 + (3 * (1 - root->root.height + 1))); + + if (le32_to_cpu(alloc->avail.first_nr) < most) { + scoutfs_inc_counter(sb, alloc_list_avail_lo); + return false; + } + + if (list_block_space(alloc->freed.first_nr) < most) { + scoutfs_inc_counter(sb, alloc_list_freed_hi); + return false; + } + + return true; +} + +static bool lhead_in_alloc(struct scoutfs_alloc *alloc, + struct scoutfs_alloc_list_head *lhead) +{ + return lhead == &alloc->avail || lhead == &alloc->freed; +} + +/* + * Move free blocks from extent items in the root into only the first + * block in the list towards the target if it's fallen below the lo + * threshold. This can return success without necessarily moving as + * much as was requested if its meta allocator runs low, the caller is + * expected to check the counts and act accordingly. + * + * -ENOSPC is returned if the root runs out of extents before the list + * reaches the target. + */ +int scoutfs_alloc_fill_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *lhead, + struct scoutfs_alloc_root *root, + u64 lo, u64 target) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + .root = root, + .type = SCOUTFS_FREE_EXTENT_LEN_TYPE, + }; + struct scoutfs_alloc_list_block *lblk; + struct scoutfs_block *bl = NULL; + struct scoutfs_extent ext; + int ret = 0; + int i; + + if (WARN_ON_ONCE(target < lo) || + WARN_ON_ONCE(lo > SCOUTFS_ALLOC_LIST_MAX_BLOCKS) || + WARN_ON_ONCE(target > SCOUTFS_ALLOC_LIST_MAX_BLOCKS) || + WARN_ON_ONCE(lhead_in_alloc(alloc, lhead))) + return -EINVAL; + + if (le32_to_cpu(lhead->first_nr) >= lo) + return 0; + + ret = dirty_list_block(sb, alloc, wri, &lhead->ref, 0, NULL, &bl); + if (ret < 0) + goto out; + lblk = bl->data; + + while (le32_to_cpu(lblk->nr) < target && + list_can_cow(sb, alloc, root)) { + + ret = scoutfs_ext_alloc(sb, &alloc_ext_ops, &args, 0, 0, + target - le32_to_cpu(lblk->nr), &ext); + if (ret < 0) { + if (ret == -ENOENT) + ret = -ENOSPC; + break; + } + + for (i = 0; i < ext.len; i++) + list_block_add(lhead, lblk, ext.start + i); + } + +out: + scoutfs_block_put(sb, bl); + return ret; +} + +/* + * Move blknos from all the blocks in the list into extents in the root, + * removing empty blocks as we go. This can return success and leave blocks + * on the list if its metadata alloc runs out of space. + */ +int scoutfs_alloc_empty_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, + struct scoutfs_alloc_list_head *lhead) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + .root = root, + .type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE, + }; + struct scoutfs_alloc_list_block *lblk = NULL; + struct scoutfs_block *bl = NULL; + struct scoutfs_extent ext; + int ret = 0; + + if (WARN_ON_ONCE(lhead_in_alloc(alloc, lhead))) + return -EINVAL; + + while (lhead->ref.blkno && list_can_cow(sb, alloc, args.root)) { + + if (lhead->first_nr == 0) { + ret = trim_empty_first_block(sb, alloc, wri, lhead); + if (ret < 0) + break; + + scoutfs_block_put(sb, bl); + bl = NULL; + continue; + } + + if (bl == NULL) { + ret = dirty_list_block(sb, alloc, wri, &lhead->ref, + 0, NULL, &bl); + if (ret < 0) + break; + lblk = bl->data; + + /* sort to encourage forming extents */ + list_block_sort(lblk); + } + + /* combine free blknos into extents and insert them */ + ext.start = list_block_peek(lblk, 0); + ext.len = 1; + while ((le32_to_cpu(lblk->nr) > ext.len) && + (list_block_peek(lblk, ext.len) == ext.start + ext.len)) + ext.len++; + + ret = scoutfs_ext_insert(sb, &alloc_ext_ops, &args, + ext.start, ext.len, 0, 0); + if (ret < 0) + break; + + list_block_remove(lhead, lblk, ext.len); + } + + scoutfs_block_put(sb, bl); + + return ret; +} + +/* + * Insert the source list at the head of the destination list, leaving + * the source empty. + * + * This looks bad because the lists are singly-linked and we have to cow + * the entire src lsit to update its tail block next ref to the start of + * the dst list. + * + * In practice, this isn't a problem because the server only calls this + * with small lists that it's going to use soon. + */ +int scoutfs_alloc_splice_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *dst, + struct scoutfs_alloc_list_head *src) +{ + struct scoutfs_alloc_list_block *lblk; + struct scoutfs_alloc_list_ref *ref; + struct scoutfs_block *prev = NULL; + struct scoutfs_block *bl = NULL; + int ret = 0; + + if (WARN_ON_ONCE(lhead_in_alloc(alloc, dst)) || + WARN_ON_ONCE(lhead_in_alloc(alloc, src))) + return -EINVAL; + + if (src->ref.blkno == 0) + return 0; + + ref = &src->ref; + while (ref->blkno) { + ret = dirty_list_block(sb, alloc, wri, ref, 0, NULL, &bl); + if (ret < 0) + goto out; + + lblk = bl->data; + ref = &lblk->next; + + scoutfs_block_put(sb, prev); + prev = bl; + bl = NULL; + } + + *ref = dst->ref; + dst->ref = src->ref; + dst->first_nr = src->first_nr; + le64_add_cpu(&dst->total_nr, le64_to_cpu(src->total_nr)); + + memset(src, 0, sizeof(struct scoutfs_alloc_list_head)); + ret = 0; +out: + scoutfs_block_put(sb, prev); + scoutfs_block_put(sb, bl); + return ret; +} + +/* + * Returns true if we're running low on avail blocks or running out of + * space for freed blocks. + * + * On the avail side, we're avoiding spurious enospc as our avail block + * runs low. If we commit it can be refilled by the server. + * + * On the freed side, we're avoiding getting errors in frees where they + * can't be recovered from. This is mostly in freeing cowed blocks in + * the data allocator btree which is related to its height. + * + * And both of these need to be mindful of multiple tasks entering the + * transaction. + */ +bool scoutfs_alloc_meta_lo_thresh(struct super_block *sb, + struct scoutfs_alloc *alloc) +{ + bool lo; + + spin_lock(&alloc->lock); + lo = le32_to_cpu(alloc->avail.first_nr) < 8 || + list_block_space(alloc->freed.first_nr) < 8; + spin_unlock(&alloc->lock); + + return lo; +} diff --git a/kmod/src/alloc.h b/kmod/src/alloc.h new file mode 100644 index 00000000..167725b0 --- /dev/null +++ b/kmod/src/alloc.h @@ -0,0 +1,124 @@ +#ifndef _SCOUTFS_ALLOC_H_ +#define _SCOUTFS_ALLOC_H_ + +#include "ext.h" + +/* + * These are implementation-specific metrics, they don't need to be + * consistent across implementations. They should probably be run-time + * knobs. + */ + +/* + * The largest extent that we'll try to allocate with fallocate. We're + * trying not to completely consume a transactions data allocation all + * at once. This is only allocation granularity, repeated allocations + * can produce large contiguous extents. + */ +#define SCOUTFS_FALLOCATE_ALLOC_LIMIT \ + (128ULL * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * The largest aligned region that we'll try to allocate at the end of + * the file as it's extended. This is also limited to the current file + * size so we can only waste at most twice the total file size when + * files are less than this. We try to keep this around the point of + * diminishing returns in streaming performance of common data devices + * to limit waste. + */ +#define SCOUTFS_DATA_EXTEND_PREALLOC_LIMIT \ + (8ULL * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * Small data allocations are satisfied by cached extents stored in + * the run-time alloc struct to minimize item operations for small + * block allocations. Large allocations come directly from btree + * extent items, and this defines the threshold beetwen them. + */ +#define SCOUTFS_ALLOC_DATA_LG_THRESH \ + (8ULL * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * Fill client alloc roots to the target when they fall below the lo + * threshold. + */ +#define SCOUTFS_SERVER_META_FILL_TARGET \ + (256ULL * 1024 * 1024 >> SCOUTFS_BLOCK_LG_SHIFT) +#define SCOUTFS_SERVER_META_FILL_LO \ + (64ULL * 1024 * 1024 >> SCOUTFS_BLOCK_LG_SHIFT) +#define SCOUTFS_SERVER_DATA_FILL_TARGET \ + (4ULL * 1024 * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) +#define SCOUTFS_SERVER_DATA_FILL_LO \ + (1ULL * 1024 * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * Each of the server meta_alloc roots will try to keep a minimum amount + * of free blocks. The server will use the next root once its current + * root gets this low. It must have room for all the largest allocation + * attempted in a transaction on the server. + */ +#define SCOUTFS_SERVER_META_ALLOC_MIN \ + (SCOUTFS_SERVER_META_FILL_TARGET * 2) + +/* + * A run-time use of a pair of persistent avail/freed roots as a + * metadata allocator. It has the machinery needed to lock and avoid + * recursion when dirtying the list blocks that are used during the + * transaction. + */ +struct scoutfs_alloc { + spinlock_t lock; + struct mutex mutex; + struct scoutfs_block *dirty_avail_bl; + struct scoutfs_block *dirty_freed_bl; + struct scoutfs_alloc_list_head avail; + struct scoutfs_alloc_list_head freed; +}; + +void scoutfs_alloc_init(struct scoutfs_alloc *alloc, + struct scoutfs_alloc_list_head *avail, + struct scoutfs_alloc_list_head *freed); +int scoutfs_alloc_prepare_commit(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri); + +int scoutfs_alloc_meta(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 *blkno); +int scoutfs_free_meta(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 blkno); + +int scoutfs_alloc_data(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, + struct scoutfs_extent *cached, u64 count, + u64 *blkno_ret, u64 *count_ret); +int scoutfs_free_data(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, u64 blkno, u64 count); + +int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *dst, + struct scoutfs_alloc_root *src, u64 total); + +int scoutfs_alloc_fill_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *lhead, + struct scoutfs_alloc_root *root, + u64 lo, u64 target); +int scoutfs_alloc_empty_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, + struct scoutfs_alloc_list_head *lhead); +int scoutfs_alloc_splice_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *dst, + struct scoutfs_alloc_list_head *src); + +bool scoutfs_alloc_meta_lo_thresh(struct super_block *sb, + struct scoutfs_alloc *alloc); + +#endif diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 2230ced2..b8686bc9 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -12,6 +12,15 @@ * other places by this macro. Don't forget to update LAST_COUNTER. */ #define EXPAND_EACH_COUNTER \ + EXPAND_COUNTER(alloc_alloc_data) \ + EXPAND_COUNTER(alloc_alloc_meta) \ + EXPAND_COUNTER(alloc_free_data) \ + EXPAND_COUNTER(alloc_free_meta) \ + EXPAND_COUNTER(alloc_list_avail_lo) \ + EXPAND_COUNTER(alloc_list_freed_hi) \ + EXPAND_COUNTER(alloc_move) \ + EXPAND_COUNTER(alloc_moved_extent) \ + EXPAND_COUNTER(alloc_stale_cached_list_block) \ EXPAND_COUNTER(block_cache_access) \ EXPAND_COUNTER(block_cache_alloc_failure) \ EXPAND_COUNTER(block_cache_alloc_page_order) \ @@ -185,7 +194,7 @@ EXPAND_COUNTER(trans_commit_timer) \ EXPAND_COUNTER(trans_commit_written) -#define FIRST_COUNTER block_cache_access +#define FIRST_COUNTER alloc_alloc_data #define LAST_COUNTER trans_commit_written #undef EXPAND_COUNTER diff --git a/kmod/src/format.h b/kmod/src/format.h index cd33a7c6..15bd7d92 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -11,6 +11,7 @@ #define SCOUTFS_BLOCK_MAGIC_RADIX 0xebeb5e65 #define SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK 0x897e4a7d #define SCOUTFS_BLOCK_MAGIC_SRCH_PARENT 0xb23a2a05 +#define SCOUTFS_BLOCK_MAGIC_ALLOC_LIST 0x8a93ac83 /* * The super block, quorum block, and file data allocation granularity @@ -266,6 +267,53 @@ struct scoutfs_btree_block { #define SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES \ (SCOUTFS_BTREE_LEAF_ITEM_HASH_NR * sizeof(__le16)) +struct scoutfs_alloc_list_ref { + __le64 blkno; + __le64 seq; +}__packed; + +/* + * first_nr tracks the nr of the first block in the list and is used for + * allocation sizing. total_nr is the sum of the nr of all the blocks in + * the list and is used for calculating total free block counts. + */ +struct scoutfs_alloc_list_head { + struct scoutfs_alloc_list_ref ref; + __le64 total_nr; + __le32 first_nr; +}__packed; + +/* + * While the main allocator uses extent items in btree blocks, metadata + * allocations for a single transaction are recorded in arrays in + * blocks. This limits the number of allocations and frees needed to + * cow and modify the structure. The blocks can be stored in a list + * which lets us create a persistent log of pending frees that are + * generated as we cow btree blocks to insert freed extents. + * + * The array floats in the block so that both adding and removing blknos + * only modifies an index. + */ +struct scoutfs_alloc_list_block { + struct scoutfs_block_header hdr; + struct scoutfs_alloc_list_ref next; + __le32 start; + __le32 nr; + __le64 blknos[0]; /* naturally aligned for sorting */ +}__packed; + +#define SCOUTFS_ALLOC_LIST_MAX_BLOCKS \ + ((SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_alloc_list_block)) / \ + (member_sizeof(struct scoutfs_alloc_list_block, blknos[0]))) + +/* + * These can safely be initialized to all-zeros. + */ +struct scoutfs_alloc_root { + __le64 total_len; + struct scoutfs_btree_root root; +}__packed; + struct scoutfs_mounted_client_btree_val { __u8 flags; } __packed; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 5ecb3a3b..a4d58bca 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2490,6 +2490,123 @@ TRACE_EVENT(scoutfs_ext_alloc, STE_ENTRY_ARGS(ext), __entry->ret) ); +TRACE_EVENT(scoutfs_alloc_alloc_meta, + TP_PROTO(struct super_block *sb, u64 blkno, int ret), + + TP_ARGS(sb, blkno, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = blkno; + __entry->ret = ret; + ), + + TP_printk(SCSBF" blkno %llu ret %d", + SCSB_TRACE_ARGS, __entry->blkno, __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_free_meta, + TP_PROTO(struct super_block *sb, u64 blkno, int ret), + + TP_ARGS(sb, blkno, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = blkno; + __entry->ret = ret; + ), + + TP_printk(SCSBF" blkno %llu ret %d", + SCSB_TRACE_ARGS, __entry->blkno, __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_alloc_data, + TP_PROTO(struct super_block *sb, u64 req, u64 blkno, u64 count, + int ret), + + TP_ARGS(sb, req, blkno, count, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, req) + __field(__u64, blkno) + __field(__u64, count) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->req = req; + __entry->blkno = blkno; + __entry->count = count; + __entry->ret = ret; + ), + + TP_printk(SCSBF" req %llu blkno %llu count %llu ret %d", + SCSB_TRACE_ARGS, __entry->req, __entry->blkno, + __entry->count, __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_free_data, + TP_PROTO(struct super_block *sb, u64 blkno, u64 count, int ret), + + TP_ARGS(sb, blkno, count, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(__u64, count) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = blkno; + __entry->count = count; + __entry->ret = ret; + ), + + TP_printk(SCSBF" blkno %llu count %llu ret %d", + SCSB_TRACE_ARGS, __entry->blkno, __entry->count, + __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_move, + TP_PROTO(struct super_block *sb, u64 total, u64 moved, int ret), + + TP_ARGS(sb, total, moved, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, total) + __field(__u64, moved) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->total = total; + __entry->moved = moved; + __entry->ret = ret; + ), + + TP_printk(SCSBF" total %llu moved %llu ret %d", + SCSB_TRACE_ARGS, __entry->total, __entry->moved, + __entry->ret) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */