From dee9fbcf662683a479ab1ba04d665ebd0fa106a3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 8 Nov 2019 10:24:44 -0800 Subject: [PATCH] scoutfs: use packed extents and bitmaps The btree forest item storage doesn't have as much item granular state as the item cache did. The item cache could tell if a cached item was populated from persistent storage or was created in memory. It could simply remove created items rather than leaving behind a deletion item. The cached btree blocks in the btree forest item storage mechanism can't do this. It has to create deletion items when deleting newly created items because it doesn't know if the item already exists in the persistent record or not. This created a problem with the extent storage we were using. The individual extent items were stored with a key set to the last logical block of their extent. As extents grew or shrank they often were deleted and created at different key values during a transaction. In the btree forest log trees this left a huge stream of deletion items beind, one for every previous version of the extent. Then searches for an extent covering a block would have to skip over all these deleted items before hitting the current stored extent. Streaming writes would operate on O(n) for every extent operation. It got to be out of hand. This large change solves the problem by using more coarse and stable item storage to track free blocks and blocks mapped into file data. For file data we now have large packed extent items which store packed representations of all the logical mappings of a fixed region of a file. The data code has loading and storage functions which transfer that persistent version to and from the version that is modified in memory. Free blocks are stored in bitmaps that are similarly efficiently packed into fixed size items. The client is no longer working with free extent items managed by the forest, it's working with free block bitmap btrees directly. It needs access to the client's metadata block allocator and block write contexts so we move those two out of the forest code and up into the transaction. Previously the client and server would exchange extents with network messages. Now the roots of the btrees that store the free block bitmap items are communicated along with the roots of the other trees involved in a transaction. The client doesn't need to send free extents back to the server so we can remove those tasks and rpcs. The server no longer has to manage free extents. It transfers block bitmap items between trees around commits. All of its extent manipulation can be removed. The item size portion of transaction item counts are removed because we're not using that level of granularity now that metadata transactions are dirty btree blocks instead of dirty items we pack into fixed sized segments. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 - kmod/src/client.c | 42 - kmod/src/client.h | 4 - kmod/src/count.h | 22 +- kmod/src/counters.h | 5 - kmod/src/data.c | 2446 ++++++++++++++++++++++++++------------ kmod/src/data.h | 27 + kmod/src/forest.c | 87 +- kmod/src/forest.h | 14 +- kmod/src/format.h | 121 +- kmod/src/ioctl.c | 14 +- kmod/src/lock.c | 2 +- kmod/src/scoutfs_trace.h | 232 +--- kmod/src/server.c | 534 +++------ kmod/src/super.c | 2 +- kmod/src/trans.c | 63 +- kmod/src/trans.h | 3 + 17 files changed, 2081 insertions(+), 1538 deletions(-) diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 8c9d0fdb..5772a9fc 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -17,7 +17,6 @@ scoutfs-y += \ data.o \ dir.o \ export.o \ - extents.o \ file.o \ forest.o \ inode.o \ diff --git a/kmod/src/client.c b/kmod/src/client.c index dd8abf03..83dbacab 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -88,48 +88,6 @@ int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, return ret; } -/* - * Ask the server for an extent of at most @blocks blocks. It can return - * smaller extents. - */ -int scoutfs_client_alloc_extent(struct super_block *sb, u64 blocks, u64 *start, - u64 *len) - -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - __le64 leblocks = cpu_to_le64(blocks); - struct scoutfs_net_extent nex; - int ret; - - ret = scoutfs_net_sync_request(sb, client->conn, - SCOUTFS_NET_CMD_ALLOC_EXTENT, - &leblocks, sizeof(leblocks), - &nex, sizeof(nex)); - if (ret == 0) { - if (nex.len == 0) { - ret = -ENOSPC; - } else { - *start = le64_to_cpu(nex.start); - *len = le64_to_cpu(nex.len); - } - } - - return ret; -} - -int scoutfs_client_free_extents(struct super_block *sb, - struct scoutfs_net_extent_list *nexl) -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - unsigned int bytes; - - bytes = SCOUTFS_NET_EXTENT_LIST_BYTES(le64_to_cpu(nexl->nr)); - - return scoutfs_net_sync_request(sb, client->conn, - SCOUTFS_NET_CMD_FREE_EXTENTS, - nexl, bytes, NULL, 0); -} - int scoutfs_client_get_log_trees(struct super_block *sb, struct scoutfs_log_trees *lt) { diff --git a/kmod/src/client.h b/kmod/src/client.h index 9ee65a25..cb77c30c 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -3,10 +3,6 @@ int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, u64 *ino, u64 *nr); -int scoutfs_client_alloc_extent(struct super_block *sb, u64 blocks, u64 *start, - u64 *len); -int scoutfs_client_free_extents(struct super_block *sb, - struct scoutfs_net_extent_list *nexl); int scoutfs_client_get_log_trees(struct super_block *sb, struct scoutfs_log_trees *lt); int scoutfs_client_commit_log_trees(struct super_block *sb, diff --git a/kmod/src/count.h b/kmod/src/count.h index 25e4fd0c..97d56ef5 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -252,7 +252,7 @@ static inline const struct scoutfs_item_count SIC_WRITE_BEGIN(void) __count_dirty_inode(&cnt); cnt.items += nr_free + nr_file; - cnt.vals += nr_file * sizeof(struct scoutfs_file_extent); + cnt.vals += nr_file; return cnt; } @@ -276,22 +276,7 @@ SIC_TRUNC_EXTENT(struct inode *inode) __count_dirty_inode(&cnt); cnt.items += nr_file + nr_free; - cnt.vals += nr_file * sizeof(struct scoutfs_file_extent); - - return cnt; -} - -/* - * Returning extents to the server can, at most: - * - delete MAX_NR extents with indexed copies - * - create an extent for the leftovers of the last extent - */ -static inline const struct scoutfs_item_count SIC_RETURN_EXTENTS(void) -{ - struct scoutfs_item_count cnt = {0,}; - unsigned int nr = SCOUTFS_NET_EXTENT_LIST_MAX_NR + 1; - - cnt.items += (nr * 2); + cnt.vals += nr_file; return cnt; } @@ -312,7 +297,7 @@ static inline const struct scoutfs_item_count SIC_FALLOCATE_ONE(void) __count_dirty_inode(&cnt); cnt.items += nr_free + nr_file; - cnt.vals += nr_file * sizeof(struct scoutfs_file_extent); + cnt.vals += nr_file; return cnt; } @@ -327,7 +312,6 @@ static inline const struct scoutfs_item_count SIC_SETATTR_MORE(void) __count_dirty_inode(&cnt); cnt.items++; - cnt.vals += sizeof(struct scoutfs_file_extent); return cnt; } diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 628d0493..932e8a4b 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -105,11 +105,6 @@ EXPAND_COUNTER(quorum_write_block) \ EXPAND_COUNTER(quorum_write_block_error) \ EXPAND_COUNTER(quorum_fenced) \ - EXPAND_COUNTER(server_extent_alloc) \ - EXPAND_COUNTER(server_extent_alloc_error) \ - EXPAND_COUNTER(server_free_extent) \ - EXPAND_COUNTER(server_free_pending_extent) \ - EXPAND_COUNTER(server_free_pending_error) \ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ EXPAND_COUNTER(trans_commit_sync_fs) \ diff --git a/kmod/src/data.c b/kmod/src/data.c index b2deb16b..1d3c29e0 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public @@ -21,7 +21,6 @@ #include #include #include -#include #include "format.h" #include "super.h" @@ -34,339 +33,1346 @@ #include "scoutfs_trace.h" #include "forest.h" #include "ioctl.h" -#include "client.h" +#include "btree.h" #include "lock.h" #include "file.h" -#include "extents.h" #include "msg.h" #include "count.h" /* - * scoutfs uses extent items to track file data block mappings and free - * blocks. + * Logical file blocks are mapped to device blocks with extents stored + * in items. Each extent item maps a fixed size logical region and can + * contain multiple extent records. Each extent record is packed to + * minimize the space it uses. The logical starting block is implicit + * so sparse extents are stored to skip unmapped blocks, and the mapped + * blkno is encoded as the difference from the previous extent and only + * its set bytes are stored. * - * Typically we'll allocate a single block in get_block if a mapping - * isn't found. + * To operate on the extents we load their item and unpack them into an + * rbtree of full extent records in memory. Once the memory extents are + * modified they can be packed back into the item. Typically there are + * very few extents that cover the region. * - * We special case extending contiguous files. In that case we'll preallocate - * an unwritten extent at the end of the file. The size of the preallocation - * is based on the file size and is capped. + * Free blocks are tracked with bitmaps that are stored in items. Again + * the bitmaps are stored in a packed form and operated on in memory in + * a native form. Only 64bit words with a mix of set and clear bits are + * stored. The bitmaps are translated into long bitmaps in memory so we + * can use the kernel's long bitmap interfaces. * - * XXX - * - truncate - * - mmap - * - better io error propagation - * - forced unmount with dirty data - * - direct IO - * - need trans around each bulk alloc + * There are two types of bitmap items: little bitmap bits track + * individual blocks and large bitmap bits track full little bitmap + * items. The logical packed extent item and little bitmap item sizes + * are chosen such that a full little bitmap represents a full packed + * extent item so we can allocate maximum size extents from the large + * bitmap bits. + * + * The client is given a tree of free block bitmap items from the server + * at the start of each transaction. The client allocates from items in + * an allocation tree and frees into items in a free tree. The server + * is responsible for filling the alloc tree and reclaiming the free + * tree as transactions are opened and committed. */ -/* - * The largest extent that we'll store in a single item. This will - * determine the granularity of interleaved concurrent allocations in a - * mount. Sequential max length allocations could still see contiguous - * physical extent allocations. It limits the amount of IO needed to - * invalidate a lock. And it determines the granularity of parallel - * writes to a file between nodes. - */ -#define MAX_EXTENT_BLOCKS (8ULL * 1024 * 1024 >> SCOUTFS_BLOCK_SHIFT) -/* - * We ask for a fixed size from the server today. - */ -#define SERVER_ALLOC_BLOCKS (MAX_EXTENT_BLOCKS * 8) -/* - * Send free extents back to the server if we have plenty locally. - */ -#define NODE_FREE_HIGH_WATER_BLOCKS (SERVER_ALLOC_BLOCKS * 16) - struct data_info { struct super_block *sb; struct rw_semaphore alloc_rwsem; - atomic64_t node_free_blocks; - struct workqueue_struct *workq; - struct work_struct return_work; + struct scoutfs_balloc_allocator *alloc; + struct scoutfs_block_writer *wri; + struct scoutfs_balloc_root data_alloc; + struct scoutfs_balloc_root data_free; }; #define DECLARE_DATA_INFO(sb, name) \ struct data_info *name = SCOUTFS_SB(sb)->data_info -static void init_file_extent_key(struct scoutfs_key *key, u64 ino, u64 last) +static void init_packed_extent_key(struct scoutfs_key *key, u64 ino, + u64 iblock, u8 part) { *key = (struct scoutfs_key) { .sk_zone = SCOUTFS_FS_ZONE, - .skfe_ino = cpu_to_le64(ino), - .sk_type = SCOUTFS_FILE_EXTENT_TYPE, - .skfe_last = cpu_to_le64(last), + .skpe_ino = cpu_to_le64(ino), + .sk_type = SCOUTFS_PACKED_EXTENT_TYPE, + .skpe_base = cpu_to_le64(iblock >> SCOUTFS_PACKEXT_BASE_SHIFT), + .skpe_part = part, }; } -static void init_free_extent_key(struct scoutfs_key *key, u8 type, u64 rid, - u64 major, u64 minor) -{ - *key = (struct scoutfs_key) { - .sk_zone = SCOUTFS_RID_ZONE, - .sknf_rid = cpu_to_le64(rid), - .sk_type = type, - .sknf_major = cpu_to_le64(major), - .sknf_minor = cpu_to_le64(minor), - }; -} - -static int init_extent_from_item(struct scoutfs_extent *ext, - struct scoutfs_key *key, - struct scoutfs_file_extent *fex) -{ - u64 owner; - u64 start; - u64 map; - u64 len; - u8 flags; - - if (key->sk_type != SCOUTFS_FILE_EXTENT_TYPE && - key->sk_type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE && - key->sk_type != SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) - return -EIO; /* XXX corruption, unknown key type */ - - if (key->sk_type == SCOUTFS_FILE_EXTENT_TYPE) { - owner = le64_to_cpu(key->skfe_ino); - len = le64_to_cpu(fex->len); - start = le64_to_cpu(key->skfe_last) - len + 1; - map = le64_to_cpu(fex->blkno); - flags = fex->flags; - - } else { - owner = le64_to_cpu(key->sknf_rid); - start = le64_to_cpu(key->sknf_major); - len = le64_to_cpu(key->sknf_minor); - if (key->sk_type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) - swap(start, len); - start -= len - 1; - map = 0; - flags = 0; - } - - return scoutfs_extent_init(ext, key->sk_type, owner, start, len, map, - flags); -} - /* - * Read and write file extent and free extent items. - * - * File extents and free extents are indexed by the last position in the - * extent so that we can find intersections with _next. - * - * We also index free extents by their length. We implement that by - * keeping their _BLOCKS_ item in sync with the primary _BLKNO_ item - * that callers operate on. - * - * The count of free blocks stored in items is kept consistent by - * updating the count every time we create or delete items. Updated - * extents are deleted and then recreated so the count can bounce around - * a bit, but it's OK for it to be imprecise at the margins. + * Packed extents are read from items and unpacked into this structure + * in memory so they can be easily manipulated before being packed and + * stored in items. */ -static int data_extent_io(struct super_block *sb, int op, - struct scoutfs_extent *ext, void *data) +struct unpacked_extents { + u64 iblock; + struct rb_root extents; + __u8 existing_parts; + bool changed; +}; + +struct unpacked_extent { + struct rb_node node; + u64 iblock; + u64 count; + u64 blkno; + u8 flags; +}; + +static void init_traced_extent(struct scoutfs_traced_extent *te, + u64 iblock, u64 count, u64 blkno, u8 flags) { - DECLARE_DATA_INFO(sb, datinf); - struct scoutfs_lock *lock = data; - struct scoutfs_file_extent fex; - struct scoutfs_key first; - struct scoutfs_key last; - struct scoutfs_key key; - struct kvec val; - bool mirror = false; - u8 mirror_type; - u8 mirror_op = 0; - int expected; - int ret; - int err; + te->iblock = iblock; + te->count = count; + te->blkno = blkno; + te->flags = flags; +} - if (WARN_ON_ONCE(ext->type != SCOUTFS_FILE_EXTENT_TYPE && - ext->type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE && - ext->type != SCOUTFS_FREE_EXTENT_BLOCKS_TYPE)) - return -EINVAL; +static void copy_traced_extent(struct scoutfs_traced_extent *te, + struct unpacked_extent *ext) +{ + te->iblock = ext->iblock; + te->count = ext->count; + te->blkno = ext->blkno; + te->flags = ext->flags; +} - if (ext->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE && - (op == SEI_INSERT || op == SEI_DELETE)) { - mirror = true; - mirror_type = SCOUTFS_FREE_EXTENT_BLOCKS_TYPE; - mirror_op = op == SEI_INSERT ? SEI_DELETE : SEI_INSERT; - } +static u64 ext_last(struct unpacked_extent *ext) +{ + return ext->iblock + ext->count - 1; +} - if (ext->type == SCOUTFS_FILE_EXTENT_TYPE) { - init_file_extent_key(&key, ext->owner, - ext->start + ext->len - 1); - init_file_extent_key(&first, ext->owner, 0); - init_file_extent_key(&last, ext->owner, U64_MAX); - fex.blkno = cpu_to_le64(ext->map); - fex.len = cpu_to_le64(ext->len); - fex.flags = ext->flags; - kvec_init(&val, &fex, sizeof(fex)); - } else { - init_free_extent_key(&key, ext->type, ext->owner, - ext->start + ext->len - 1, ext->len); - if (ext->type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) - swap(key.sknf_major, key.sknf_minor); - init_free_extent_key(&first, ext->type, ext->owner, - 0, 0); - init_free_extent_key(&last, ext->type, ext->owner, - U64_MAX, U64_MAX); - kvec_init(&val, NULL, 0); - } +static u64 bitmap_base(u64 blkno) +{ + return blkno >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; +} - if (op == SEI_NEXT || op == SEI_PREV) { - expected = val.iov_len; +/* The first possible iblock in an item that contains the given iblock */ +static u64 first_iblock(u64 iblock) +{ + return iblock & SCOUTFS_PACKEXT_BASE_MASK; +} - if (op == SEI_NEXT) - ret = scoutfs_forest_next(sb, &key, &last, &val, lock); - else - ret = scoutfs_forest_prev(sb, &key, &first, &val, lock); - if (ret >= 0 && ret != expected) - ret = -EIO; - if (ret == expected) - ret = init_extent_from_item(ext, &key, &fex); +/* The last possible iblock in an item that contains the given iblock */ +static u64 last_iblock(u64 iblock) +{ + return iblock | ~SCOUTFS_PACKEXT_BASE_MASK; +} - } else if (op == SEI_INSERT) { - ret = scoutfs_forest_create(sb, &key, &val, lock); +/* + * Extents can merge if they're logically contiguous, have block + * mappings or not which also must be contiguous, and have matching + * flags. + * + * We also require that a given extent's allocation be from only one + * bitmap item because the block bitmap clearing functions only operate + * on one item. + */ +static bool extents_merge(struct unpacked_extent *left, + struct unpacked_extent *right) +{ + return (left->iblock + left->count == right->iblock) && + ((!left->blkno && !right->blkno) || + (left->blkno + left->count == right->blkno)) && + (left->flags == right->flags) && + (bitmap_base(left->blkno) == bitmap_base(right->blkno)); +} - } else if (op == SEI_DELETE) { - ret = scoutfs_forest_delete(sb, &key, lock); +static struct unpacked_extent *first_extent(struct unpacked_extents *unpe) +{ + return rb_entry_safe(rb_first(&unpe->extents), + struct unpacked_extent, node); +} - } else { - ret = WARN_ON_ONCE(-EINVAL); - } +static struct unpacked_extent *last_extent(struct unpacked_extents *unpe) +{ + return rb_entry_safe(rb_last(&unpe->extents), + struct unpacked_extent, node); +} - if (ret == 0 && mirror) { - swap(ext->type, mirror_type); - ret = data_extent_io(sb, op, ext, data); - swap(ext->type, mirror_type); - if (ret) { - err = data_extent_io(sb, mirror_op, ext, data); - BUG_ON(err); +static struct unpacked_extent *next_extent(struct unpacked_extent *ext) +{ + return rb_entry_safe(rb_next(&ext->node), + struct unpacked_extent, node); +} + +static struct unpacked_extent *prev_extent(struct unpacked_extent *ext) +{ + return rb_entry_safe(rb_prev(&ext->node), + struct unpacked_extent, node); +} + +/* + * Find the first extent that intersects the requested range. NULL is + * returned if no extents intersect. + */ +static struct unpacked_extent *find_extent(struct unpacked_extents *unpe, + u64 iblock, u64 last) +{ + + struct rb_node *node = unpe->extents.rb_node; + struct unpacked_extent *ret = NULL; + struct unpacked_extent *ext; + + if (iblock > last) + return NULL; + + while (node) { + ext = rb_entry(node, struct unpacked_extent, node); + + if (last < ext->iblock) { + node = node->rb_left; + } else if (iblock > ext_last(ext)) { + node = node->rb_right; + } else { + ret = ext; + node = node->rb_left; } } - if (ret == 0 && ext->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { - if (op == SEI_INSERT) - atomic64_add(ext->len, &datinf->node_free_blocks); - else if (op == SEI_DELETE) - atomic64_sub(ext->len, &datinf->node_free_blocks); + return ret; +} + +static void track_blocks(struct unpacked_extent *ext, s64 delta, + s64 *on, s64 *off) +{ + if (ext->blkno && !(ext->flags & SEF_UNWRITTEN)) + *on += delta; + else if (ext->flags & SEF_OFFLINE) + *off += delta; +} + +static void modify_and_track_count(struct unpacked_extent *ext, u64 count, + s64 *on, s64 *off) +{ + track_blocks(ext, count - ext->count, on, off); + ext->count = count; +} + +/* + * Callers can temporarily insert extents with equal starting iblocks. + * We're careful to insert those to the left so that caller's can find + * these existing overlapping extents by iterating with next. + */ +static void insert_extent(struct unpacked_extents *unpe, + struct unpacked_extent *ins, s64 *on, s64 *off) +{ + struct rb_node **node = &unpe->extents.rb_node; + struct rb_node *parent = NULL; + struct unpacked_extent *ext; + int cmp; + + while (*node) { + parent = *node; + ext = rb_entry(*node, struct unpacked_extent, node); + + cmp = scoutfs_cmp_u64s(ins->iblock, ext->iblock); + if (cmp <= 0) + node = &(*node)->rb_left; + else + node = &(*node)->rb_right; + } + + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, &unpe->extents); + + track_blocks(ins, ins->count, on, off); +} + +static void remove_extent(struct unpacked_extents *unpe, + struct unpacked_extent *ext, s64 *on, s64 *off) +{ + rb_erase(&ext->node, &unpe->extents); + track_blocks(ext, -ext->count, on, off); + kfree(ext); +} + +static void free_unpacked_extents(struct unpacked_extents *unpe) +{ + struct unpacked_extent *ext; + struct unpacked_extent *tmp; + + if (unpe) { + rbtree_postorder_for_each_entry_safe(ext, tmp, &unpe->extents, + node) { + kfree(ext); + } + kfree(unpe); + } +} + +static int unpack_extent(struct unpacked_extent *ext, u64 iblock, + struct scoutfs_packed_extent *pe, int size, + u64 prev_blkno) +{ + __le64 lediff; + u64 blkno; + u64 diff; + + if (size < sizeof(struct scoutfs_packed_extent) || + size < (sizeof(struct scoutfs_packed_extent) + pe->diff_bytes)) + return 0; + + if (pe->diff_bytes) { + lediff = 0; + memcpy(&lediff, pe->le_blkno_diff, pe->diff_bytes); + diff = le64_to_cpu(lediff); + diff = (diff >> 1) ^ (-(diff & 1)); + blkno = prev_blkno + diff; + } else { + blkno = 0; + } + + ext->iblock = iblock; + ext->blkno = blkno; + ext->count = le16_to_cpu(pe->count); + ext->flags = pe->flags; + + return sizeof(struct scoutfs_packed_extent) + pe->diff_bytes; +} + +static int load_unpacked_extents(struct super_block *sb, u64 ino, + u64 iblock, u64 last, bool empty_enoent, + struct unpacked_extents **unpe_ret, + struct scoutfs_lock *lock) +{ + struct unpacked_extents *unpe = NULL; + struct scoutfs_packed_extent *pe; + struct unpacked_extent *ext; + struct scoutfs_key key; + struct scoutfs_key end; + struct rb_node *parent; + struct rb_node **node; + void *buf = NULL; + struct kvec val; + u64 prev_blkno; + bool saw_final; + int size; + int ret; + int p; + + *unpe_ret = NULL; + + unpe = kzalloc(sizeof(struct unpacked_extents), GFP_NOFS); + if (!unpe) { + ret = -ENOMEM; + goto out; + } + + unpe->extents = RB_ROOT; + unpe->changed = true; + /* updated later if _next gives us a greater key */ + unpe->iblock = first_iblock(iblock); + + buf = kmalloc(SCOUTFS_PACKEXT_MAX_BYTES, GFP_NOFS); + if (!buf) { + ret = -ENOMEM; + goto out; + } + + if (last > iblock) + init_packed_extent_key(&end, ino, last, 0); + + parent = NULL; + node = &unpe->extents.rb_node; + prev_blkno = 0; + saw_final = false; + + for (p = 0; !saw_final; p++) { + init_packed_extent_key(&key, ino, iblock, p); + kvec_init(&val, buf, SCOUTFS_PACKEXT_MAX_BYTES); + + /* maybe search for next initial item, lookup more parts */ + if (p == 0 && last > iblock) + ret = scoutfs_forest_next(sb, &key, &end, &val, lock); + else + ret = scoutfs_forest_lookup(sb, &key, &val, lock); + if (ret < 0) { + if (p == 0 && ret == -ENOENT && empty_enoent) + ret = 0; + goto out; + } + + if (key.skpe_part != p) { + ret = -EIO; /* corruption */ + goto out; + } + + if (p == 0) { + iblock = le64_to_cpu(key.skpe_base) << + SCOUTFS_PACKEXT_BASE_SHIFT; + unpe->iblock = iblock; + } + pe = buf; + size = ret; + + while (size > 0) { + ext = kmalloc(sizeof(struct unpacked_extent), GFP_NOFS); + if (!ext) { + ret = -ENOMEM; + goto out; + } + + ret = unpack_extent(ext, iblock, pe, size, prev_blkno); + if (ret == 0) { /* XXX corruption? */ + kfree(ext); + ret = -EIO; + goto out; + } + + saw_final = pe->final; + pe = (void *)pe + ret; + size -= ret; + + /* sparse packed extents advance iblock */ + if (ext->flags == 0 && ext->blkno == 0) { + iblock += ext->count; + kfree(ext); + ext = NULL; + continue; + } + + iblock += ext->count; + prev_blkno = ext->blkno + ext->count - 1; + + /* building the rbtree from sorted nodes */ + rb_link_node(&ext->node, parent, node); + rb_insert_color(&ext->node, &unpe->extents); + parent = &ext->node; + node = &ext->node.rb_right; + + if (saw_final) + unpe->existing_parts = p + 1; + } + } + + ret = 0; +out: + kfree(buf); + if (ret < 0) + free_unpacked_extents(unpe); + else + *unpe_ret = unpe; + + return ret; +} + +static int pack_extent(struct scoutfs_packed_extent *pe, int size, + struct unpacked_extent *ext, + u64 prev_blkno, bool final) +{ + int diff_bytes; + __le64 lediff; + u64 diff; + int bytes; + int last; + + diff = ext->blkno - prev_blkno; + diff = (diff << 1) ^ ((s64)diff >> 63); /* shift sign extend */ + lediff = cpu_to_le64(diff); + last = fls64(diff); + diff_bytes = (last + 7) >> 3; + + bytes = offsetof(struct scoutfs_packed_extent, + le_blkno_diff[diff_bytes]); + if (size < bytes) + return 0; + + pe->count = cpu_to_le16(ext->count); + pe->diff_bytes = diff_bytes; + pe->flags = ext->flags; + pe->final = !!final; + if (diff_bytes) + memcpy(pe->le_blkno_diff, &lediff, diff_bytes); + + return bytes; +} + +static int store_packed_extents(struct super_block *sb, u64 ino, + struct unpacked_extents *unpe, + struct scoutfs_lock *lock) +{ + struct scoutfs_packed_extent *pe; + struct unpacked_extent *final; + struct unpacked_extent *ext; + struct scoutfs_key key; + struct kvec val; + void *buf = NULL; + u64 prev_blkno; + u64 iblock; + int space; + int size; + int ret; + int p; + int i; + + if (!unpe->changed) + return 0; + + if (RB_EMPTY_ROOT(&unpe->extents)) { + for (p = 0; p < unpe->existing_parts; p++) { + init_packed_extent_key(&key, ino, unpe->iblock, p); + ret = scoutfs_forest_delete(sb, &key, lock); + BUG_ON(ret); /* XXX inconsistent between parts */ + } + unpe->existing_parts = 0; + unpe->changed = false; + return 0; + } + + buf = kmalloc(SCOUTFS_PACKEXT_MAX_BYTES, GFP_NOFS); + if (!buf) { + ret = -ENOMEM; + goto out; + } + + final = last_extent(unpe); + prev_blkno = 0; + + pe = buf; + space = SCOUTFS_PACKEXT_MAX_BYTES; + size = 0; + p = 0; + iblock = unpe->iblock; + + ext = first_extent(unpe); + while (ext) { + /* encode sparse extent to advance iblock */ + if (ext->iblock > iblock && space >= sizeof(*pe)) { + pe->count = cpu_to_le16(ext->iblock - iblock); + pe->diff_bytes = 0; + pe->flags = 0; + pe->final = 0; + pe++; + space -= sizeof(*pe); + size += sizeof(*pe); + iblock = ext->iblock; + } + + /* encode actual extent */ + if (ext->iblock == iblock && + (ret = pack_extent(pe, space, ext, prev_blkno, + ext == final)) > 0) { + pe = (void *)pe + ret; + space -= ret; + size += ret; + iblock += ext->count; + prev_blkno = ext->blkno + ext->count - 1; + ext = next_extent(ext); + if (ext) + continue; + } + + /* store full item or after packing final extent */ + init_packed_extent_key(&key, ino, unpe->iblock, p); + kvec_init(&val, buf, size); + if (p < unpe->existing_parts) + ret = scoutfs_forest_update(sb, &key, &val, lock); + else + ret = scoutfs_forest_create(sb, &key, &val, lock); + BUG_ON(ret); /* XXX inconsistent between parts */ + + pe = buf; + space = SCOUTFS_PACKEXT_MAX_BYTES; + size = 0; + p++; + } + + /* delete any remaining previous part items */ + for (i = p; i < unpe->existing_parts; i++) { + init_packed_extent_key(&key, ino, unpe->iblock, i); + ret = scoutfs_forest_delete(sb, &key, lock); + BUG_ON(ret); /* XXX inconsistent between parts */ + } + + /* the next store has to know our stored parts */ + unpe->existing_parts = p; + unpe->changed = false; + ret = 0; +out: + kfree(buf); + + return ret; +} + +/* + * Set a logical extent mapping in the unpacked extents for a region of + * a file. The caller's extent is authoritative, any existing + * overlapping extents are trimmed or removed. The new extent can be + * merged with remaining adjacent and compatible extents. + * + * If the caller provides an inode struct then we'll keep the inode + * block counts in sync with flagged extents because updating the inode + * counts won't fail. The caller is expected to keep all other state + * consistent with the extents (i_size, i_blocks, allocator bitmaps). + */ +static int set_extent(struct super_block *sb, struct inode *inode, + u64 ino, struct unpacked_extents *unpe, + u64 iblock, u64 blkno, u64 count, u8 flags) +{ + struct unpacked_extent *split; + struct unpacked_extent *next; + struct unpacked_extent *prev; + struct unpacked_extent *ext; + u64 offset; + s64 on = 0; + s64 off = 0; + + /* make sure the given extent fits entirely within one item */ + if (WARN_ON_ONCE(first_iblock(iblock) != + first_iblock(iblock + count - 1))) + return -EINVAL; + + ext = kmalloc(sizeof(struct unpacked_extent), GFP_NOFS); + split = kmalloc(sizeof(struct unpacked_extent), GFP_NOFS); + if (!ext || !split) { + kfree(ext); + kfree(split); + return -ENOMEM; + } + + unpe->changed = true; + + ext->iblock = iblock; + ext->blkno = blkno; + ext->count = count; + ext->flags = flags; + + insert_extent(unpe, ext, &on, &off); + + prev = prev_extent(ext); + + /* splitting an existing extent? */ + if (prev && ext_last(prev) > ext_last(ext)) { + split->iblock = ext_last(ext) + 1; + split->count = ext_last(prev) - split->iblock + 1; + split->blkno = prev->blkno ? + prev->blkno + prev->count - split->count : 0; + split->flags = prev->flags; + + modify_and_track_count(prev, ext->iblock - prev->iblock, + &on, &off); + + insert_extent(unpe, split, &on, &off); + next = split; + split = NULL; + } else { + next = NULL; + } + + /* trimming a prev extent? */ + if (prev && ext_last(prev) >= ext->iblock) { + modify_and_track_count(prev, ext->iblock - prev->iblock, + &on, &off); + } + + /* merging with a prev extent? */ + if (prev && extents_merge(prev, ext)) { + ext->iblock = prev->iblock; + ext->blkno = prev->blkno; + modify_and_track_count(ext, ext->count + prev->count, + &on, &off); + remove_extent(unpe, prev, &on, &off); + } + + /* if didn't split find next, removing any totally within ours */ + if (!next) { + while ((next = next_extent(ext)) && + ext_last(next) <= ext_last(ext)) { + remove_extent(unpe, next, &on, &off); + } + } + + /* trimming a next extent? */ + if (next && next->iblock <= ext_last(ext)) { + offset = (ext_last(ext) + 1) - next->iblock; + next->iblock += offset; + next->blkno = next->blkno ? next->blkno + offset : 0; + modify_and_track_count(next, next->count - offset, + &on, &off); + } + + /* merging with a next extent? */ + if (next && extents_merge(ext, next)) { + modify_and_track_count(ext, ext->count + next->count, + &on, &off); + remove_extent(unpe, next, &on, &off); + } + + /* and finally remove our extent if it was only removing others */ + if (ext->blkno == 0 && ext->flags == 0) + remove_extent(unpe, ext, &on, &off); + + if (inode) + scoutfs_inode_add_onoff(inode, on, off); + + kfree(split); + return 0; +} + +static bool block_bitmap_fits(u64 blkno, u64 count) +{ + return ((blkno & SCOUTFS_BLOCK_BITMAP_BIT_MASK) + count) <= + SCOUTFS_BLOCK_BITMAP_BITS; +} + +static void block_bitmap_bit(u64 *base, int *bit, u64 blkno, u8 type) +{ + if (type == SCOUTFS_BLOCK_BITMAP_BIG) + blkno >>= SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; + + *bit = blkno & SCOUTFS_BLOCK_BITMAP_BIT_MASK; + *base = blkno >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; +} + +static u64 block_bitmap_blkno(u64 base, int bit, u8 type) +{ + u64 blkno; + + blkno = (base << SCOUTFS_BLOCK_BITMAP_BASE_SHIFT) + bit; + + if (type == SCOUTFS_BLOCK_BITMAP_BIG) + blkno <<= SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; + + return blkno; +} + +struct block_bitmap { + u64 base; + u8 type; + bool exists; + unsigned long bits[DIV_ROUND_UP(SCOUTFS_BLOCK_BITMAP_BITS, + BITS_PER_LONG)]; +}; + +static inline __le64 long_bits_to_le64(unsigned long *bits, unsigned int i) +{ +#if BITS_PER_LONG == 64 + return cpu_to_le64(bits[i]); +#elif BITS_PER_LONG == 32 + i <<= 1; + return cpu_to_le64(bits[i] | ((u64)bits[i + 1] << 32)); +#else +#error "unexpected BITS_PER_LONG value?" +#endif +} + +static inline void u64_to_long_bits(unsigned long *bits, unsigned int i, u64 x) +{ +#if BITS_PER_LONG == 64 + bits[i] = x; +#else + i <<= 1; + bits[i] = x; + bits[i + 1] = x >> 32; +#endif +} + +/* + * Block bitmaps are unpacked into native long bitmaps in memory for use + * with the kernel's bitmap functions. This requires a bit of finesse + * to make sure that we translate the bits appropriately to + * architectures with different word size and endian. + */ +static int unpack_block_bitmap(struct block_bitmap *bb, + struct scoutfs_btree_item_ref *iref) +{ + struct scoutfs_block_bitmap_key *bbk; + struct scoutfs_packed_bitmap *pb; + unsigned int nr; + u64 present; + u64 set; + u64 b; + int ret; + int w; + int i; + + if (iref->key_len != sizeof(struct scoutfs_block_bitmap_key) || + iref->val_len < sizeof(struct scoutfs_packed_bitmap)) { + ret = -EIO; + goto out; + } + pb = iref->val; + + bbk = iref->key; + bb->type = bbk->type; + bb->base = be64_to_cpu(bbk->base); + + nr = hweight64(le64_to_cpu(pb->present)); + + if (iref->val_len != + offsetof(struct scoutfs_packed_bitmap, words[nr])) { + ret = -EIO; + goto out; + } + + present = le64_to_cpu(pb->present); + set = le64_to_cpu(pb->set); + w = 0; + for (i = 0, b = 1; + (present | set) && i < SCOUTFS_PACKED_BITMAP_WORDS; + i++, b <<= 1) { + if (set & b) + u64_to_long_bits(bb->bits, i, ~0ULL); + else if (present & b) + u64_to_long_bits(bb->bits, i, + le64_to_cpu(pb->words[w++])); + } + ret = 0; + +out: + return ret; +} + +static int load_block_bitmap(struct super_block *sb, + struct scoutfs_btree_root *root, + u64 blkno, u8 type, bool next, bool zero_enoent, + struct block_bitmap **bb_ret) +{ + struct scoutfs_block_bitmap_key bbk; + struct block_bitmap *bb = NULL; + SCOUTFS_BTREE_ITEM_REF(iref); + u64 base; + int bit; + int ret; + + bb = kzalloc(sizeof(struct block_bitmap), GFP_NOFS); + if (!bb) { + ret = -ENOMEM; + goto out; + } + + block_bitmap_bit(&base, &bit, blkno, type); + + bbk.type = type; + bbk.base = cpu_to_be64(base); + + if (next) + ret = scoutfs_btree_next(sb, root, &bbk, sizeof(bbk), &iref); + else + ret = scoutfs_btree_lookup(sb, root, &bbk, sizeof(bbk), &iref); + if (ret == 0) { + ret = unpack_block_bitmap(bb, &iref); + bb->exists = true; + scoutfs_btree_put_iref(&iref); + } + if (ret == -ENOENT && zero_enoent) { + bb->base = base; + bb->type = type; + ret = 0; + } + +out: + if (ret < 0) { + kfree(bb); + *bb_ret = NULL; + } else { + *bb_ret = bb; + } + return ret; +} + +/* + * Block bitmaps start with two flag words that indicate if logical + * words are all 0s, all 1s, or a mix of set and clear bits stored in + * the item payload. Typically the allocators will have long runs of + * set or clear bits so we don't store most of the bitmaps. Badly + * fragmented allocators will be (2/64 = ~3%) larger. + */ +static int pack_block_bitmap(struct scoutfs_packed_bitmap *pb, + struct block_bitmap *bb) +{ + __le64 word; + u64 present = 0; + u64 set = 0; + u64 b; + int w; + int i; + + w = 0; + for (i = 0, b = 1; i < SCOUTFS_PACKED_BITMAP_WORDS; i++, b <<= 1) { + word = long_bits_to_le64(bb->bits, i); + + if (word == cpu_to_le64(~0ULL)) { + set |= b; + } else if (word != 0) { + present |= b; + pb->words[w++] = word; + } + } + + pb->set = cpu_to_le64(set); + pb->present = cpu_to_le64(present); + + return offsetof(struct scoutfs_packed_bitmap, words[w]); +} + +static int store_block_bitmap(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct block_bitmap *bb) +{ + struct scoutfs_block_bitmap_key bbk; + struct scoutfs_packed_bitmap *pb; + int size; + int ret; + + bbk.type = bb->type; + bbk.base = cpu_to_be64(bb->base); + + if (bitmap_empty(bb->bits, SCOUTFS_BLOCK_BITMAP_BITS)) { + if (!bb->exists) { + ret = 0; + goto out; + } + + ret = scoutfs_btree_delete(sb, alloc, wri, root, + &bbk, sizeof(bbk)); + + } else { + pb = kmalloc(SCOUTFS_PACKED_BITMAP_MAX_BYTES, GFP_NOFS); + if (!pb) { + ret = -ENOMEM; + goto out; + } + + size = pack_block_bitmap(pb, bb); + + ret = scoutfs_btree_force(sb, alloc, wri, root, + &bbk, sizeof(bbk), pb, size); + kfree(pb); + if (ret == 0) + bb->exists = true; + } +out: + return ret; +} + +/* + * Set a region of bitmaps which must fit in one item. The caller's + * blkno is translated to an item base and then the number of bits are + * set. The caller is specifying a number of bits to set, not a block + * extent. + */ +static int set_block_bits(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u8 type, u64 blkno, + int nbits) +{ + struct block_bitmap *bb = NULL; + u64 base; + int bit; + int ret; + + if (WARN_ON_ONCE(!block_bitmap_fits(blkno, nbits))) + return -EINVAL; + + ret = load_block_bitmap(sb, root, blkno, type, false, true, &bb); + if (ret < 0) + goto out; + + block_bitmap_bit(&base, &bit, blkno, type); + + bitmap_set(bb->bits, bit, nbits); + + /* if a little bitmap is full, set it's big and delete it */ + if (type == SCOUTFS_BLOCK_BITMAP_LITTLE && + bitmap_full(bb->bits, SCOUTFS_PACKED_BITMAP_BITS)) { + ret = set_block_bits(sb, alloc, wri, root, + SCOUTFS_BLOCK_BITMAP_BIG, blkno, 1); + if (ret < 0) + goto out; + + bitmap_zero(bb->bits, SCOUTFS_PACKED_BITMAP_BITS); + } + + ret = store_block_bitmap(sb, alloc, wri, root, bb); + BUG_ON(ret < 0); /* cleared bit out of sync with existing littles */ + +out: + kfree(bb); + return ret; +} + +/* + * Find a region of free blocks for the caller. The caller can ask for + * an arbitrarily large extent but we'll only return at most a bitmap's + * worth of blocks from one allocation. + * + * Big bitmap items are stored before little items. This let's large + * allocations naturally fall back to being satisfied by little items + * when there are no more remaining big items. Small allocations first + * look for little items and then search again for big items that they + * can break up. + * + * We always simply look for the first free region. This is operating + * in the client on trees whose items are populated by the server + * between each transaction. The server is responsible for distributing + * the items such that the client tends to allocate across the device + * over time. + */ +static int alloc_blocks(struct super_block *sb, u64 count, u64 *blkno_ret, + u64 *count_ret) +{ + DECLARE_DATA_INFO(sb, datinf); + struct scoutfs_balloc_root *broot = &datinf->data_alloc; + struct block_bitmap *bb = NULL; + u64 blkno; + u8 type; + int bit; + int end; + int ret; + + if (WARN_ON_ONCE(count == 0)) + return -EINVAL; + + /* will only allocate from one block bitmap item at a time */ + count = min_t(u64, count, SCOUTFS_BLOCK_BITMAP_BITS); + + /* small allocations first look for little items, then check big */ + if (count < SCOUTFS_BLOCK_BITMAP_BITS) + type = SCOUTFS_BLOCK_BITMAP_LITTLE; + else + type = SCOUTFS_BLOCK_BITMAP_BIG; + + do { + ret = load_block_bitmap(sb, &broot->root, 0, type, + true, false, &bb); + } while ((ret == -ENOENT && type == SCOUTFS_BLOCK_BITMAP_LITTLE) && + (type = SCOUTFS_BLOCK_BITMAP_BIG, 1)); + if (ret < 0) { + if (ret == -ENOENT) + ret = -ENOSPC; + goto out; + } + + bit = find_first_bit(bb->bits, SCOUTFS_BLOCK_BITMAP_BITS); + if (WARN_ON_ONCE(bit >= SCOUTFS_BLOCK_BITMAP_BITS)) { + ret = -EIO; /* stored items should have bits set */ + goto out; + } + + blkno = block_bitmap_blkno(bb->base, bit, bb->type); + + if (bb->type == SCOUTFS_BLOCK_BITMAP_BIG) { + /* set remaining little bits if using big for partial small */ + if (count != SCOUTFS_BLOCK_BITMAP_BITS) { + ret = set_block_bits(sb, datinf->alloc, datinf->wri, + &broot->root, + SCOUTFS_BLOCK_BITMAP_LITTLE, + blkno + count, + SCOUTFS_BLOCK_BITMAP_BITS - count); + if (ret < 0) + goto out; + } + + clear_bit(bit, bb->bits); + + } else { + end = find_next_zero_bit(bb->bits, SCOUTFS_BLOCK_BITMAP_BITS, + bit + 1); + end = min(end, SCOUTFS_BLOCK_BITMAP_BITS); /* catch > size */ + count = min_t(u64, count, end - bit); + + bitmap_clear(bb->bits, bit, count); + } + + ret = store_block_bitmap(sb, datinf->alloc, datinf->wri, + &broot->root, bb); + BUG_ON(ret < 0); /* little partial out of sync with big */ + + le64_add_cpu(&broot->total_free, -count); + + *blkno_ret = blkno; + *count_ret = count; +out: + kfree(bb); + return ret; +} + +/* + * Set free block bits in the block bitmaps and update the root's + * total_free count. The caller can specifiy the root so that this can + * be used both to free used allocations as well as to return unused + * allocations in error paths. The caller must ensure that the block + * regions fit in a single block bitmap (by for the blocks in an + * extent). + */ +static int free_blocks(struct super_block *sb, + struct scoutfs_balloc_root *broot, u64 blkno, u64 count) +{ + DECLARE_DATA_INFO(sb, datinf); + int ret; + + if (count == SCOUTFS_BLOCK_BITMAP_BITS) + ret = set_block_bits(sb, datinf->alloc, datinf->wri, + &broot->root, SCOUTFS_BLOCK_BITMAP_BIG, + blkno, 1); + else + ret = set_block_bits(sb, datinf->alloc, datinf->wri, + &broot->root, SCOUTFS_BLOCK_BITMAP_LITTLE, + blkno, count); + + if (ret == 0) + le64_add_cpu(&broot->total_free, count); + + return ret; +} + +/* + * Ensure that the destination free block bitmap tree has the minimum + * total free blocks by moving bits from the source tree. It will first + * try to find big bits starting at the cursor but will fall back to + * little bits after having wrapped the cursor. + * + * This will move all the items from the source to the destination if + * that's what it takes to reach the minimum. + * + * This is called by the server which provides its writer and metadata + * allocation contexts. It has locked the two allocation trees that + * will be modified. + */ +int scoutfs_data_move_alloc_bits(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_balloc_root *dst, + struct scoutfs_balloc_root *src, + __le64 *cursor, u64 min_dst_total) + +{ + struct block_bitmap *sbb = NULL; + struct block_bitmap *dbb = NULL; + u64 needed; + u64 blocks; + u64 moved; + u64 blkno; + u64 base; + u64 curs; + u8 type; + int nbits; + int bit; + int end; + int ret = 0; + + /* start moving big bitmap items */ + type = SCOUTFS_BLOCK_BITMAP_BIG; + curs = le64_to_cpup(cursor); + + while (le64_to_cpu(dst->total_free) < min_dst_total) { + + /* find the next source bitmap item with bits to move */ + kfree(sbb); + ret = load_block_bitmap(sb, &src->root, curs, type, + true, false, &sbb); + if (ret == 0 && sbb->type != type) + ret = -ENOENT; + if (ret < 0) { + if (ret == -ENOENT) { + if (curs > 0) { + curs = 0; + continue; + } + if (type == SCOUTFS_BLOCK_BITMAP_BIG) { + type = SCOUTFS_BLOCK_BITMAP_LITTLE; + curs = le64_to_cpup(cursor); + continue; + } + ret = -ENOSPC; + } + break; + } + + /* load the destination bitmap */ + blkno = block_bitmap_blkno(sbb->base, 0, type); + kfree(dbb); + ret = load_block_bitmap(sb, &dst->root, blkno, type, + false, true, &dbb); + if (ret < 0) + break; + + /* figure out how many bits to move, can overshoot */ + needed = min_dst_total - le64_to_cpu(dst->total_free); + if (type == SCOUTFS_BLOCK_BITMAP_BIG) { + needed = (needed + SCOUTFS_BLOCK_BITMAP_BITS - 1) + >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; + } + + /* start searching from the cursor if within item */ + if (curs > blkno) + blkno = curs; + block_bitmap_bit(&base, &bit, blkno, type); + + moved = 0; + while (moved < needed) { + bit = find_next_bit(sbb->bits, + SCOUTFS_BLOCK_BITMAP_BITS, bit); + if (bit >= SCOUTFS_BLOCK_BITMAP_BITS) + break; + + end = find_next_zero_bit(sbb->bits, + SCOUTFS_BLOCK_BITMAP_BITS, + bit + 1); + end = min(end, SCOUTFS_BLOCK_BITMAP_BITS); + nbits = min_t(u64, needed - moved, end - bit); + + bitmap_clear(sbb->bits, bit, nbits); + bitmap_set(dbb->bits, bit, nbits); + + curs = block_bitmap_blkno(dbb->base, bit + nbits, type); + moved += nbits; + } + + ret = store_block_bitmap(sb, alloc, wri, &dst->root, dbb); + if (ret < 0) + break; + + ret = store_block_bitmap(sb, alloc, wri, &src->root, sbb); + BUG_ON(ret); /* inconsistent src/dst, save orig src */ + + blocks = moved; + if (sbb->type == SCOUTFS_BLOCK_BITMAP_BIG) + blocks <<= SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; + + le64_add_cpu(&dst->total_free, blocks); + le64_add_cpu(&src->total_free, -blocks); + + *cursor = cpu_to_le64(curs); + } + + kfree(sbb); + kfree(dbb); + + return ret; +} + +/* + * The server caller is making their way through free data blocks + * initializing free block bitmap bits for the first time. This is the + * only mechanism that initializes free block bitmap items so we know + * that we never have to merge with existing items as long as we always + * write a full item. + * + * The caller gives us the fully extent of blknos that we could + * initialize and we figure out the size of the largest item and its + * bits which cover the start of the extent. We can set big bits if the + * extent is aligned to a small bitmap and is large enough. + */ +int scoutfs_data_add_free_blocks(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_balloc_root *broot, + u64 blkno, u64 count) + +{ + u64 base; + u8 type; + int nbits; + int bit; + int ret; + + type = SCOUTFS_BLOCK_BITMAP_LITTLE; + block_bitmap_bit(&base, &bit, blkno, type); + + if (bit == 0 && count >= SCOUTFS_BLOCK_BITMAP_BITS) { + type = SCOUTFS_BLOCK_BITMAP_BIG; + block_bitmap_bit(&base, &bit, blkno, type); + nbits = min_t(u64, count >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT, + SCOUTFS_BLOCK_BITMAP_BITS - bit); + count = (u64)nbits << SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; + } else { + nbits = min_t(u64, count, SCOUTFS_BLOCK_BITMAP_BITS - bit); + count = nbits; + } + + ret = set_block_bits(sb, alloc, wri, &broot->root, type, blkno, nbits); + if (ret == 0) { + le64_add_cpu(&broot->total_free, count); + ret = count; } return ret; } /* - * Find and remove or mark offline the next extent that intersects with - * the caller's range. The caller is responsible for transactions and - * locks. + * Find and remove or mark offline the block mappings that intersect + * with the caller's range. The caller is responsible for transactions + * and locks. * * Returns: * - -errno on errors * - 0 if there are no more extents to stop iteration * - +iblock of next logical block to truncate the next block from - * - * Since our extents are block granular we can never have > S64_MAX - * iblock values. Returns -ENOENT if no extent was found and -errno on - * errors. */ -static s64 truncate_one_extent(struct super_block *sb, struct inode *inode, - u64 ino, u64 iblock, u64 last, bool offline, - struct scoutfs_lock *lock) +static s64 truncate_extents(struct super_block *sb, struct inode *inode, + u64 ino, u64 iblock, u64 last, bool offline, + struct scoutfs_lock *lock) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_DATA_INFO(sb, datinf); - struct scoutfs_extent next; - struct scoutfs_extent rem; - struct scoutfs_extent fr; - struct scoutfs_extent ofl; - bool rem_fr = false; - bool add_rem = false; - s64 offline_delta = 0; - s64 online_delta = 0; + struct unpacked_extents *unpe = NULL; + struct unpacked_extent *ext; + struct scoutfs_traced_extent te; + u64 offset; + u64 blkno; + u64 count; + u8 flags; s64 ret; + int err; - scoutfs_extent_init(&next, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &next, lock); + ret = load_unpacked_extents(sb, ino, iblock, last, false, &unpe, lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; goto out; } - trace_scoutfs_data_truncate_next(sb, &next); + flags = offline ? SEF_OFFLINE : 0; - scoutfs_extent_init(&rem, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock, last - iblock + 1, 0, 0); - if (!scoutfs_extent_intersection(&rem, &next)) { - ret = 0; - goto out; + ret = 0; + ext = find_extent(unpe, iblock, last); + while (ext && ext->iblock <= last) { + + /* nothing to do when already offline and unmapped */ + if ((offline && (ext->flags & SEF_OFFLINE)) && !ext->blkno) { + ext = next_extent(ext); + continue; + } + + iblock = max(ext->iblock, iblock); + offset = iblock - ext->iblock; + blkno = ext->blkno + offset; + count = min(ext->count - offset, last - iblock + 1); + + if (ext->blkno) { + down_write(&datinf->alloc_rwsem); + err = free_blocks(sb, &datinf->data_free, blkno, count); + up_write(&datinf->alloc_rwsem); + if (err < 0) { + ret = err; + break; + } + } + + init_traced_extent(&te, iblock, count, 0, flags); + trace_scoutfs_data_extent_truncated(sb, ino, &te); + + err = set_extent(sb, inode, ino, unpe, iblock, 0, count, flags); + BUG_ON(err); /* inconsistent alloc and extents */ + + /* modifying could have merged and deleted ext, search again */ + iblock += count; + if (iblock > last) + break; + ext = find_extent(unpe, iblock, last); } - trace_scoutfs_data_truncate_remove(sb, &rem); + err = store_packed_extents(sb, ino, unpe, lock); + BUG_ON(err); /* inconsistent alloc and extents */ - /* nothing to do if the extent's already offline and unallocated */ - if ((offline && (rem.flags & SEF_OFFLINE)) && !rem.map) { - ret = 1; - goto out; - } - - /* free an allocated mapping */ - if (rem.map) { - scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->rid, rem.map, rem.len, 0, 0); - ret = scoutfs_extent_add(sb, data_extent_io, &fr, - sbi->rid_lock); - if (ret) - goto out; - rem_fr = true; - } - - /* remove the extent */ - ret = scoutfs_extent_remove(sb, data_extent_io, &rem, lock); - if (ret) - goto out; - add_rem = true; - - /* add an offline extent */ - if (offline) { - scoutfs_extent_init(&ofl, SCOUTFS_FILE_EXTENT_TYPE, rem.owner, - rem.start, rem.len, 0, SEF_OFFLINE); - trace_scoutfs_data_truncate_offline(sb, &ofl); - ret = scoutfs_extent_add(sb, data_extent_io, &ofl, lock); - if (ret) - goto out; - } - - if (rem.map && !(rem.flags & SEF_UNWRITTEN)) - online_delta += -rem.len; - if (!offline && (rem.flags & SEF_OFFLINE)) - offline_delta += -rem.len; - if (offline && !(rem.flags & SEF_OFFLINE)) - offline_delta += ofl.len; - - scoutfs_inode_add_onoff(inode, online_delta, offline_delta); - - /* start returning free extents to the server after a small delay */ - if (rem.map && (atomic64_read(&datinf->node_free_blocks) > - NODE_FREE_HIGH_WATER_BLOCKS)) - queue_work(datinf->workq, &datinf->return_work); - - ret = 1; + /* continue after the packed extent item if we exhausted extents */ + if (ret == 0) + ret = unpe->iblock + SCOUTFS_PACKEXT_BLOCKS; out: - scoutfs_extent_cleanup(ret < 0 && add_rem, scoutfs_extent_add, sb, - data_extent_io, &rem, lock, - SC_DATA_EXTENT_TRUNC_CLEANUP, - corrupt_data_extent_trunc_cleanup, &rem); - scoutfs_extent_cleanup(ret < 0 && rem_fr, scoutfs_extent_remove, sb, - data_extent_io, &fr, sbi->rid_lock, - SC_DATA_EXTENT_TRUNC_CLEANUP, - corrupt_data_extent_trunc_cleanup, &rem); - - if (ret > 0) - ret = rem.start + rem.len; - + free_unpacked_extents(unpe); return ret; } @@ -382,7 +1388,7 @@ out: * and offline blocks. If it's not provided then the inode is being * destroyed and isn't reachable, we don't need to update it. * - * The caller is in charge of locking the inode and extents, but we may + * The caller is in charge of locking the inode and data, but we may * have to modify far more items than fit in a transaction so we're in * charge of batching updates into transactions. If the inode is * provided then we're responsible for updating its item as we go. @@ -392,7 +1398,6 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, struct scoutfs_lock *lock) { struct scoutfs_item_count cnt = SIC_TRUNC_EXTENT(inode); - DECLARE_DATA_INFO(sb, datinf); LIST_HEAD(ind_locks); s64 ret = 0; @@ -421,11 +1426,9 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, else ret = 0; - down_write(&datinf->alloc_rwsem); if (ret == 0) - ret = truncate_one_extent(sb, inode, ino, iblock, last, - offline, lock); - up_write(&datinf->alloc_rwsem); + ret = truncate_extents(sb, inode, ino, iblock, last, + offline, lock); if (inode) scoutfs_update_inode_item(inode, lock, &ind_locks); @@ -443,94 +1446,13 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, return ret; } -static int get_server_extent(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_extent ext; - u64 start; - u64 len; - int ret; - - ret = scoutfs_client_alloc_extent(sb, SERVER_ALLOC_BLOCKS, - &start, &len); - if (ret) - goto out; - - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->rid, start, len, 0, 0); - trace_scoutfs_data_get_server_extent(sb, &ext); - ret = scoutfs_extent_add(sb, data_extent_io, &ext, sbi->rid_lock); - /* XXX don't free extent on error, crash recovery with server */ - -out: - return ret; -} - /* - * Find a free extent to satisfy an allocation of at most @len blocks. - * - * Returns 0 and fills the caller's extent with a _BLKNO_TYPE extent if - * we found a match. It's len may be less than desired. No stored - * extents have been modified. - * - * Returns -errno on error and -ENOSPC if no free extents were found. - * - * The caller's extent is always clobbered. - */ -static int find_free_extent(struct super_block *sb, u64 len, - struct scoutfs_extent *ext) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - int ret; - - len = min(len, MAX_EXTENT_BLOCKS); - - for (;;) { - /* first try to find the first sufficient extent */ - scoutfs_extent_init(ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, - sbi->rid, 0, len, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, ext, - sbi->rid_lock); - - /* if none big enough, look for last largest smaller */ - if (ret == -ENOENT && len > 1) - ret = scoutfs_extent_prev(sb, data_extent_io, ext, - sbi->rid_lock); - - /* ask the server for more if we think it'll help */ - if (ret == -ENOENT || ext->len < len) { - ret = get_server_extent(sb); - if (ret == 0) - continue; - } - - /* use the extent we found or return errors */ - break; - } - - if (ret == 0) - scoutfs_extent_init(ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->rid, ext->start, - min(ext->len, len), 0, 0); - - trace_scoutfs_data_find_free_extent(sb, ext); - return ret; -} - -/* - * The caller is writing to a logical block that doesn't have an + * The caller is writing to a logical iblock that doesn't have an * allocated extent. * - * We always allocate an extent starting at the logical block. The - * caller has considered overlapping and following extents and has given - * us a maximum length that we could safely allocate. Preallocation - * heuristics decide to use this length or only a single block. - * - * If the caller passes in an existing extent then we remove the - * allocated region from the existing extent. We then add a single - * block extent for the caller to write into. Then if we allocated - * multiple blocks we add an unwritten extent for the rest of the blocks - * in the extent. + * We always allocate an extent starting at the logical iblock. The + * caller has searched for an extent containing iblock. If it already + * existed then it must be unallocated and offline. * * Preallocation is used if we're strictly contiguously extending * writes. That is, if the logical block offset equals the number of @@ -542,111 +1464,99 @@ static int find_free_extent(struct super_block *sb, u64 len, * staging, sparse files, multi-node writes, etc. fallocate() is always * a better tool to use. * - * On success we update the caller's extent to the single block - * allocated extent for the logical block for use in block mapping. + * We can mangle the extents so the caller is going to search for the + * intersecting extent again if we succeed. */ static int alloc_block(struct super_block *sb, struct inode *inode, - struct scoutfs_extent *ext, u64 iblock, u64 len, + struct unpacked_extents *unpe, + struct unpacked_extent *ext, u64 iblock, struct scoutfs_lock *lock) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_DATA_INFO(sb, datinf); const u64 ino = scoutfs_ino(inode); - struct scoutfs_extent unwr; - struct scoutfs_extent old; - struct scoutfs_extent blk; - struct scoutfs_extent fr; - bool add_old = false; - bool add_fr = false; - bool rem_blk = false; - u64 offline; + struct scoutfs_traced_extent te; + u64 blkno = 0; + u64 count; u64 online; + u64 offline; + u64 last; + u8 flags; int ret; + int err; + + /* can only allocate over existing unallocated offline extent */ + if (WARN_ON_ONCE(ext && + !(iblock >= ext->iblock && iblock <= ext_last(ext) && + ext->blkno == 0 && (ext->flags & SEF_OFFLINE)))) + return -EINVAL; down_write(&datinf->alloc_rwsem); scoutfs_inode_get_onoff(inode, &online, &offline); - /* strictly contiguous extending writes will try to preallocate */ + if (ext) { + /* limit preallocation to remaining existing (offline) extent */ + count = ext->count - (iblock - ext->iblock); + flags = ext->flags; + } else { + /* otherwise alloc to next extent or end of packed item */ + last = last_iblock(iblock); + ext = find_extent(unpe, iblock, last); + if (ext) + count = ext->iblock - iblock; + else + count = last - iblock + 1; + flags = 0; + } + + /* only strictly contiguous extending writes will try to preallocate */ if (iblock > 1 && iblock == online) - len = min3(len, iblock, MAX_EXTENT_BLOCKS); + count = min(iblock, count); else - len = 1; + count = 1; - trace_scoutfs_data_alloc_block(sb, inode, ext, iblock, len, - online, offline); - - ret = find_free_extent(sb, len, &fr); + ret = alloc_blocks(sb, count, &blkno, &count); if (ret < 0) goto out; - trace_scoutfs_data_alloc_block_next(sb, &fr); - - /* initialize the new mapped block extent, referenced by cleanup */ - scoutfs_extent_init(&blk, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock, 1, fr.start, 0); - - /* remove the free extent that we're allocating */ - ret = scoutfs_extent_remove(sb, data_extent_io, &fr, sbi->rid_lock); - if (ret) + ret = set_extent(sb, inode, ino, unpe, iblock, blkno, 1, 0); + if (ret < 0) goto out; - add_fr = true; - /* remove an existing offline or unwritten block extent */ - if (ext->flags) { - scoutfs_extent_init(&old, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock, len, 0, ext->flags); - ret = scoutfs_extent_remove(sb, data_extent_io, &old, lock); - if (ret) - goto out; - add_old = true; + init_traced_extent(&te, iblock, blkno, 1, 0); + trace_scoutfs_data_alloc_block(sb, ino, &te); + + if (count > 1) { + ret = set_extent(sb, inode, ino, unpe, iblock + 1, + blkno + 1, count - 1, flags | SEF_UNWRITTEN); + if (ret < 0) { + err = set_extent(sb, inode, ino, unpe, iblock, 0, 1, + flags); + BUG_ON(err); /* couldn't restore original */ + } + + init_traced_extent(&te, iblock + 1, blkno + 1, count - 1, + flags | SEF_UNWRITTEN); + trace_scoutfs_data_prealloc_unwritten(sb, ino, &te); } - /* add the block that the caller is writing */ - ret = scoutfs_extent_add(sb, data_extent_io, &blk, lock); - if (ret) - goto out; - rem_blk = true; + ret = store_packed_extents(sb, ino, unpe, lock); + BUG_ON(ret); /* inconsistent previous extent state */ - /* and maybe add the remaining unwritten extent */ - if (len > 1) { - scoutfs_extent_init(&unwr, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock + 1, len - 1, fr.start + 1, - ext->flags | SEF_UNWRITTEN); - ret = scoutfs_extent_add(sb, data_extent_io, &unwr, lock); - if (ret) - goto out; - } - - scoutfs_inode_add_onoff(inode, 1, - (ext->flags & SEF_OFFLINE) ? -1ULL : 0); - ret = 0; out: - scoutfs_extent_cleanup(ret < 0 && rem_blk, scoutfs_extent_remove, sb, - data_extent_io, &blk, lock, - SC_DATA_EXTENT_ALLOC_CLEANUP, - corrupt_data_extent_alloc_cleanup, &blk); - scoutfs_extent_cleanup(ret < 0 && add_old, scoutfs_extent_add, sb, - data_extent_io, &old, lock, - SC_DATA_EXTENT_ALLOC_CLEANUP, - corrupt_data_extent_alloc_cleanup, &blk); - scoutfs_extent_cleanup(ret < 0 && add_fr, scoutfs_extent_add, sb, - data_extent_io, &fr, sbi->rid_lock, - SC_DATA_EXTENT_ALLOC_CLEANUP, - corrupt_data_extent_alloc_cleanup, &blk); + if (ret < 0 && blkno > 0) { + err = free_blocks(sb, &datinf->data_alloc, blkno, count); + BUG_ON(err); /* leaked free blocks */ + } up_write(&datinf->alloc_rwsem); - trace_scoutfs_data_alloc_block_ret(sb, ext, ret); - if (ret == 0) - *ext = blk; return ret; } /* - * A caller is writing into unwritten allocated space. This can also be - * called for staging writes so we clear both the unwritten and offline - * flags. We record the extent as online as allocating writes would. + * A caller is writing into an unwritten block. This can also be called + * for staging writes so we clear both the unwritten and offline flags. * * We don't have to wait for dirty block IO to complete before clearing * the unwritten flag in metadata because we have strict synchronization @@ -655,36 +1565,30 @@ out: * is committed. */ static int convert_unwritten(struct super_block *sb, struct inode *inode, - struct scoutfs_extent *ext, u64 start, u64 len, + struct unpacked_extents *unpe, + struct unpacked_extent *ext, u64 iblock, struct scoutfs_lock *lock) { - struct scoutfs_extent conv; + u64 blkno; + u8 ext_fl; int err; int ret; - if (WARN_ON_ONCE(!ext->map) || - WARN_ON_ONCE(!(ext->flags & SEF_UNWRITTEN))) - return -EINVAL; + blkno = ext->blkno + (iblock - ext->iblock); + ext_fl = ext->flags; - scoutfs_extent_init(&conv, ext->type, ext->owner, start, len, - ext->map + (start - ext->start), ext->flags); - ret = scoutfs_extent_remove(sb, data_extent_io, &conv, lock); - if (ret) + ret = set_extent(sb, inode, scoutfs_ino(inode), unpe, iblock, + blkno, 1, ext_fl & ~(SEF_OFFLINE|SEF_UNWRITTEN)); + if (ret < 0) goto out; - conv.flags &= ~(SEF_UNWRITTEN | SEF_OFFLINE); - ret = scoutfs_extent_add(sb, data_extent_io, &conv, lock); - if (ret) { - conv.flags = ext->flags; - err = scoutfs_extent_add(sb, data_extent_io, &conv, lock); - BUG_ON(err); - goto out; + ret = store_packed_extents(sb, scoutfs_ino(inode), unpe, lock); + if (ret < 0) { + err = set_extent(sb, inode, scoutfs_ino(inode), unpe, iblock, + blkno, 1, ext_fl); + BUG_ON(err); /* packed and unpacked inconsistent */ } - scoutfs_inode_add_onoff(inode, len, - (ext->flags & SEF_OFFLINE) ? -len : 0); - *ext = conv; - ret = 0; out: return ret; } @@ -693,12 +1597,13 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, struct buffer_head *bh, int create) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); + const u64 ino = scoutfs_ino(inode); struct super_block *sb = inode->i_sb; struct scoutfs_lock *lock = NULL; - struct scoutfs_extent ext; - u64 next_iblock = 0; + struct unpacked_extents *unpe = NULL; + struct unpacked_extent *ext = NULL; + DECLARE_TRACED_EXTENT(te); u64 offset; - u64 len; int ret; WARN_ON_ONCE(create && !mutex_is_locked(&inode->i_mutex)); @@ -711,69 +1616,53 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, goto out; } - /* look for the extent that overlaps our iblock */ - scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, - scoutfs_ino(inode), iblock, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, lock); - if (ret && ret != -ENOENT) + ret = load_unpacked_extents(sb, ino, iblock, iblock, true, &unpe, lock); + if (ret < 0) goto out; - if (ret == 0) { - trace_scoutfs_data_get_block_next(sb, &ext); - /* remember start of next to limit preallocation */ - if (ext.start > iblock) - next_iblock = ext.start; - } - - /* didn't find an extent or it's past our iblock */ - if (ret == -ENOENT || ext.start > iblock) - memset(&ext, 0, sizeof(ext)); - - if (ext.len) - trace_scoutfs_data_get_block_intersection(sb, &ext); + ext = find_extent(unpe, iblock, iblock); /* non-staging callers should have waited on offline blocks */ - if (WARN_ON_ONCE((ext.flags & SEF_OFFLINE) && !si->staging)) { + if (WARN_ON_ONCE(ext && (ext->flags & SEF_OFFLINE) && !si->staging)) { ret = -EIO; goto out; } /* convert unwritten to written */ - if (create && (ext.flags & SEF_UNWRITTEN)) { - ret = convert_unwritten(sb, inode, &ext, iblock, 1, lock); - if (ret == 0) + if (create && ext && (ext->flags & SEF_UNWRITTEN)) { + ret = convert_unwritten(sb, inode, unpe, ext, iblock, lock); + if (ret == 0) { set_buffer_new(bh); + ext = find_extent(unpe, iblock, iblock); + } goto out; } - /* allocate an extent from our logical block */ - if (create && !ext.map) { - /* limit possible alloc to this extent, next, or logical max */ - if (ext.len > 0) - len = ext.len - (iblock - ext.start); - else if (next_iblock > iblock) - len = ext.start - iblock; - else - len = SCOUTFS_BLOCK_MAX - iblock; - - ret = alloc_block(sb, inode, &ext, iblock, len, lock); - if (ret == 0) + /* allocate and map blocks containing our logical block */ + if (create && (!ext || !ext->blkno)) { + ret = alloc_block(sb, inode, unpe, ext, iblock, lock); + if (ret == 0) { set_buffer_new(bh); + ext = find_extent(unpe, iblock, iblock); + } } else { ret = 0; } - out: /* map usable extent, else leave bh unmapped for sparse reads */ - if (ret == 0 && ext.map && !(ext.flags & SEF_UNWRITTEN)) { - offset = iblock - ext.start; - map_bh(bh, inode->i_sb, ext.map + offset); + if (ret == 0 && ext && ext->blkno && !(ext->flags & SEF_UNWRITTEN)) { + offset = iblock - ext->iblock; + map_bh(bh, inode->i_sb, ext->blkno + offset); bh->b_size = min_t(u64, bh->b_size, - (ext.len - offset) << SCOUTFS_BLOCK_SHIFT); + (ext->count - offset) << SCOUTFS_BLOCK_SHIFT); } + if (ext) + copy_traced_extent(&te, ext); + trace_scoutfs_get_block(sb, scoutfs_ino(inode), iblock, create, - ret, bh->b_blocknr, bh->b_size); + &te, ret, bh->b_blocknr, bh->b_size); + free_unpacked_extents(unpe); return ret; } @@ -1035,67 +1924,95 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, } /* - * Allocate one extent on behalf of fallocate. The caller has given us - * the largest extent we can add, its flags, and the flags of an - * existing overlapping extent to remove. + * Try to allocate unwritten extents for any unallocated regions of the + * logical block extent from the caller. We work one packed extent item + * at a time. * - * We allocate the largest extent that we can and return its length or - * -errno. + * We return an error or the numbet of contiguous blocks starting at + * iblock that were successfully processed. */ -static s64 fallocate_one_extent(struct super_block *sb, u64 ino, u64 start, - u64 len, u8 flags, u8 rem_flags, - struct scoutfs_lock *lock) +static int fallocate_extents(struct super_block *sb, struct inode *inode, + u64 iblock, u64 last, struct scoutfs_lock *lock) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_extent fal; - struct scoutfs_extent rem; - struct scoutfs_extent fr; - bool add_rem = false; - bool add_fr = false; - s64 ret; + DECLARE_DATA_INFO(sb, datinf); + const u64 ino = scoutfs_ino(inode); + struct unpacked_extents *unpe = NULL; + struct unpacked_extent *ext; + u8 ext_fl; + u64 blkno; + u64 count; + int done; + int ret; + int err; - if (WARN_ON_ONCE(len == 0) || - WARN_ON_ONCE(start + len < start)) { - ret = -EINVAL; - goto out; - } + /* work with the extents in one item at a time */ + last = min(last, last_iblock(iblock)); + done = 0; - ret = find_free_extent(sb, len, &fr); + ret = load_unpacked_extents(sb, ino, iblock, iblock, true, &unpe, lock); if (ret < 0) goto out; - ret = scoutfs_extent_init(&fal, SCOUTFS_FILE_EXTENT_TYPE, ino, - start, fr.len, fr.start, flags); - if (WARN_ON_ONCE(ret)) - goto out; + ext = find_extent(unpe, iblock, last); + while (iblock <= last) { - ret = scoutfs_extent_remove(sb, data_extent_io, &fr, sbi->rid_lock); - if (ret) - goto out; - add_fr = true; + /* default to allocate to end of region */ + count = last - iblock + 1; + ext_fl = 0; - /* remove a region of the existing extent */ - if (rem_flags) { - scoutfs_extent_init(&rem, SCOUTFS_FILE_EXTENT_TYPE, ino, - fal.start, fal.len, 0, rem_flags); - ret = scoutfs_extent_remove(sb, data_extent_io, &rem, lock); - if (ret) - goto out; - add_rem = true; + if (!ext) { + /* no extent, default alloc from above */ + + } else if (ext->iblock <= iblock && ext->blkno) { + /* skip portion of allocated extent */ + count = min(count, ext->count - (iblock - ext->iblock)); + iblock += count; + done += count; + ext = next_extent(ext); + continue; + + } else if (ext->iblock <= iblock && !ext->blkno) { + /* alloc portion of unallocated extent */ + count = min(count, ext->count - (iblock - ext->iblock)); + ext_fl = ext->flags; + + } else if (iblock < ext->iblock) { + /* alloc hole until next extent */ + count = min(count, ext->iblock - iblock); + } + + down_write(&datinf->alloc_rwsem); + + ret = alloc_blocks(sb, count, &blkno, &count); + if (ret == 0) { + ret = set_extent(sb, inode, ino, unpe, iblock, blkno, + count, ext_fl | SEF_UNWRITTEN); + if (ret < 0) { + err = free_blocks(sb, &datinf->data_alloc, + blkno, count); + BUG_ON(err); /* inconsistent */ + } + } + + up_write(&datinf->alloc_rwsem); + + if (ret < 0) + break; + + iblock += count; + done += count; + ext = find_extent(unpe, iblock, last); } - ret = scoutfs_extent_add(sb, data_extent_io, &fal, lock); + ret = store_packed_extents(sb, ino, unpe, lock); + BUG_ON(ret); /* inconsistent with unpacked and alloc */ + if (ret == 0) - ret = fal.len; + ret = done; + out: - scoutfs_extent_cleanup(ret < 0 && add_rem, scoutfs_extent_add, sb, - data_extent_io, &rem, lock, - SC_DATA_EXTENT_FALLOCATE_CLEANUP, - corrupt_data_extent_fallocate_cleanup, &fal); - scoutfs_extent_cleanup(ret < 0 && add_fr, scoutfs_extent_add, sb, - data_extent_io, &fr, sbi->rid_lock, - SC_DATA_EXTENT_FALLOCATE_CLEANUP, - corrupt_data_extent_alloc_cleanup, &fal); + free_unpacked_extents(unpe); + return ret; } @@ -1105,6 +2022,9 @@ out: * * The caller has only prevented freezing by entering a fs write * context. We're responsible for all other locking and consistency. + * + * This can be used to preallocate files for staging. We find existing + * offline extents and allocate block for them and set unwritten. */ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) { @@ -1112,15 +2032,10 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) struct super_block *sb = inode->i_sb; const u64 ino = scoutfs_ino(inode); struct scoutfs_lock *lock = NULL; - DECLARE_DATA_INFO(sb, datinf); - struct scoutfs_extent ext; LIST_HEAD(ind_locks); - u64 last_block; - u64 iblock; - s64 blocks; loff_t end; - u8 rem_flags; - u8 flags; + u64 iblock; + u64 last; int ret; mutex_lock(&inode->i_mutex); @@ -1157,78 +2072,34 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) } iblock = offset >> SCOUTFS_BLOCK_SHIFT; - last_block = (offset + len - 1) >> SCOUTFS_BLOCK_SHIFT; + last = (offset + len - 1) >> SCOUTFS_BLOCK_SHIFT; - while(iblock <= last_block) { - - scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, - ino, iblock, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, lock); - if (ret < 0 && ret != -ENOENT) - goto out; - - blocks = last_block - iblock + 1; - flags = SEF_UNWRITTEN; - rem_flags = 0; - - if (ret == -ENOENT || ext.start > last_block) { - /* no next extent or past us, all remaining blocks */ - - } else if (iblock < ext.start) { - /* sparse region until next extent */ - blocks = min_t(u64, blocks, ext.start - iblock); - - } else if (ext.map > 0) { - /* skip past an allocated extent */ - blocks = min_t(u64, blocks, - (ext.start + ext.len) - iblock); - iblock += blocks; - blocks = 0; - - } else { - /* allocating a portion of an unallocated extent */ - blocks = min_t(u64, blocks, - (ext.start + ext.len) - iblock); - flags |= ext.flags; - rem_flags = ext.flags; - /* XXX corruption; why'd we store map == flags == 0? */ - if (rem_flags == 0) { - ret = -EIO; - goto out; - } - } + while(iblock <= last) { ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, SIC_FALLOCATE_ONE()); if (ret) goto out; - if (blocks > 0) { - down_write(&datinf->alloc_rwsem); - blocks = fallocate_one_extent(sb, ino, iblock, blocks, - flags, rem_flags, lock); - up_write(&datinf->alloc_rwsem); - if (blocks < 0) - ret = blocks; - else - ret = 0; - } + ret = fallocate_extents(sb, inode, iblock, last, lock); - if (ret == 0 && !(mode & FALLOC_FL_KEEP_SIZE)) { - end = (iblock + blocks) << SCOUTFS_BLOCK_SHIFT; - if (end == 0 || end > offset + len) + if (ret >= 0 && !(mode & FALLOC_FL_KEEP_SIZE)) { + end = (iblock + ret) << SCOUTFS_BLOCK_SHIFT; + if (end > offset + len) end = offset + len; if (end > i_size_read(inode)) i_size_write(inode, end); - scoutfs_update_inode_item(inode, lock, &ind_locks); } + if (ret >= 0) + scoutfs_update_inode_item(inode, lock, &ind_locks); scoutfs_release_trans(sb); scoutfs_inode_index_unlock(sb, &ind_locks); - if (ret) + if (ret <= 0) goto out; - iblock += blocks; + iblock += ret; + ret = 0; } out: @@ -1240,48 +2111,106 @@ out: } /* - * A special case of initialzing a single large offline extent. This + * A special case of initializing a single large offline extent. This * chooses not to deal with any existing extents. It can only be used * on regular files with no data extents. It's used to restore a file * with an offline extent which can then trigger staging. * - * The caller has taken care of locking and holding a transaction. - * - * This could be an fallocate mode. + * The caller has taken care of locking. We're creating many packed + * extent items which may have to be written in multiple transactions. + * We create exetnts from the front of the file and use the offline + * block count to figure out where to continue from. */ int scoutfs_data_init_offline_extent(struct inode *inode, u64 size, struct scoutfs_lock *lock) { struct super_block *sb = inode->i_sb; - struct scoutfs_extent ext; + struct unpacked_extents *unpe = NULL; u64 ino = scoutfs_ino(inode); - u64 len; + LIST_HEAD(ind_locks); + bool held = false; + u64 blocks; + u64 iblock; + u64 count; + u64 on; + u64 off; int ret; - if (!S_ISREG(inode->i_mode)) { - ret = -EINVAL; - goto out; + blocks = DIV_ROUND_UP(size, SCOUTFS_BLOCK_SIZE); + + scoutfs_inode_get_onoff(inode, &on, &off); + iblock = off; + + while (iblock < blocks) { + /* we're updating meta_seq with offline block count */ + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, + SIC_SETATTR_MORE()); + if (ret < 0) + goto out; + held = true; + + ret = scoutfs_dirty_inode_item(inode, lock); + if (ret < 0) + goto out; + + ret = load_unpacked_extents(sb, ino, iblock, iblock, true, + &unpe, lock); + if (ret < 0) + goto out; + + count = min(blocks, last_iblock(iblock) - iblock + 1); + + ret = set_extent(sb, inode, ino, unpe, iblock, 0, count, + SEF_OFFLINE); + if (ret < 0) + goto out; + + free_unpacked_extents(unpe); + unpe = NULL; + + scoutfs_update_inode_item(inode, lock, &ind_locks); + + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + held = false; + + iblock += count; } - scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, ino, 0, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, lock); - if (ret != -ENOENT) { - if (ret == 0) - ret = -EINVAL; - goto out; - } - - len = (size + SCOUTFS_BLOCK_SIZE - 1) >> SCOUTFS_BLOCK_SHIFT; - scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, ino, - 0, len, 0, SEF_OFFLINE); - ret = scoutfs_extent_add(sb, data_extent_io, &ext, lock); - if (ret == 0) - scoutfs_inode_add_onoff(inode, 0, len); + ret = 0; out: + if (held) { + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + } + free_unpacked_extents(unpe); return ret; } +/* + * This copies to userspace :/ + */ +static int fill_extent(struct fiemap_extent_info *fieinfo, + struct unpacked_extent *ext, u32 fiemap_flags) +{ + u32 flags; + + if (ext->count == 0) + return 0; + + flags = fiemap_flags; + if (ext->flags & SEF_OFFLINE) + flags |= FIEMAP_EXTENT_UNKNOWN; + else if (ext->flags & SEF_UNWRITTEN) + flags |= FIEMAP_EXTENT_UNWRITTEN; + + return fiemap_fill_next_extent(fieinfo, + ext->iblock << SCOUTFS_BLOCK_SHIFT, + ext->blkno << SCOUTFS_BLOCK_SHIFT, + ext->count << SCOUTFS_BLOCK_SHIFT, + flags); +} /* * Return all the file's extents whose blocks overlap with the caller's @@ -1292,15 +2221,20 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { struct super_block *sb = inode->i_sb; - struct scoutfs_lock *inode_lock = NULL; - struct scoutfs_extent ext; - u64 blk_off; - u64 logical = 0; - u64 phys = 0; - u64 size = 0; - u32 flags = 0; + const u64 ino = scoutfs_ino(inode); + struct scoutfs_lock *lock = NULL; + struct unpacked_extents *unpe = NULL; + struct unpacked_extent *ext; + struct unpacked_extent cur; + struct scoutfs_traced_extent te; + u32 last_flags; + u64 iblock; + u64 last; int ret; + if (len == 0) + return 0; + ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); if (ret) return ret; @@ -1308,51 +2242,64 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, /* XXX overkill? */ mutex_lock(&inode->i_mutex); - ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &inode_lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &lock); if (ret) goto out; - blk_off = start >> SCOUTFS_BLOCK_SHIFT; + /* use a dummy extent to track */ + memset(&cur, 0, sizeof(cur)); + last_flags = 0; + + iblock = start >> SCOUTFS_BLOCK_SHIFT; + last = (start + len - 1) >> SCOUTFS_BLOCK_SHIFT; for (;;) { - scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, - scoutfs_ino(inode), blk_off, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, inode_lock); - /* fiemap will return last and stop when we see enoent */ - if (ret < 0 && ret != -ENOENT) + ret = load_unpacked_extents(sb, ino, iblock, last, false, + &unpe, lock); + if (ret < 0) { + last_flags = FIEMAP_EXTENT_LAST; break; - - if (ret == 0) - trace_scoutfs_data_fiemap_extent(sb, &ext); - - if (size) { - if (ret == -ENOENT) - flags |= FIEMAP_EXTENT_LAST; - ret = fiemap_fill_next_extent(fieinfo, logical, phys, - size, flags); - if (ret || (logical + size >= (start + len))) { - if (ret == 1) - ret = 0; - break; - } } - logical = ext.start << SCOUTFS_BLOCK_SHIFT; - phys = ext.map << SCOUTFS_BLOCK_SHIFT; - size = ext.len << SCOUTFS_BLOCK_SHIFT; - flags = 0; - if (ext.flags & SEF_OFFLINE) - flags |= FIEMAP_EXTENT_UNKNOWN; - if (ext.flags & SEF_UNWRITTEN) - flags |= FIEMAP_EXTENT_UNWRITTEN; + for (ext = find_extent(unpe, iblock, last); ext; + ext = next_extent(ext)) { - blk_off = ext.start + ext.len; + copy_traced_extent(&te, ext); + trace_scoutfs_data_fiemap_extent(sb, ino, &te); + + if (ext->iblock > last) { + /* not setting _LAST, it's for end of file */ + ret = 0; + break; + } + + if (extents_merge(&cur, ext)) { + cur.count += ext->count; + continue; + } + + ret = fill_extent(fieinfo, &cur, 0); + if (ret != 0) + goto out; + cur = *ext; + } + + iblock = unpe->iblock + SCOUTFS_PACKEXT_BLOCKS; + free_unpacked_extents(unpe); + unpe = NULL; } - scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + if (cur.count) + ret = fill_extent(fieinfo, &cur, last_flags); out: + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); mutex_unlock(&inode->i_mutex); + free_unpacked_extents(unpe); + + if (ret == 1) + ret = 0; + return ret; } @@ -1438,9 +2385,12 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, struct scoutfs_lock *lock) { struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); DECLARE_DATA_WAIT_ROOT(sb, rt); DECLARE_DATA_WAITQ(inode, wq); - struct scoutfs_extent ext = {0,}; + struct unpacked_extents *unpe = NULL; + struct unpacked_extent *ext; + DECLARE_TRACED_EXTENT(te); u64 iblock; u64 last_block; u64 on; @@ -1467,41 +2417,51 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, last_block = (pos + len - 1) >> SCOUTFS_BLOCK_SHIFT; while(iblock <= last_block) { - scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, - scoutfs_ino(inode), iblock, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, lock); + + free_unpacked_extents(unpe); + ret = load_unpacked_extents(sb, ino, iblock, last_block, false, + &unpe, lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; - break; + goto out; } - if (ext.start > last_block) - break; + for (ext = find_extent(unpe, iblock, last_block); ext; + ext = next_extent(ext)) { - if (sef & ext.flags) { - if (dw) { - dw->chg = atomic64_read(&wq->changed); - dw->ino = scoutfs_ino(inode); - dw->iblock = max(iblock, ext.start); - dw->op = op; - - spin_lock(&rt->lock); - insert_offline_waiting(&rt->root, dw); - spin_unlock(&rt->lock); + if (ext->iblock > last_block) { + ret = 0; + goto out; + } + + if (sef & ext->flags) { + if (dw) { + dw->chg = atomic64_read(&wq->changed); + dw->ino = ino; + dw->iblock = max(iblock, ext->iblock); + dw->op = op; + + spin_lock(&rt->lock); + insert_offline_waiting(&rt->root, dw); + spin_unlock(&rt->lock); + } + + copy_traced_extent(&te, ext); + ret = 1; + goto out; } - ret = 1; - break; } - iblock = ext.start + ext.len; + iblock = unpe->iblock + SCOUTFS_PACKEXT_BLOCKS; } out: - trace_scoutfs_data_wait_check(sb, scoutfs_ino(inode), pos, len, - sef, op, ext.start, ext.len, ext.flags, - ret); + trace_scoutfs_data_wait_check(sb, ino, pos, len, sef, op, &te, ret); + + free_unpacked_extents(unpe); + return ret; } @@ -1609,93 +2569,34 @@ const struct file_operations scoutfs_file_fops = { .fallocate = scoutfs_fallocate, }; -/* - * Return extents to the server if we're over the high water mark. Each - * work call sends one batch of extents so that the work can be easily - * canceled to stop progress during unmount. - */ -static void scoutfs_data_return_server_extents_worker(struct work_struct *work) +void scoutfs_data_init_btrees(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_log_trees *lt) { - struct data_info *datinf = container_of(work, struct data_info, - return_work); - struct super_block *sb = datinf->sb; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_net_extent_list *nexl; - struct scoutfs_extent ext; - u64 nr = 0; - u64 free; - int bytes; - int ret; - int err; - - trace_scoutfs_data_return_server_extents_enter(sb, 0, 0); - - bytes = SCOUTFS_NET_EXTENT_LIST_BYTES(SCOUTFS_NET_EXTENT_LIST_MAX_NR); - nexl = kmalloc(bytes, GFP_NOFS); - if (!nexl) { - ret = -ENOMEM; - goto out; - } - - ret = scoutfs_hold_trans(sb, SIC_RETURN_EXTENTS()); - if (ret) - goto out; + DECLARE_DATA_INFO(sb, datinf); down_write(&datinf->alloc_rwsem); - free = atomic64_read(&datinf->node_free_blocks); - - while (nr < SCOUTFS_NET_EXTENT_LIST_MAX_NR && - free > NODE_FREE_HIGH_WATER_BLOCKS) { - - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, - sbi->rid, 0, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, - sbi->rid_lock); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - break; - } - - trace_scoutfs_data_return_server_extent(sb, &ext); - - ext.type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; - ext.len = min(ext.len, free - NODE_FREE_HIGH_WATER_BLOCKS); - - ret = scoutfs_extent_remove(sb, data_extent_io, &ext, - sbi->rid_lock); - if (ret) - break; - - nexl->extents[nr].start = cpu_to_le64(ext.start); - nexl->extents[nr].len = cpu_to_le64(ext.len); - - nr++; - free -= ext.len; - } - - nexl->nr = cpu_to_le64(nr); + datinf->alloc = alloc; + datinf->wri = wri; + datinf->data_alloc = lt->data_alloc; + datinf->data_free = lt->data_free; up_write(&datinf->alloc_rwsem); +} - if (nr > 0) { - err = scoutfs_client_free_extents(sb, nexl); - /* XXX leaked extents if free failed */ - if (ret == 0 && err < 0) - ret = err; - } +void scoutfs_data_get_btrees(struct super_block *sb, + struct scoutfs_log_trees *lt) +{ + DECLARE_DATA_INFO(sb, datinf); - scoutfs_release_trans(sb); -out: - kfree(nexl); + down_read(&datinf->alloc_rwsem); - trace_scoutfs_data_return_server_extents_exit(sb, nr, ret); + lt->data_alloc = datinf->data_alloc; + lt->data_free = datinf->data_free; - /* keep returning if we're still over the water mark */ - if (ret == 0 && (atomic64_read(&datinf->node_free_blocks) > - NODE_FREE_HIGH_WATER_BLOCKS)) - queue_work(datinf->workq, &datinf->return_work); + up_read(&datinf->alloc_rwsem); } int scoutfs_data_setup(struct super_block *sb) @@ -1709,15 +2610,6 @@ int scoutfs_data_setup(struct super_block *sb) datinf->sb = sb; init_rwsem(&datinf->alloc_rwsem); - atomic64_set(&datinf->node_free_blocks, 0); - INIT_WORK(&datinf->return_work, - scoutfs_data_return_server_extents_worker); - - datinf->workq = alloc_workqueue("scoutfs_data", WQ_UNBOUND, 1); - if (!datinf->workq) { - kfree(datinf); - return -ENOMEM; - } sbi->data_info = datinf; return 0; @@ -1729,12 +2621,6 @@ void scoutfs_data_destroy(struct super_block *sb) struct data_info *datinf = sbi->data_info; if (datinf) { - if (datinf->workq) { - cancel_work_sync(&datinf->return_work); - destroy_workqueue(datinf->workq); - datinf->workq = NULL; - } - sbi->data_info = NULL; kfree(datinf); } diff --git a/kmod/src/data.h b/kmod/src/data.h index 21deeac4..912284d9 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -36,8 +36,17 @@ struct scoutfs_data_wait { .node.__rb_parent_color = (unsigned long)(&nm.node), \ } +struct scoutfs_traced_extent { + u64 iblock; + u64 count; + u64 blkno; + u8 flags; +}; + extern const struct address_space_operations scoutfs_file_aops; extern const struct file_operations scoutfs_file_fops; +struct scoutfs_balloc_allocator; +struct scoutfs_block_writer; int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, u64 ino, u64 iblock, u64 last, bool offline, @@ -63,6 +72,24 @@ int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, struct scoutfs_ioctl_data_waiting_entry *dwe, unsigned int nr); +int scoutfs_data_move_alloc_bits(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_balloc_root *dst, + struct scoutfs_balloc_root *src, + __le64 *cursor, u64 min_dst_total); +int scoutfs_data_add_free_blocks(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_balloc_root *broot, + u64 blkno, u64 count); +void scoutfs_data_init_btrees(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_log_trees *lt); +void scoutfs_data_get_btrees(struct super_block *sb, + struct scoutfs_log_trees *lt); + int scoutfs_data_setup(struct super_block *sb); void scoutfs_data_destroy(struct super_block *sb); diff --git a/kmod/src/forest.c b/kmod/src/forest.c index b405457f..ac49459c 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -61,8 +61,8 @@ struct forest_info { struct rw_semaphore rwsem; - struct scoutfs_balloc_allocator alloc; - struct scoutfs_block_writer wri; + struct scoutfs_balloc_allocator *alloc; + struct scoutfs_block_writer *wri; struct scoutfs_log_trees our_log; }; @@ -1025,14 +1025,14 @@ static int set_lock_bloom_bits(struct super_block *sb, if (!ref->blkno || !scoutfs_block_writer_is_dirty(sb, bl)) { - ret = scoutfs_balloc_alloc(sb, &finf->alloc, &finf->wri, + ret = scoutfs_balloc_alloc(sb, finf->alloc, finf->wri, &blkno); if (ret < 0) goto unlock; new_bl = scoutfs_block_create(sb, blkno); if (IS_ERR(new_bl)) { - err = scoutfs_balloc_free(sb, &finf->alloc, &finf->wri, + err = scoutfs_balloc_free(sb, finf->alloc, finf->wri, blkno); BUG_ON(err); /* could have dirtied */ ret = PTR_ERR(new_bl); @@ -1040,7 +1040,7 @@ static int set_lock_bloom_bits(struct super_block *sb, } if (bl) { - err = scoutfs_balloc_free(sb, &finf->alloc, &finf->wri, + err = scoutfs_balloc_free(sb, finf->alloc, finf->wri, le64_to_cpu(ref->blkno)); BUG_ON(err); /* could have dirtied */ memcpy(new_bl->data, bl->data, SCOUTFS_BLOCK_SIZE); @@ -1048,7 +1048,7 @@ static int set_lock_bloom_bits(struct super_block *sb, memset(new_bl->data, 0, SCOUTFS_BLOCK_SIZE); } - scoutfs_block_writer_mark_dirty(sb, &finf->wri, new_bl); + scoutfs_block_writer_mark_dirty(sb, finf->wri, new_bl); scoutfs_block_put(sb, bl); bl = new_bl; @@ -1165,7 +1165,7 @@ static int forest_insert(struct super_block *sb, struct scoutfs_key *key, scoutfs_key_to_be(&kbe, key); down_write(&finf->rwsem); - ret = scoutfs_btree_force(sb, &finf->alloc, &finf->wri, + ret = scoutfs_btree_force(sb, finf->alloc, finf->wri, &finf->our_log.item_root, &kbe, sizeof(kbe), iv->iov_base, iv->iov_len); up_write(&finf->rwsem); @@ -1249,7 +1249,7 @@ static int forest_delete(struct super_block *sb, struct scoutfs_key *key, liv.flags = SCOUTFS_LOG_ITEM_FLAG_DELETION; down_write(&finf->rwsem); - ret = scoutfs_btree_force(sb, &finf->alloc, &finf->wri, + ret = scoutfs_btree_force(sb, finf->alloc, finf->wri, &finf->our_log.item_root, &kbe, sizeof(kbe), &liv, sizeof(liv)); up_write(&finf->rwsem); @@ -1318,69 +1318,41 @@ void scoutfs_forest_free_batch(struct super_block *sb, struct list_head *list) /* * This is called from transactions as a new transaction opens and is - * serialized with all writers. Get the tree roots that we'll need - * for the transaction. + * serialized with all writers. */ -int scoutfs_forest_get_log_trees(struct super_block *sb) +void scoutfs_forest_init_btrees(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_log_trees *lt) { DECLARE_FOREST_INFO(sb, finf); - struct scoutfs_log_trees lt; - int ret; - - ret = scoutfs_client_get_log_trees(sb, <); - if (ret) - goto out; down_write(&finf->rwsem); - scoutfs_balloc_init(&finf->alloc, <.alloc_root, <.free_root); - scoutfs_block_writer_init(sb, &finf->wri); - finf->our_log = lt; + + finf->alloc = alloc; + finf->wri = wri; + + /* we use the item and bloom trees */ + memset(&finf->our_log, 0, sizeof(finf->our_log)); + finf->our_log.item_root = lt->item_root; + finf->our_log.bloom_ref = lt->bloom_ref; + up_write(&finf->rwsem); - - ret = 0; -out: - return ret; -} - -bool scoutfs_forest_has_dirty(struct super_block *sb) -{ - DECLARE_FOREST_INFO(sb, finf); - - return scoutfs_block_writer_has_dirty(sb, &finf->wri); -} - -unsigned long scoutfs_forest_dirty_bytes(struct super_block *sb) -{ - DECLARE_FOREST_INFO(sb, finf); - - return scoutfs_block_writer_dirty_bytes(sb, &finf->wri); -} - -int scoutfs_forest_write(struct super_block *sb) -{ - DECLARE_FOREST_INFO(sb, finf); - - return scoutfs_block_writer_write(sb, &finf->wri); } /* * This is called during transaction commit which excludes forest writer * calls. The caller has already written all the dirty blocks that the - * forest roots reference. + * forest roots reference. They're getting the roots to send to the server + * for the commit. */ -int scoutfs_forest_commit(struct super_block *sb) +void scoutfs_forest_get_btrees(struct super_block *sb, + struct scoutfs_log_trees *lt) { DECLARE_FOREST_INFO(sb, finf); - struct scoutfs_log_trees lt = { - .alloc_root = finf->alloc.alloc_root, - .free_root = finf->alloc.free_root, - .item_root = finf->our_log.item_root, - .bloom_ref = finf->our_log.bloom_ref, - .rid = finf->our_log.rid, - .nr = finf->our_log.nr, - }; - return scoutfs_client_commit_log_trees(sb, <); + lt->item_root = finf->our_log.item_root; + lt->bloom_ref = finf->our_log.bloom_ref; } int scoutfs_forest_setup(struct super_block *sb) @@ -1395,7 +1367,7 @@ int scoutfs_forest_setup(struct super_block *sb) goto out; } - /* the finf fields will be setup as we open a transaction */ + /* the finf fields will be setup as we open a transaction */ init_rwsem(&finf->rwsem); sbi->forest_info = finf; @@ -1413,7 +1385,6 @@ void scoutfs_forest_destroy(struct super_block *sb) struct forest_info *finf = SCOUTFS_SB(sb)->forest_info; if (finf) { - scoutfs_block_writer_forget_all(sb, &finf->wri); kfree(finf); sbi->forest_info = NULL; } diff --git a/kmod/src/forest.h b/kmod/src/forest.h index 1b1d5641..7757d1f1 100644 --- a/kmod/src/forest.h +++ b/kmod/src/forest.h @@ -1,6 +1,9 @@ #ifndef _SCOUTFS_FOREST_H_ #define _SCOUTFS_FOREST_H_ +struct scoutfs_balloc_allocator; +struct scoutfs_block_writer; + int scoutfs_forest_lookup(struct super_block *sb, struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock); int scoutfs_forest_lookup_exact(struct super_block *sb, @@ -36,11 +39,12 @@ int scoutfs_forest_restore(struct super_block *sb, struct list_head *list, struct scoutfs_lock *lock); void scoutfs_forest_free_batch(struct super_block *sb, struct list_head *list); -int scoutfs_forest_get_log_trees(struct super_block *sb); -bool scoutfs_forest_has_dirty(struct super_block *sb); -unsigned long scoutfs_forest_dirty_bytes(struct super_block *sb); -int scoutfs_forest_write(struct super_block *sb); -int scoutfs_forest_commit(struct super_block *sb); +void scoutfs_forest_init_btrees(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_log_trees *lt); +void scoutfs_forest_get_btrees(struct super_block *sb, + struct scoutfs_log_trees *lt); void scoutfs_forest_clear_lock(struct super_block *sb, struct scoutfs_lock *lock); diff --git a/kmod/src/format.h b/kmod/src/format.h index cb804c4e..9b9fd06d 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -105,11 +105,6 @@ struct scoutfs_key { #define skxi_ino _sk_second #define skxi_id _sk_third -/* node free extent */ -#define sknf_rid _sk_first -#define sknf_major _sk_second -#define sknf_minor _sk_third - /* node orphan inode */ #define sko_rid _sk_first #define sko_ino _sk_second @@ -132,9 +127,10 @@ struct scoutfs_key { #define sks_ino _sk_first #define sks_nr _sk_second -/* file extent */ -#define skfe_ino _sk_first -#define skfe_last _sk_second +/* packed extents */ +#define skpe_ino _sk_first +#define skpe_base _sk_second +#define skpe_part _sk_fourth /* * The btree still uses memcmp() to compare keys. We should fix that @@ -235,18 +231,32 @@ struct scoutfs_balloc_item_val { } __packed; /* - * Free extents are stored in the server in an allocation btree. The - * type differentiates whether start or length is in stored in the major - * value and is the primary sort key. 'start' is set to the final block - * in the extent so that overlaping queries can be done with next - * instead prev. + * Free data blocks are tracked in bitmaps stored in btree items. */ -struct scoutfs_extent_btree_key { +struct scoutfs_block_bitmap_key { __u8 type; - __be64 major; - __be64 minor; + __be64 base; } __packed; +#define SCOUTFS_BLOCK_BITMAP_BIG 0 +#define SCOUTFS_BLOCK_BITMAP_LITTLE 1 + +#define SCOUTFS_PACKED_BITMAP_WORDS 32 +#define SCOUTFS_PACKED_BITMAP_BITS (SCOUTFS_PACKED_BITMAP_WORDS * 64) +#define SCOUTFS_PACKED_BITMAP_MAX_BYTES \ + offsetof(struct scoutfs_packed_bitmap, \ + words[SCOUTFS_PACKED_BITMAP_WORDS]) + +#define SCOUTFS_BLOCK_BITMAP_BITS SCOUTFS_PACKED_BITMAP_BITS +#define SCOUTFS_BLOCK_BITMAP_BIT_MASK (SCOUTFS_PACKED_BITMAP_BITS - 1) +#define SCOUTFS_BLOCK_BITMAP_BASE_SHIFT (ilog2(SCOUTFS_PACKED_BITMAP_BITS)) + +struct scoutfs_packed_bitmap { + __le64 present; + __le64 set; + __le64 words[0]; +}; + /* * The lock server keeps a persistent record of connected clients so that * server failover knows who to wait for before resuming operations. @@ -277,11 +287,18 @@ struct scoutfs_mounted_client_btree_val { #define SCOUTFS_MOUNTED_CLIENT_VOTER (1 << 0) +/* + * XXX I imagine we should rename these now that they've evolved to track + * all the btrees that clients use during a transaction. It's not just + * about item logs, it's about clients making changes to trees. + */ struct scoutfs_log_trees { struct scoutfs_balloc_root alloc_root; struct scoutfs_balloc_root free_root; struct scoutfs_btree_root item_root; struct scoutfs_btree_ref bloom_ref; + struct scoutfs_balloc_root data_alloc; + struct scoutfs_balloc_root data_free; __le64 rid; __le64 nr; } __packed; @@ -296,6 +313,8 @@ struct scoutfs_log_trees_val { struct scoutfs_balloc_root free_root; struct scoutfs_btree_root item_root; struct scoutfs_btree_ref bloom_ref; + struct scoutfs_balloc_root data_alloc; + struct scoutfs_balloc_root data_free; } __packed; struct scoutfs_log_item_value { @@ -351,9 +370,7 @@ struct scoutfs_bloom_block { #define SCOUTFS_XATTR_INDEX_NAME_TYPE 1 /* rid zone (also used in server alloc btree) */ -#define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 1 -#define SCOUTFS_FREE_EXTENT_BLOCKS_TYPE 2 -#define SCOUTFS_ORPHAN_TYPE 3 +#define SCOUTFS_ORPHAN_TYPE 1 /* fs zone */ #define SCOUTFS_INODE_TYPE 1 @@ -362,23 +379,45 @@ struct scoutfs_bloom_block { #define SCOUTFS_READDIR_TYPE 4 #define SCOUTFS_LINK_BACKREF_TYPE 5 #define SCOUTFS_SYMLINK_TYPE 6 -#define SCOUTFS_FILE_EXTENT_TYPE 7 +#define SCOUTFS_PACKED_EXTENT_TYPE 7 /* lock zone, only ever found in lock ranges, never in persistent items */ #define SCOUTFS_RENAME_TYPE 1 #define SCOUTFS_MAX_TYPE 8 /* power of 2 is efficient */ + /* - * File extents have more data than easily fits in the key so we move - * the non-indexed fields into the value. + * The extents that map blocks in a fixed-size logical region of a file + * are packed and stored in item values. The packed extents are + * contiguous so the starting logical block is implicit from the length + * of previous extents. Sparse regions are represented by 0 flags and + * blkno. The blkno of a packed extent is encoded as the zigzag (lsb is + * sign bit) difference from the last blkno of the previous extent. + * This guarantees that non-sparse extents must have a blkno delta of at + * least -1/1. High zero byte aren't stored. */ -struct scoutfs_file_extent { - __le64 blkno; - __le64 len; - __u8 flags; +struct scoutfs_packed_extent { + __le16 count; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 diff_bytes:4, + flags:3, + final:1; +#elif defined(__BIG_ENDIAN_BITFIELD) + __u8 final:1, + flags:3, + diff_bytes:4; +#else +#error "no {BIG,LITTLE}_ENDIAN_BITFIELD defined?" +#endif + __u8 le_blkno_diff[0]; } __packed; +#define SCOUTFS_PACKEXT_BLOCKS (8 * 1024 * 1024 / SCOUTFS_BLOCK_SIZE) +#define SCOUTFS_PACKEXT_BASE_SHIFT (ilog2(SCOUTFS_PACKEXT_BLOCKS)) +#define SCOUTFS_PACKEXT_BASE_MASK (~((__u64)SCOUTFS_PACKEXT_BLOCKS - 1)) +#define SCOUTFS_PACKEXT_MAX_BYTES SCOUTFS_MAX_VAL_SIZE + #define SEF_OFFLINE (1 << 0) #define SEF_UNWRITTEN (1 << 1) #define SEF_UNKNOWN (U8_MAX << 2) @@ -451,8 +490,12 @@ struct scoutfs_super_block { __le64 next_ino; __le64 next_trans_seq; __le64 total_blocks; - __le64 next_uninit_free_block; + __le64 next_uninit_meta_blkno; + __le64 last_uninit_meta_blkno; + __le64 next_uninit_data_blkno; + __le64 last_uninit_data_blkno; __le64 core_balloc_cursor; + __le64 core_data_alloc_cursor; __le64 free_blocks; __le64 first_fs_blkno; __le64 last_fs_blkno; @@ -463,7 +506,8 @@ struct scoutfs_super_block { struct scoutfs_inet_addr server_addr; struct scoutfs_balloc_root core_balloc_alloc; struct scoutfs_balloc_root core_balloc_free; - struct scoutfs_btree_root alloc_root; + struct scoutfs_balloc_root core_data_alloc; + struct scoutfs_balloc_root core_data_free; struct scoutfs_btree_root fs_root; struct scoutfs_btree_root logs_root; struct scoutfs_btree_root lock_clients; @@ -653,8 +697,6 @@ struct scoutfs_net_header { enum { SCOUTFS_NET_CMD_GREETING = 0, SCOUTFS_NET_CMD_ALLOC_INODES, - SCOUTFS_NET_CMD_ALLOC_EXTENT, - SCOUTFS_NET_CMD_FREE_EXTENTS, SCOUTFS_NET_CMD_GET_LOG_TREES, SCOUTFS_NET_CMD_COMMIT_LOG_TREES, SCOUTFS_NET_CMD_ADVANCE_SEQ, @@ -705,25 +747,6 @@ struct scoutfs_net_statfs { __u8 uuid[SCOUTFS_UUID_BYTES]; /* logical volume uuid */ } __packed; -struct scoutfs_net_extent { - __le64 start; - __le64 len; -} __packed; - -struct scoutfs_net_extent_list { - __le64 nr; - struct { - __le64 start; - __le64 len; - } __packed extents[0]; -} __packed; - -#define SCOUTFS_NET_EXTENT_LIST_BYTES(nr) \ - offsetof(struct scoutfs_net_extent_list, extents[nr]) - -/* arbitrarily makes a nice ~1k extent list payload */ -#define SCOUTFS_NET_EXTENT_LIST_MAX_NR 64 - struct scoutfs_net_lock { struct scoutfs_key key; __le64 write_version; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 10196df4..eeebec8e 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -587,6 +587,13 @@ static long scoutfs_ioc_setattr_more(struct file *file, unsigned long arg) if (ret) goto unlock; + /* create offline extents in potentially many transactions */ + if (sm.flags & SCOUTFS_IOC_SETATTR_MORE_OFFLINE) { + ret = scoutfs_data_init_offline_extent(inode, sm.i_size, lock); + if (ret) + goto unlock; + } + /* can only change size/dv on untouched regular files */ if ((sm.i_size != 0 || sm.data_version != 0) && ((!S_ISREG(inode->i_mode) || @@ -602,12 +609,6 @@ static long scoutfs_ioc_setattr_more(struct file *file, unsigned long arg) if (ret) goto unlock; - if (sm.flags & SCOUTFS_IOC_SETATTR_MORE_OFFLINE) { - ret = scoutfs_data_init_offline_extent(inode, sm.i_size, lock); - if (ret) - goto release; - } - if (sm.data_version) scoutfs_inode_set_data_version(inode, sm.data_version); if (sm.i_size) @@ -618,7 +619,6 @@ static long scoutfs_ioc_setattr_more(struct file *file, unsigned long arg) scoutfs_update_inode_item(inode, lock, &ind_locks); ret = 0; -release: scoutfs_release_trans(sb); unlock: scoutfs_inode_index_unlock(sb, &ind_locks); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index c491ed22..cdb6e561 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -157,7 +157,7 @@ static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, mode != SCOUTFS_LOCK_NULL); /* any transition from a mode allowed to dirty items has to write */ - if (lock_mode_can_write(prev) && scoutfs_forest_has_dirty(sb)) { + if (lock_mode_can_write(prev) && scoutfs_trans_has_dirty(sb)) { ret = scoutfs_trans_sync(sb, 1); if (ret < 0) return ret; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index affdd3d1..712ab69b 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -34,12 +34,33 @@ #include "count.h" #include "export.h" #include "dir.h" -#include "extents.h" #include "server.h" #include "net.h" +#include "data.h" struct lock_info; +#define STE_FMT "[%llu %llu %llu 0x%x]" +#define STE_ARGS(te) (te)->iblock, (te)->count, (te)->blkno, (te)->flags +#define STE_FIELDS(pref) \ + __field(__u64, pref##_iblock) \ + __field(__u64, pref##_count) \ + __field(__u64, pref##_blkno) \ + __field(__u8, pref##_flags) +#define STE_ASSIGN(pref, te) \ + __entry->pref##_iblock = (te)->iblock; \ + __entry->pref##_count = (te)->count; \ + __entry->pref##_blkno = (te)->blkno; \ + __entry->pref##_flags = (te)->flags; +#define STE_ENTRY_ARGS(pref) \ + __entry->pref##_iblock, \ + __entry->pref##_count, \ + __entry->pref##_blkno, \ + __entry->pref##_flags + +#define DECLARE_TRACED_EXTENT(name) \ + struct scoutfs_traced_extent name = {0} + TRACE_EVENT(scoutfs_setattr, TP_PROTO(struct dentry *dentry, struct iattr *attr), @@ -152,15 +173,17 @@ TRACE_EVENT(scoutfs_data_fiemap, TRACE_EVENT(scoutfs_get_block, TP_PROTO(struct super_block *sb, __u64 ino, __u64 iblock, - int create, int ret, __u64 blkno, size_t size), + int create, struct scoutfs_traced_extent *te, + int ret, __u64 blkno, size_t size), - TP_ARGS(sb, ino, iblock, create, ret, blkno, size), + TP_ARGS(sb, ino, iblock, create, te, ret, blkno, size), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, ino) __field(__u64, iblock) __field(int, create) + STE_FIELDS(ext) __field(int, ret) __field(__u64, blkno) __field(size_t, size) @@ -171,68 +194,58 @@ TRACE_EVENT(scoutfs_get_block, __entry->ino = ino; __entry->iblock = iblock; __entry->create = create; + STE_ASSIGN(ext, te) __entry->ret = ret; __entry->blkno = blkno; __entry->size = size; ), - TP_printk(SCSBF" ino %llu iblock %llu create %d ret %d bnr %llu " - "size %zu", SCSB_TRACE_ARGS, __entry->ino, __entry->iblock, - __entry->create, __entry->ret, __entry->blkno, __entry->size) + TP_printk(SCSBF" ino %llu iblock %llu create %d ext "STE_FMT" ret %d bnr %llu size %zu", + SCSB_TRACE_ARGS, __entry->ino, __entry->iblock, + __entry->create, STE_ENTRY_ARGS(ext), __entry->ret, + __entry->blkno, __entry->size) ); -TRACE_EVENT(scoutfs_data_alloc_block, - TP_PROTO(struct super_block *sb, struct inode *inode, - struct scoutfs_extent *ext, u64 iblock, u64 len, - u64 online_blocks, u64 offline_blocks), +TRACE_EVENT(scoutfs_data_file_extent_class, + TP_PROTO(struct super_block *sb, __u64 ino, + struct scoutfs_traced_extent *te), - TP_ARGS(sb, inode, ext, iblock, len, online_blocks, offline_blocks), + TP_ARGS(sb, ino, te), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, ino) - se_trace_define(ext) - __field(__u64, iblock) - __field(__u64, len) - __field(__u64, online_blocks) - __field(__u64, offline_blocks) + STE_FIELDS(ext) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->ino = scoutfs_ino(inode); - se_trace_assign(ext, ext); - __entry->iblock = iblock; - __entry->len = len; - __entry->online_blocks = online_blocks; - __entry->offline_blocks = offline_blocks; + __entry->ino = ino; + STE_ASSIGN(ext, te) ), - TP_printk(SCSBF" ino %llu ext "SE_FMT" iblock %llu len %llu online_blocks %llu offline_blocks %llu", - SCSB_TRACE_ARGS, __entry->ino, se_trace_args(ext), - __entry->iblock, __entry->len, __entry->online_blocks, - __entry->offline_blocks) + TP_printk(SCSBF" ino %llu ext "STE_FMT, + SCSB_TRACE_ARGS, __entry->ino, STE_ENTRY_ARGS(ext)) ); - -TRACE_EVENT(scoutfs_data_alloc_block_ret, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext, int ret), - - TP_ARGS(sb, ext, ret), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - se_trace_define(ext) - __field(int, ret) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - se_trace_assign(ext, ext); - __entry->ret = ret; - ), - - TP_printk(SCSBF" ext "SE_FMT" ret %d", SCSB_TRACE_ARGS, - se_trace_args(ext), __entry->ret) +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_alloc_block, + TP_PROTO(struct super_block *sb, __u64 ino, + struct scoutfs_traced_extent *te), + TP_ARGS(sb, ino, te) +); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_prealloc_unwritten, + TP_PROTO(struct super_block *sb, __u64 ino, + struct scoutfs_traced_extent *te), + TP_ARGS(sb, ino, te) +); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_extent_truncated, + TP_PROTO(struct super_block *sb, __u64 ino, + struct scoutfs_traced_extent *te), + TP_ARGS(sb, ino, te) +); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_fiemap_extent, + TP_PROTO(struct super_block *sb, __u64 ino, + struct scoutfs_traced_extent *te), + TP_ARGS(sb, ino, te) ); TRACE_EVENT(scoutfs_data_truncate_items, @@ -260,10 +273,9 @@ TRACE_EVENT(scoutfs_data_truncate_items, TRACE_EVENT(scoutfs_data_wait_check, TP_PROTO(struct super_block *sb, __u64 ino, __u64 pos, __u64 len, - __u8 sef, __u8 op, __u64 ext_start, __u64 ext_len, - __u8 ext_flags, int ret), + __u8 sef, __u8 op, struct scoutfs_traced_extent *te, int ret), - TP_ARGS(sb, ino, pos, len, sef, op, ext_start, ext_len, ext_flags, ret), + TP_ARGS(sb, ino, pos, len, sef, op, te, ret), TP_STRUCT__entry( SCSB_TRACE_FIELDS @@ -272,9 +284,7 @@ TRACE_EVENT(scoutfs_data_wait_check, __field(__u64, len) __field(__u8, sef) __field(__u8, op) - __field(__u64, ext_start) - __field(__u64, ext_len) - __field(__u8, ext_flags) + STE_FIELDS(ext) __field(int, ret) ), @@ -285,16 +295,13 @@ TRACE_EVENT(scoutfs_data_wait_check, __entry->len = len; __entry->sef = sef; __entry->op = op; - __entry->ext_start = ext_start; - __entry->ext_len = ext_len; - __entry->ext_flags = ext_flags; + STE_ASSIGN(ext, te) __entry->ret = ret; ), - TP_printk(SCSBF" ino %llu pos %llu len %llu sef 0x%x op 0x%x ext_start %llu ext_len %llu ext_flags 0x%x ret %d", + TP_printk(SCSBF" ino %llu pos %llu len %llu sef 0x%x op 0x%x ext "STE_FMT" ret %d", SCSB_TRACE_ARGS, __entry->ino, __entry->pos, __entry->len, - __entry->sef, __entry->op, __entry->ext_start, - __entry->ext_len, __entry->ext_flags, __entry->ret) + __entry->sef, __entry->op, STE_ENTRY_ARGS(ext), __entry->ret) ); TRACE_EVENT(scoutfs_sync_fs, @@ -1588,115 +1595,6 @@ TRACE_EVENT(scoutfs_btree_dirty_block, __entry->bt_blkno, __entry->bt_seq) ); -DECLARE_EVENT_CLASS(scoutfs_extent_class, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - - TP_ARGS(sb, ext), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - se_trace_define(ext) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - se_trace_assign(ext, ext); - ), - - TP_printk(SCSBF" ext "SE_FMT, - SCSB_TRACE_ARGS, se_trace_args(ext)) -); - -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_insert, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_delete, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_next_input, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_next_output, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_prev_input, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_prev_output, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_add, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_remove, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); - -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_truncate_next, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_truncate_remove, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_truncate_offline, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_get_server_extent, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_find_free_extent, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_alloc_block_next, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_get_block_next, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_get_block_intersection, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_fiemap_extent, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_return_server_extent, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_alloc_extent_next, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_alloc_extent_allocated, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_free_pending_extent, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_extent_io, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); - TRACE_EVENT(scoutfs_online_offline_blocks, TP_PROTO(struct inode *inode, s64 on_delta, s64 off_delta, u64 on_now, u64 off_now), diff --git a/kmod/src/server.c b/kmod/src/server.c index ee378ea8..54dee331 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -68,9 +68,7 @@ struct server_info { /* server tracks seq use */ struct rw_semaphore seq_rwsem; - /* server tracks pending frees to be applied during commit */ struct rw_semaphore alloc_rwsem; - struct list_head pending_frees; struct list_head clients; unsigned long nr_clients; @@ -103,291 +101,6 @@ struct commit_waiter { int ret; }; -static void init_extent_btree_key(struct scoutfs_extent_btree_key *ebk, - u8 type, u64 major, u64 minor) -{ - ebk->type = type; - ebk->major = cpu_to_be64(major); - ebk->minor = cpu_to_be64(minor); -} - -static int init_extent_from_btree_key(struct scoutfs_extent *ext, u8 type, - struct scoutfs_extent_btree_key *ebk, - unsigned int key_bytes) -{ - u64 start; - u64 len; - - /* btree _next doesn't have last key limit */ - if (ebk->type != type) - return -ENOENT; - - if (key_bytes != sizeof(struct scoutfs_extent_btree_key) || - (ebk->type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE && - ebk->type != SCOUTFS_FREE_EXTENT_BLOCKS_TYPE)) - return -EIO; /* XXX corruption, bad key */ - - start = be64_to_cpu(ebk->major); - len = be64_to_cpu(ebk->minor); - if (ebk->type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) - swap(start, len); - start -= len - 1; - - return scoutfs_extent_init(ext, ebk->type, 0, start, len, 0, 0); -} - -/* - * This is called by the extent core on behalf of the server who holds - * the appropriate locks to protect the many btree items that can be - * accessed on behalf of one extent operation. - * - * The free_blocks count in the super tracks the number of blocks in - * the primary extent index. We update it here instead of expecting - * callers to remember. - */ -static int server_extent_io(struct super_block *sb, int op, - struct scoutfs_extent *ext, void *data) -{ - DECLARE_SERVER_INFO(sb, server); - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_extent_btree_key ebk; - SCOUTFS_BTREE_ITEM_REF(iref); - bool mirror = false; - u8 mirror_type; - u8 mirror_op = 0; - int ret; - int err; - - trace_scoutfs_server_extent_io(sb, ext); - - if (WARN_ON_ONCE(ext->type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE && - ext->type != SCOUTFS_FREE_EXTENT_BLOCKS_TYPE)) - return -EINVAL; - - if (ext->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE && - (op == SEI_INSERT || op == SEI_DELETE)) { - mirror = true; - mirror_type = SCOUTFS_FREE_EXTENT_BLOCKS_TYPE; - mirror_op = op == SEI_INSERT ? SEI_DELETE : SEI_INSERT; - } - - init_extent_btree_key(&ebk, ext->type, ext->start + ext->len - 1, - ext->len); - if (ext->type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) - swap(ebk.major, ebk.minor); - - if (op == SEI_NEXT || op == SEI_PREV) { - if (op == SEI_NEXT) - ret = scoutfs_btree_next(sb, &super->alloc_root, - &ebk, sizeof(ebk), &iref); - else - ret = scoutfs_btree_prev(sb, &super->alloc_root, - &ebk, sizeof(ebk), &iref); - if (ret == 0) { - ret = init_extent_from_btree_key(ext, ext->type, - iref.key, - iref.key_len); - scoutfs_btree_put_iref(&iref); - } - - } else if (op == SEI_INSERT) { - ret = scoutfs_btree_insert(sb, &server->alloc, &server->wri, - &super->alloc_root, - &ebk, sizeof(ebk), NULL, 0); - - } else if (op == SEI_DELETE) { - ret = scoutfs_btree_delete(sb, &server->alloc, &server->wri, - &super->alloc_root, - &ebk, sizeof(ebk)); - - } else { - ret = WARN_ON_ONCE(-EINVAL); - } - - if (ret == 0 && mirror) { - swap(ext->type, mirror_type); - ret = server_extent_io(sb, op, ext, data); - swap(ext->type, mirror_type); - if (ret < 0) { - err = server_extent_io(sb, mirror_op, ext, data); - if (err) - scoutfs_corruption(sb, - SC_SERVER_EXTENT_CLEANUP, - corrupt_server_extent_cleanup, - "op %u ext "SE_FMT" ret %d", - op, SE_ARG(ext), err); - } - } - - if (ret == 0 && ext->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { - if (op == SEI_INSERT) - le64_add_cpu(&super->free_blocks, ext->len); - else if (op == SEI_DELETE) - le64_add_cpu(&super->free_blocks, -ext->len); - } - - return ret; -} - -/* - * Allocate an extent of the given length in the first smallest free - * extent that contains it. - */ -static int alloc_extent(struct super_block *sb, u64 blocks, - u64 *start, u64 *len) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct scoutfs_extent ext; - int ret; - - *start = 0; - *len = 0; - - down_write(&server->alloc_rwsem); - - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, 0, - 0, blocks, 0, 0); - ret = scoutfs_extent_next(sb, server_extent_io, &ext, NULL); - if (ret == -ENOENT) - ret = scoutfs_extent_prev(sb, server_extent_io, &ext, NULL); - if (ret) { - if (ret == -ENOENT) - ret = -ENOSPC; - goto out; - } - - trace_scoutfs_server_alloc_extent_next(sb, &ext); - - ext.type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; - ext.len = min(blocks, ext.len); - - ret = scoutfs_extent_remove(sb, server_extent_io, &ext, NULL); - if (ret) - goto out; - - trace_scoutfs_server_alloc_extent_allocated(sb, &ext); - - *start = ext.start; - *len = ext.len; - ret = 0; - -out: - up_write(&server->alloc_rwsem); - - if (ret) - scoutfs_inc_counter(sb, server_extent_alloc_error); - else - scoutfs_inc_counter(sb, server_extent_alloc); - - return ret; -} - -struct pending_free_extent { - struct list_head head; - u64 start; - u64 len; -}; - -/* - * Now that the transaction's done we can apply all the pending frees. - * The list entries are totally unsorted so this is the first time that - * we can discover corruption from duplicated frees, etc. This can also - * fail on normal transient io or memory errors. - * - * We can't unwind if this fails. The caller can freak out or keep - * trying forever. - */ -static int apply_pending_frees(struct super_block *sb) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct pending_free_extent *pfe; - struct pending_free_extent *tmp; - struct scoutfs_extent ext; - int ret; - - down_write(&server->alloc_rwsem); - - list_for_each_entry_safe(pfe, tmp, &server->pending_frees, head) { - scoutfs_inc_counter(sb, server_free_pending_extent); - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, 0, - pfe->start, pfe->len, 0, 0); - trace_scoutfs_server_free_pending_extent(sb, &ext); - ret = scoutfs_extent_add(sb, server_extent_io, &ext, NULL); - if (ret) { - scoutfs_inc_counter(sb, server_free_pending_error); - break; - } - - list_del_init(&pfe->head); - kfree(pfe); - } - - up_write(&server->alloc_rwsem); - - return 0; -} - -/* - * If there are still pending frees to destroy it means the server didn't - * shut down cleanly and that's not well supported today so we want to - * have it holler if this happens. In the future we'd cleanly support - * forced shutdown that had been told that it's OK to throw away dirty - * state. - */ -static int destroy_pending_frees(struct super_block *sb) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct pending_free_extent *pfe; - struct pending_free_extent *tmp; - - WARN_ON_ONCE(!list_empty(&server->pending_frees)); - - down_write(&server->alloc_rwsem); - - list_for_each_entry_safe(pfe, tmp, &server->pending_frees, head) { - list_del_init(&pfe->head); - kfree(pfe); - } - - up_write(&server->alloc_rwsem); - - return 0; -} - -/* - * We can't satisfy allocations with freed extents until the removed - * references to the freed extents have been committed. We add freed - * extents to a list that is only applied to the persistent indexes as - * the transaction is being committed and the current transaction won't - * try to allocate any more extents. If we didn't do this then we could - * write to referenced data as part of the commit that frees it. If the - * commit was interrupted the stable data could have been overwritten. - */ -static int free_extent(struct super_block *sb, u64 start, u64 len) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct pending_free_extent *pfe; - int ret; - - scoutfs_inc_counter(sb, server_free_extent); - - down_write(&server->alloc_rwsem); - - pfe = kmalloc(sizeof(struct pending_free_extent), GFP_NOFS); - if (!pfe) { - ret = -ENOMEM; - } else { - pfe->start = start; - pfe->len = len; - list_add_tail(&pfe->head, &server->pending_frees); - ret = 0; - } - - up_write(&server->alloc_rwsem); - - return ret; -} - static void stop_server(struct server_info *server) { /* wait_event/wake_up provide barriers */ @@ -444,23 +157,60 @@ static int add_uninit_balloc_items(struct super_block *sb, struct server_info *server, struct scoutfs_super_block *super) { - u64 next = le64_to_cpu(super->next_uninit_free_block); - u64 total = le64_to_cpu(super->total_blocks); + u64 next = le64_to_cpu(super->next_uninit_meta_blkno); + u64 last = le64_to_cpu(super->last_uninit_meta_blkno); u64 nr; int ret; + if (next > last) + return 0; + /* next_uninit should always start a new item */ if (WARN_ON_ONCE(next & SCOUTFS_BALLOC_ITEM_BIT_MASK)) return -EIO; - nr = min_t(u64, total - next, + nr = min_t(u64, last - next + 1, round_up(512 * 1024 * 1024 / SCOUTFS_BLOCK_SIZE, SCOUTFS_BALLOC_ITEM_BITS)); ret = scoutfs_balloc_add_alloc_bulk(sb, &server->alloc, &server->wri, next, nr); if (ret == 0) - le64_add_cpu(&super->next_uninit_free_block, nr); + le64_add_cpu(&super->next_uninit_meta_blkno, nr); + + return ret; +} + +/* + * Add newly initialized free block bitmap items. + */ +static int add_uninit_data_alloc_items(struct super_block *sb, + struct server_info *server, + struct scoutfs_super_block *super) +{ + int nr = 16; + u64 next; + u64 last; + int ret; + + while (nr-- > 0) { + next = le64_to_cpu(super->next_uninit_data_blkno); + last = le64_to_cpu(super->last_uninit_data_blkno); + if (next > last) { + ret = 0; + break; + } + + ret = scoutfs_data_add_free_blocks(sb, &server->alloc, + &server->wri, + &super->core_data_alloc, + next, last - next + 1); + if (ret <= 0) + break; + + le64_add_cpu(&super->next_uninit_data_blkno, ret); + ret = 0; + } return ret; } @@ -499,17 +249,14 @@ static void scoutfs_server_commit_func(struct work_struct *work) down_write(&server->commit_rwsem); - /* try to free first which can dirty the btrees */ - ret = apply_pending_frees(sb); - if (ret) { - scoutfs_err(sb, "server error freeing extents: %d", ret); - goto out; - } - /* XXX not sure what to do about failure here */ ret = add_uninit_balloc_items(sb, server, super); BUG_ON(ret); + /* XXX not sure what to do about failure here */ + ret = add_uninit_data_alloc_items(sb, server, super); + BUG_ON(ret); + ret = scoutfs_block_writer_write(sb, &server->wri); if (ret) { scoutfs_err(sb, "server error writing btree blocks: %d", ret); @@ -579,94 +326,6 @@ out: return scoutfs_net_response(sb, conn, cmd, id, ret, &ial, sizeof(ial)); } -/* - * Give the client an extent allocation of len blocks. We leave the - * details to the extent allocator. - */ -static int server_alloc_extent(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len) -{ - DECLARE_SERVER_INFO(sb, server); - struct commit_waiter cw; - struct scoutfs_net_extent nex = {0,}; - __le64 leblocks; - u64 start; - u64 len; - int ret; - - if (arg_len != sizeof(leblocks)) { - ret = -EINVAL; - goto out; - } - - memcpy(&leblocks, arg, arg_len); - - down_read(&server->commit_rwsem); - ret = alloc_extent(sb, le64_to_cpu(leblocks), &start, &len); - if (ret == 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - if (ret == 0) - ret = wait_for_commit(&cw); - if (ret) - goto out; - - nex.start = cpu_to_le64(start); - nex.len = cpu_to_le64(len); -out: - return scoutfs_net_response(sb, conn, cmd, id, ret, &nex, sizeof(nex)); -} - -static bool invalid_net_extent_list(struct scoutfs_net_extent_list *nexl, - unsigned data_len) -{ - return (data_len < sizeof(struct scoutfs_net_extent_list)) || - (le64_to_cpu(nexl->nr) > SCOUTFS_NET_EXTENT_LIST_MAX_NR) || - (data_len != offsetof(struct scoutfs_net_extent_list, - extents[le64_to_cpu(nexl->nr)])); -} - -static int server_free_extents(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len) -{ - DECLARE_SERVER_INFO(sb, server); - struct scoutfs_net_extent_list *nexl; - struct commit_waiter cw; - int ret = 0; - int err; - u64 i; - - nexl = arg; - if (invalid_net_extent_list(nexl, arg_len)) { - ret = -EINVAL; - goto out; - } - - down_read(&server->commit_rwsem); - - for (i = 0; i < le64_to_cpu(nexl->nr); i++) { - ret = free_extent(sb, le64_to_cpu(nexl->extents[i].start), - le64_to_cpu(nexl->extents[i].len)); - if (ret) - break; - } - - if (i > 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - - if (i > 0) { - err = wait_for_commit(&cw); - if (ret == 0) - ret = err; - } - -out: - return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); -} - /* * Give the client references to stable persistent trees that they'll * use to write their next transaction. @@ -728,9 +387,8 @@ static int server_get_log_trees(struct super_block *sb, memset(<v, 0, sizeof(ltv)); } + /* ensure client has enough free metadata blocks for a transaction */ target = (64*1024*1024) / SCOUTFS_BLOCK_SIZE; - - /* XXX arbitrarily give client enough metadata for a transaction */ while (le64_to_cpu(ltv.alloc_root.total_free) < target) { from = le64_to_cpu(super->core_balloc_cursor); at_least = target - le64_to_cpu(ltv.alloc_root.total_free); @@ -747,9 +405,20 @@ static int server_get_log_trees(struct super_block *sb, goto unlock; super->core_balloc_cursor = cpu_to_le64(next_past); - } + /* fill client's data block allocator */ + target = (2ULL*1024*1024*1024) / SCOUTFS_BLOCK_SIZE; + down_write(&server->alloc_rwsem); + ret = scoutfs_data_move_alloc_bits(sb, &server->alloc, &server->wri, + <v.data_alloc, + &super->core_data_alloc, + &super->core_data_alloc_cursor, + target); + up_write(&server->alloc_rwsem); + if (ret < 0) + goto unlock; + /* update client's log tree's item */ ret = scoutfs_btree_force(sb, &server->alloc, &server->wri, &super->logs_root, <k, sizeof(ltk), @@ -768,6 +437,8 @@ unlock: lt.free_root = ltv.free_root; lt.item_root = ltv.item_root; lt.bloom_ref = ltv.bloom_ref; + lt.data_alloc = ltv.data_alloc; + lt.data_free = ltv.data_free; lt.rid = be64_to_le64(ltk.rid); lt.nr = be64_to_le64(ltk.nr); } @@ -824,10 +495,14 @@ static int server_commit_log_trees(struct super_block *sb, goto unlock; } + /* XXX probably want to merge free blocks */ + ltv.alloc_root = lt->alloc_root; ltv.free_root = lt->free_root; ltv.item_root = lt->item_root; ltv.bloom_ref = lt->bloom_ref; + ltv.data_alloc = lt->data_alloc; + ltv.data_free = lt->data_free; ret = scoutfs_btree_update(sb, &server->alloc, &server->wri, &super->logs_root, <k, sizeof(ltk), @@ -846,6 +521,82 @@ out: return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); } +/* + * A client is being evicted so we want to reclaim resources from their + * log tree items. The item trees and bloom refs stay around to be read + * and eventually merged and we reclaim all the allocator items. + * + * The caller holds the commit rwsem which means we do all this work + * in one server commit. We'll need to keep the total amount of blocks + * in trees in check. + * + * By the time we're evicting a client they've either synced their data + * or have been forcefully removed. The free blocks in the allocator + * roots are stable and can be merged back into allocator items for use + * without risking overwriting stable data. + */ +static int reclaim_log_trees(struct super_block *sb, u64 rid) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + DECLARE_SERVER_INFO(sb, server); + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_log_trees_key ltk; + struct scoutfs_log_trees_val ltv; + __le64 curs; + u64 tot; + int ret; + + memset(<k, 0, sizeof(ltk)); + memset(<v, 0, sizeof(ltv)); + + mutex_lock(&server->logs_mutex); + down_write(&server->alloc_rwsem); + + /* find the client's existing item */ + ltk.rid = cpu_to_be64(rid); + ltk.nr = 0; + ret = scoutfs_btree_next(sb, &super->logs_root, + <k, sizeof(ltk), &iref); + if (ret == 0) { + if (iref.key_len == sizeof(struct scoutfs_log_trees_key) && + iref.val_len == sizeof(struct scoutfs_log_trees_val)) { + memcpy(<k, iref.key, iref.key_len); + memcpy(<v, iref.val, iref.val_len); + if (be64_to_cpu(ltk.rid) != rid) + ret = -ENOENT; + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + goto out; + } + + tot = le64_to_cpu(super->core_data_alloc.total_free) + + le64_to_cpu(ltv.data_alloc.total_free); + curs = 0; + ret = scoutfs_data_move_alloc_bits(sb, &server->alloc, &server->wri, + &super->core_data_alloc, + <v.data_alloc, &curs, tot); + if (ret < 0) + goto out; + + tot = le64_to_cpu(super->core_data_alloc.total_free) + + le64_to_cpu(ltv.data_free.total_free); + curs = 0; + ret = scoutfs_data_move_alloc_bits(sb, &server->alloc, &server->wri, + &super->core_data_alloc, + <v.data_free, &curs, tot); +out: + up_write(&server->alloc_rwsem); + mutex_unlock(&server->logs_mutex); + + return ret; +} + /* * Give the client the next sequence number for their transaction. They * provide their previous transaction sequence number that they've @@ -1455,6 +1206,7 @@ static void farewell_worker(struct work_struct *work) ret = scoutfs_lock_server_farewell(sb, fw->rid) ?: remove_trans_seq(sb, fw->rid) ?: + reclaim_log_trees(sb, fw->rid) ?: delete_mounted_client(sb, fw->rid); if (ret == 0) queue_commit_work(server, &cw); @@ -1564,8 +1316,6 @@ static int server_farewell(struct super_block *sb, static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_GREETING] = server_greeting, [SCOUTFS_NET_CMD_ALLOC_INODES] = server_alloc_inodes, - [SCOUTFS_NET_CMD_ALLOC_EXTENT] = server_alloc_extent, - [SCOUTFS_NET_CMD_FREE_EXTENTS] = server_free_extents, [SCOUTFS_NET_CMD_GET_LOG_TREES] = server_get_log_trees, [SCOUTFS_NET_CMD_COMMIT_LOG_TREES] = server_commit_log_trees, [SCOUTFS_NET_CMD_ADVANCE_SEQ] = server_advance_seq, @@ -1697,7 +1447,6 @@ shutdown: flush_work(&server->commit_work); server->conn = NULL; - destroy_pending_frees(sb); scoutfs_lock_server_destroy(sb); out: @@ -1793,7 +1542,6 @@ int scoutfs_server_setup(struct super_block *sb) INIT_WORK(&server->commit_work, scoutfs_server_commit_func); init_rwsem(&server->seq_rwsem); init_rwsem(&server->alloc_rwsem); - INIT_LIST_HEAD(&server->pending_frees); INIT_LIST_HEAD(&server->clients); mutex_init(&server->farewell_mutex); INIT_LIST_HEAD(&server->farewell_requests); diff --git a/kmod/src/super.c b/kmod/src/super.c index 9387a4b5..69de8ebe 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -441,7 +441,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_client_setup(sb) ?: scoutfs_lock_rid(sb, SCOUTFS_LOCK_WRITE, 0, sbi->rid, &sbi->rid_lock) ?: - scoutfs_forest_get_log_trees(sb); + scoutfs_trans_get_log_trees(sb); if (ret) goto out; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index deb25f20..679ef1b1 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -25,6 +25,8 @@ #include "counters.h" #include "client.h" #include "inode.h" +#include "balloc.h" +#include "block.h" #include "scoutfs_trace.h" /* @@ -60,6 +62,10 @@ struct trans_info { unsigned reserved_vals; unsigned holders; bool writing; + + struct scoutfs_log_trees lt; + struct scoutfs_balloc_allocator alloc; + struct scoutfs_block_writer wri; }; #define DECLARE_TRANS_INFO(sb, name) \ @@ -77,6 +83,48 @@ static bool drained_holders(struct trans_info *tri) return drained; } +static int commit_btrees(struct super_block *sb) +{ + DECLARE_TRANS_INFO(sb, tri); + struct scoutfs_log_trees lt; + + lt = tri->lt; + lt.alloc_root = tri->alloc.alloc_root; + lt.free_root = tri->alloc.free_root; + scoutfs_forest_get_btrees(sb, <); + scoutfs_data_get_btrees(sb, <); + + return scoutfs_client_commit_log_trees(sb, <); +} + +/* + * This gets all the resources from the server that the client will + * need during the transaction. + */ +int scoutfs_trans_get_log_trees(struct super_block *sb) +{ + DECLARE_TRANS_INFO(sb, tri); + struct scoutfs_log_trees lt; + int ret = 0; + + ret = scoutfs_client_get_log_trees(sb, <); + if (ret == 0) { + tri->lt = lt; + scoutfs_balloc_init(&tri->alloc, <.alloc_root, <.free_root); + scoutfs_block_writer_init(sb, &tri->wri); + + scoutfs_forest_init_btrees(sb, &tri->alloc, &tri->wri, <); + scoutfs_data_init_btrees(sb, &tri->alloc, &tri->wri, <); + } + return ret; +} + +bool scoutfs_trans_has_dirty(struct super_block *sb) +{ + DECLARE_TRANS_INFO(sb, tri); + + return scoutfs_block_writer_has_dirty(sb, &tri->wri); +} /* * This work func is responsible for writing out all the dirty blocks * that make up the current dirty transaction. It prevents writers from @@ -113,18 +161,19 @@ void scoutfs_trans_write_func(struct work_struct *work) wait_event(sbi->trans_hold_wq, drained_holders(tri)); - trace_scoutfs_trans_write_func(sb, scoutfs_forest_dirty_bytes(sb)); + trace_scoutfs_trans_write_func(sb, + scoutfs_block_writer_dirty_bytes(sb, &tri->wri)); - if (scoutfs_forest_has_dirty(sb)) { + if (scoutfs_block_writer_has_dirty(sb, &tri->wri)) { if (sbi->trans_deadline_expired) scoutfs_inc_counter(sb, trans_commit_timer); ret = scoutfs_inode_walk_writeback(sb, true) ?: - scoutfs_forest_write(sb) ?: + scoutfs_block_writer_write(sb, &tri->wri) ?: scoutfs_inode_walk_writeback(sb, false) ?: - scoutfs_forest_commit(sb) ?: + commit_btrees(sb) ?: scoutfs_client_advance_seq(sb, &sbi->trans_seq) ?: - scoutfs_forest_get_log_trees(sb); + scoutfs_trans_get_log_trees(sb); if (ret) goto out; @@ -297,7 +346,8 @@ static bool acquired_hold(struct super_block *sb, vals = tri->reserved_vals + cnt->vals; /* XXX arbitrarily limit to 8 meg transactions */ - if (scoutfs_forest_dirty_bytes(sb) >= (8 * 1024 * 1024)) { + if (scoutfs_block_writer_dirty_bytes(sb, &tri->wri) >= + (8 * 1024 * 1024)) { scoutfs_inc_counter(sb, trans_commit_full); queue_trans_work(sbi); goto out; @@ -481,6 +531,7 @@ void scoutfs_shutdown_trans(struct super_block *sb) DECLARE_TRANS_INFO(sb, tri); if (tri) { + scoutfs_block_writer_forget_all(sb, &tri->wri); if (sbi->trans_write_workq) { cancel_delayed_work_sync(&sbi->trans_write_work); destroy_workqueue(sbi->trans_write_workq); diff --git a/kmod/src/trans.h b/kmod/src/trans.h index 04e28f9c..e5f3228e 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -16,6 +16,9 @@ void scoutfs_release_trans(struct super_block *sb); void scoutfs_trans_track_item(struct super_block *sb, signed items, signed vals); +int scoutfs_trans_get_log_trees(struct super_block *sb); +bool scoutfs_trans_has_dirty(struct super_block *sb); + int scoutfs_setup_trans(struct super_block *sb); void scoutfs_shutdown_trans(struct super_block *sb);