diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 828cce93..3e7c9b35 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -3,5 +3,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o block.o btree.o buddy.o compact.o counters.o crc.o \ - dir.o filerw.o kvec.o inode.o ioctl.o item.o key.o manifest.o \ + data.o dir.o kvec.o inode.o ioctl.o item.o key.o manifest.o \ msg.o name.o seg.o scoutfs_trace.o super.o trans.o treap.o xattr.o diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 1ccb08d4..137ebbae 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -24,6 +24,12 @@ EXPAND_COUNTER(compact_segment_skipped) \ EXPAND_COUNTER(compact_segment_read) \ EXPAND_COUNTER(compact_segment_written) \ + EXPAND_COUNTER(data_readpage) \ + EXPAND_COUNTER(data_write_begin) \ + EXPAND_COUNTER(data_write_end) \ + EXPAND_COUNTER(data_invalidatepage) \ + EXPAND_COUNTER(data_writepage) \ + EXPAND_COUNTER(data_end_writeback_page) \ EXPAND_COUNTER(item_create) \ EXPAND_COUNTER(item_create_ephemeral) \ EXPAND_COUNTER(item_update_ephemeral) \ diff --git a/kmod/src/data.c b/kmod/src/data.c new file mode 100644 index 00000000..88843d69 --- /dev/null +++ b/kmod/src/data.c @@ -0,0 +1,616 @@ +/* +* Copyright (C) 2017 Versity Software, Inc. All rights reserved. +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public +* License v2 as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +*/ +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "super.h" +#include "inode.h" +#include "key.h" +#include "data.h" +#include "trans.h" +#include "counters.h" +#include "scoutfs_trace.h" +#include "btree.h" +#include "item.h" +#include "ioctl.h" + +/* + * scoutfs stores data in items that can be up to the small 4K block + * size. The page cache address space callbacks work with the item + * cache. Each OS page can be stored in multiple of our smaller fixed + * size items. The code doesn't understand OS pages that are smaller + * than our block size. + * + * readpage does a blocking read of the item and then copies its + * contents into the page. Since the segments are huge we sort of get + * limited read-ahead by reading in segments at a time. + * + * Writing is quite a bit more fiddly. We want to pack small files. + * The item cache and transactions want to accurately track the size of + * dirty items to fill the next segment. And we would like to minimize + * cpu copying as much as we can. + * + * This simplest first pass creates dirty items as pages are dirtied + * whose values reference the page contents. They're freed after + * they're written to the segment so that we don't have to worry about + * items that reference clean pages. Invalidatepage forgets any items + * if a dirty page is truncated away. + * + * Writeback is built around all the dirty items being written by a + * commit. This can happen naturally in the backgroud. Or writepage + * can initiate it to start by kicking the commit thread. In either + * case our dirty pages are "in writeback" by being put on a list that + * is walked by the end of the commit. Because writes and page dirtying + * are serialized with the commit we know that there can be no dirty + * pages after the commit and we can mark writeback complete on all the + * pages that started writeback before the commit finished. motivate + * having items in the item cache while there are dirty pages. + * + * Data is copied from the dirty page contents into the segment pages + * for writing. This lets us easily pack small files without worrying + * about DMA alignment and avoids the stable page problem of the page + * being modified after the cpu calculates the checksum but before the + * DMA reads to the device. + * + * XXX + * - truncate + * - mmap + * - better io error propagation + * - async readpages for more concurrent readahead + * - forced unmount with dirty data + * - direct IO + * - probably stitch page vecs into block struct page fragments for bios + * - maybe cut segment boundaries on aligned data offsets + * - maybe decouple metadata and data segment writes + */ + +struct data_info { + struct llist_head writeback_pages; +}; + +#define DECLARE_DATA_INFO(sb, name) \ + struct data_info *name = SCOUTFS_SB(sb)->data_info + +/* + * trace_printk() doesn't support %c? + * + * 1 - 1ocked + * a - uptodAte + * d - Dirty + * b - writeBack + * e - Error + */ +#define page_hexflag(page, name, val, shift) \ + (Page##name(page) ? (val << (shift * 4)) : 0) + +#define page_hexflags(page) \ + (page_hexflag(page, Locked, 0x1, 4) | \ + page_hexflag(page, Uptodate, 0xa, 3) | \ + page_hexflag(page, Dirty, 0xd, 2) | \ + page_hexflag(page, Writeback, 0xb, 1) | \ + page_hexflag(page, Error, 0xe, 0)) + +#define PGF "page %p [index %lu flags %x]" +#define PGA(page) \ + (page), (page)->index, page_hexflags(page) \ + +#define BHF "bh %p [blocknr %llu size %zu state %lx]" +#define BHA(bh) \ + (bh), (u64)(bh)->b_blocknr, (bh)->b_size, (bh)->b_state \ + +/* + * Free extents whose blocks fall inside the specified blocks. The + * caller holds a transaction. + * + * If 'release' is given then blocks are freed inside i_size but the + * extent items are left behind and their _OFFLINE flag is set. + * + * This is the low level extent item truncate code. Callers manage + * higher order truncation and orphan cleanup. + */ +int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, + u64 len, bool offline) +{ + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + struct scoutfs_extent extent; + struct scoutfs_btree_val val; + struct scoutfs_key key; + struct scoutfs_key first; + u64 seq; + int ret; + + /* XXX not yet updated */ + + scoutfs_set_key(&first, ino, SCOUTFS_EXTENT_KEY, iblock); + scoutfs_set_key(&key, ino, SCOUTFS_EXTENT_KEY, iblock + len - 1); + + trace_printk("iblock %llu\n", iblock); + + scoutfs_btree_init_val(&val, &extent, sizeof(extent)); + val.check_size_eq = 1; + + for (;;) { + ret = scoutfs_btree_prev(sb, meta, &first, &key, &key, &seq, + &val); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + len = le64_to_cpu(extent.len); + if (WARN_ON_ONCE(len != 1)) { + ret = -EIO; + break; + } + + /* XXX corruption: offline and allocation are exclusive */ + if (!!extent.blkno == + !!(extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE)) { + ret = -EIO; + break; + } + + if (offline && (extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE)) + continue; + + /* make sure we can delete the extent after freeing */ + if (extent.blkno) { + ret = scoutfs_btree_dirty(sb, meta, &key); + if (ret) + break; + + ret = scoutfs_buddy_free(sb, cpu_to_le64(seq), + le64_to_cpu(extent.blkno), 0); + if (ret) + break; + } + + if (offline) { + extent.blkno = 0; + extent.flags |= SCOUTFS_EXTENT_FLAG_OFFLINE; + scoutfs_btree_update(sb, meta, &key, &val); + } else { + ret = scoutfs_btree_delete(sb, meta, &key); + if (ret) + break; + } + + /* XXX sync transaction if it's enormous */ + scoutfs_dec_key(&key); + } + + return ret; +} + +static inline struct page *page_from_llist_node(struct llist_node *node) +{ + BUILD_BUG_ON(member_sizeof(struct page, private) != + sizeof(struct llist_node)); + + return container_of((void *)node, struct page, private); +} + +static inline struct llist_node *llist_node_from_page(struct page *page) +{ + return (void *)&page->private; +} + +static inline void page_llist_add(struct page *page, struct llist_head *head) +{ + llist_add(llist_node_from_page(page), head); +} + +/* + * The transaction has committed so there are no more dirty items. End + * writeback on all the dirty pages that started writeback before the + * commit finished. The commit doesn't start until all holders which + * could dirty are released so there couldn't have been new dirty pages + * and writeback entries while the commit was in flight. + */ +void scoutfs_data_end_writeback(struct super_block *sb, int err) +{ + DECLARE_DATA_INFO(sb, datinf); + struct llist_node *node; + struct page *page; + + /* XXX haven't thought about errors here */ + BUG_ON(err); + + node = llist_del_all(&datinf->writeback_pages); + + while (node) { + page = page_from_llist_node(node); + node = llist_next(node); + + trace_printk("ending writeback "PGF"\n", PGA(page)); + scoutfs_inc_counter(sb, data_end_writeback_page); + + + set_page_private(page, 0); + end_page_writeback(page); + page_cache_release(page); + } +} + +static void init_data_key(struct scoutfs_key_buf *key, + struct scoutfs_data_key *dkey, + struct inode *inode, u64 block) +{ + dkey->type = SCOUTFS_DATA_KEY; + dkey->ino = cpu_to_be64(scoutfs_ino(inode)); + dkey->block = cpu_to_be64(block); + + scoutfs_key_init(key, dkey, sizeof(struct scoutfs_data_key)); +} + +/* Iterate over all the data block items that make up the page. */ +#define for_each_page_block(page, start, loff, block, key, dkey, val) \ + for (start = 0; \ + start < PAGE_CACHE_SIZE && \ + (loff = ((loff_t)page->index << PAGE_CACHE_SHIFT) + start, \ + block = loff >> SCOUTFS_BLOCK_SHIFT, \ + init_data_key(&key, &dkey, page->mapping->host, block), \ + scoutfs_kvec_init(val, page_address(page) + start, \ + SCOUTFS_BLOCK_SIZE), \ + 1); \ + start += SCOUTFS_BLOCK_SIZE) + +/* + * Copy the contents of each item that makes up the page into their + * regions of the page, zeroing any page contents not covered by items. + * + * This is the simplest loop that looks up every possible block. We + * could instead have a readpages() that iterates over present items and + * puts them in the pages in the batch. + */ +static int scoutfs_readpage(struct file *file, struct page *page) +{ + struct inode *inode = page->mapping->host; + struct super_block *sb = inode->i_sb; + loff_t size = i_size_read(inode); + struct scoutfs_data_key dkey; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); + unsigned start; + loff_t loff; + u64 block; + int ret = 0; + + + trace_printk(PGF"\n", PGA(page)); + scoutfs_inc_counter(sb, data_readpage); + + for_each_page_block(page, start, loff, block, key, dkey, val) { + /* the rest of the page is zero when block is past i_size */ + if (loff >= size) + break; + + /* copy the block item contents into the page */ + ret = scoutfs_item_lookup(sb, &key, val); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + else + break; + } + + /* + * XXX do we need to clamp the item length by i_size? + * truncate should purge the item cache and create + * truncation range items that'd merge away old data + * items, and invalidatepage should shrink any ephemeral + * vecs. Seems like the item length should be accurate? + */ + + /* zero the tail of the block */ + if (ret < SCOUTFS_BLOCK_SIZE) + zero_user(page, start, SCOUTFS_BLOCK_SIZE - ret); + } + + /* zero any remaining tail blocks */ + if (start < PAGE_CACHE_SIZE) + zero_user(page, start, PAGE_CACHE_SIZE - start); + + if (ret == 0) + SetPageUptodate(page); + else + SetPageError(page); + + trace_printk("ret %d\n", ret); + unlock_page(page); + return ret; +} + +/* + * Start writeback on a dirty page. We always try to kick off a commit. + * Repeated calls harmlessly bounce off the thread work's pending bit. + * (we could probably test that the writeback pgaes list is empty before + * trying to kick off a commit.) + * + * We add ourselves to a list of pages that the commit will end + * writeback on once its done. If there's no dirty data the commit + * thread will end writeback after not doing anything. + */ +static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) +{ + struct inode *inode = page->mapping->host; + struct super_block *sb = inode->i_sb; + DECLARE_DATA_INFO(sb, datinf); + + trace_printk(PGF"\n", PGA(page)); + scoutfs_inc_counter(sb, data_writepage); + + BUG_ON(PageWriteback(page)); + BUG_ON(page->private != 0); + + ClearPagePrivate(page); /* invalidatepage not needed */ + set_page_writeback(page); + page_cache_get(page); + page_llist_add(page, &datinf->writeback_pages); + unlock_page(page); + scoutfs_sync_fs(sb, 0); + + return 0; +} + +/* + * Truncate is invalidating part of the contents of a page. + * + * We can't return errors here so our job is not to create dirty items + * that end up executing the truncate. That's the job of higher level + * callers. Our job is to make sure that we update references to the + * page from existing ephemeral items if they already exist. + */ +static void scoutfs_invalidatepage(struct page *page, unsigned long offset) +{ + struct inode *inode = page->mapping->host; + struct super_block *sb = inode->i_sb; + struct scoutfs_data_key dkey; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); + unsigned start; + loff_t loff; + u64 block; + + trace_printk(PGF"\n", PGA(page)); + scoutfs_inc_counter(sb, data_invalidatepage); + + for_each_page_block(page, start, loff, block, key, dkey, val) { + if (offset) { + /* XXX maybe integrate offset into foreach */ + /* XXX ugh, kvecs are still clumsy :) */ + if (start + SCOUTFS_BLOCK_SIZE > offset) + val[0].iov_len = offset - start; + scoutfs_item_update_ephemeral(sb, &key, val); + } else { + scoutfs_item_forget(sb, &key); + } + } +} + +/* + * Start modifying a page cache page. + * + * We hold the transaction for write_end's inode updates before + * acquiring the page lock. + * + * We give the writer the current page contents in the relatively rare + * case of writing a partial page inside i_size. write_end will zero + * any region around the write if the page isn't uptodate. + */ +static int scoutfs_write_begin(struct file *file, + struct address_space *mapping, loff_t pos, + unsigned len, unsigned flags, + struct page **pagep, void **fsdata) +{ + struct inode *inode = mapping->host; + struct super_block *sb = inode->i_sb; + pgoff_t index = pos >> PAGE_SHIFT; + loff_t size = i_size_read(inode); + struct page *page; + int ret; + + trace_printk("ino %llu pos %llu len %u flags %x\n", + scoutfs_ino(inode), (u64)pos, len, flags); + scoutfs_inc_counter(sb, data_write_begin); + + ret = scoutfs_hold_trans(sb); + if (ret) + return ret; + + /* can't re-enter fs, have trans */ + flags |= AOP_FLAG_NOFS; + + ret = scoutfs_dirty_inode_item(inode); + if (ret) + goto out; + +retry: + page = grab_cache_page_write_begin(mapping, index, flags); + if (!page) { + ret = -ENOMEM; + goto out; + } + + trace_printk(PGF"\n", PGA(page)); + + if (!PageUptodate(page) && (pos < size && len < PAGE_CACHE_SIZE)) { + ClearPageError(page); + ret = scoutfs_readpage(file, page); + if (!ret) { + wait_on_page_locked(page); + if (!PageUptodate(page)) + ret = -EIO; + } + page_cache_release(page); + if (ret) + goto out; + + /* let grab_ lock and check for truncated pages */ + goto retry; + } + + *pagep = page; + ret = 0; +out: + if (ret) + scoutfs_release_trans(sb); + + trace_printk("ret %d\n", ret); + return ret; +} + +/* + * Finish modification of a page cache page. + * + * write_begin has held the transaction and dirtied the inode. We + * create items for each dirty block whose value references the page + * contents that will be written. + * + * We Modify the dirty item and its dependent metadata items while + * holding the transaction so that we never get missing data. + * + * XXX + * - detect no change with copied == 0? + * - only iterate over written blocks, not the whole page? + * - make sure page granular locking and concurrent extending writes works + * - error handling needs work, truncate partial writes on failure? + */ +static int scoutfs_write_end(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct page *page, void *fsdata) +{ + struct inode *inode = page->mapping->host; + struct super_block *sb = inode->i_sb; + struct scoutfs_data_key dkey; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); + loff_t old_size = i_size_read(inode); + bool update_inode = false; + loff_t new_size; + unsigned start; + loff_t loff; + u64 block; + int ret; + + trace_printk("ino %llu "PGF" pos %llu len %u copied %d\n", + scoutfs_ino(inode), PGA(page), (u64)pos, len, copied); + scoutfs_inc_counter(sb, data_write_end); + + /* zero any unwritten portions of a new page around the write */ + if (!PageUptodate(page)) { + if (copied != PAGE_CACHE_SIZE) { + start = pos & ~PAGE_CACHE_MASK; + zero_user_segments(page, 0, start, + start + copied, PAGE_CACHE_SIZE); + } + SetPageUptodate(page); + } + + new_size = pos + copied; + + for_each_page_block(page, start, loff, block, key, dkey, val) { + + /* only put data inside i_size in items */ + /* XXX ugh, kvecs are still clumsy :) */ + if (loff + SCOUTFS_BLOCK_SIZE > new_size) + val[0].iov_len = new_size - loff; + + ret = scoutfs_item_create_ephemeral(sb, &key, val); + if (ret) + goto out; + } + + /* update i_size if we extended */ + if (new_size > inode->i_size) { + i_size_write(inode, new_size); + update_inode = true; + } + + if (old_size < pos) + pagecache_isize_extended(inode, old_size, pos); + + if (copied) { + scoutfs_inode_inc_data_version(inode); + update_inode = true; + } + + if (update_inode) + scoutfs_update_inode_item(inode); + + flush_dcache_page(page); + set_page_dirty(page); + SetPagePrivate(page); /* call invalidatepage */ + + ret = copied; +out: + unlock_page(page); + scoutfs_release_trans(sb); + + /* XXX error handling needs work */ + WARN_ON_ONCE(ret < 0); + return ret; +} + +const struct address_space_operations scoutfs_file_aops = { + .readpage = scoutfs_readpage, + .writepage = scoutfs_writepage, + .set_page_dirty = __set_page_dirty_nobuffers, + .invalidatepage = scoutfs_invalidatepage, + .write_begin = scoutfs_write_begin, + .write_end = scoutfs_write_end, +}; + +const struct file_operations scoutfs_file_fops = { + .read = do_sync_read, + .write = do_sync_write, + .aio_read = generic_file_aio_read, + .aio_write = generic_file_aio_write, + .unlocked_ioctl = scoutfs_ioctl, + .fsync = scoutfs_file_fsync, +}; + +int scoutfs_data_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct data_info *datinf; + + /* page block iteration doesn't understand multiple pages per block */ + BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_BLOCK_SIZE); + + datinf = kzalloc(sizeof(struct data_info), GFP_KERNEL); + if (!datinf) + return -ENOMEM; + sbi->data_info = datinf; + + init_llist_head(&datinf->writeback_pages); + + return 0; +} + +void scoutfs_data_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct data_info *datinf = sbi->data_info; + + if (datinf) { + WARN_ON_ONCE(!llist_empty(&datinf->writeback_pages)); + kfree(datinf); + } +} diff --git a/kmod/src/data.h b/kmod/src/data.h new file mode 100644 index 00000000..189b2cba --- /dev/null +++ b/kmod/src/data.h @@ -0,0 +1,14 @@ +#ifndef _SCOUTFS_FILERW_H_ +#define _SCOUTFS_FILERW_H_ + +extern const struct address_space_operations scoutfs_file_aops; +extern const struct file_operations scoutfs_file_fops; + +int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, + u64 len, bool offline); +void scoutfs_data_end_writeback(struct super_block *sb, int err); + +int scoutfs_data_setup(struct super_block *sb); +void scoutfs_data_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c deleted file mode 100644 index 5cdce097..00000000 --- a/kmod/src/filerw.c +++ /dev/null @@ -1,667 +0,0 @@ -/* -* Copyright (C) 2016 Versity Software, Inc. All rights reserved. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public -* License v2 as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -* General Public License for more details. -*/ -#include -#include -#include -#include -#include - -#include "format.h" -#include "super.h" -#include "inode.h" -#include "key.h" -#include "filerw.h" -#include "trans.h" -#include "scoutfs_trace.h" -#include "btree.h" -#include "ioctl.h" - -/* - * scoutfs uses an extent item to map logical file data blocks to - * physical block locations. - * - * The small block size is set to the smallest supported page size. - * This means that our file IO code never has to worry about the - * situation where a page write is smaller than the block size. We - * never have to perform RMW of blocks larger than pages, nor do we have - * to punch a whole and worry about block tracking items that could be - * sharing references to a block on either side of a smaller dirty page. - * We can simply use the kernel's buffer head code, loathed though it - * is, and have a 1:1 relationship between block writes and block - * mapping item entries. - * - * Dirty extents are only written to free space. The first time a block - * hits write_page in a transaction it gets a newly allocated block. We - * get decent contiguous allocations by having per-task preallocation - * streams. These are trimmed back as the transaction is committed. We - * don't bother worrying about small transactions. - * - * Because we only write to allocated space we can't naively use the - * buffer head get_blocks support functions. They assume that they can - * write dirty buffers to existing clean mappings which is absolutely - * not true for us. We clear mappings for clean pages before we call - * block_write_begin() so that it won't write to blocks that were caned - * from previous reads. We make sure that the page is uptodate ourself - * so that it won't use readpage to read the existing block and then - * turn around and write to it. - * - * Data blocks aren't pinned for the duration of the transaction. They - * can be written out and read back in and redirtied during the lifetime - * of a transaction. As we map dirty pages we see if its current allocation - * is newly allocated in the transaction and can reuse it. - * - * XXX - * - need to wire up dirty inode? - * - enforce writing to free blknos - * - per-task allocation regions - * - tear down dirty extents left by write errors on unmount - * - should invalidate dirty blocks if freed - * - data block checksumming (stable pages) - * - mmap creating dirty unmapped pages at writepage - * - pack small tails into inline items - * - direct IO - */ - - -/* - * trace_printk() doesn't support %c? - * - * 1 - 1ocked - * a - uptodAte - * d - Dirty - * b - writeBack - * e - Error - */ -#define page_hexflag(page, name, val, shift) \ - (Page##name(page) ? (val << (shift * 4)) : 0) - -#define page_hexflags(page) \ - (page_hexflag(page, Locked, 0x1, 4) | \ - page_hexflag(page, Uptodate, 0xa, 3) | \ - page_hexflag(page, Dirty, 0xd, 2) | \ - page_hexflag(page, Writeback, 0xb, 1) | \ - page_hexflag(page, Error, 0xe, 0)) - -#define PGF "page %p [index %lu flags %x]" -#define PGA(page) \ - (page), (page)->index, page_hexflags(page) \ - -#define BHF "bh %p [blocknr %llu size %zu state %lx]" -#define BHA(bh) \ - (bh), (u64)(bh)->b_blocknr, (bh)->b_size, (bh)->b_state \ - -/* - * For now this is super cheesy. We just have one allocation on the - * super that is consumed as buffered writes make their way through unmapped - * buffer heads and alloc in get_block. - */ -static int alloc_file_block(struct super_block *sb, u64 *blkno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - u64 alloc_blkno; - int order = 0; - int ret; - - *blkno = 0; - - spin_lock(&sbi->file_alloc_lock); - - if (sbi->file_alloc_count == 0) { - spin_unlock(&sbi->file_alloc_lock); - - order = scoutfs_buddy_alloc(sb, &alloc_blkno, - SCOUTFS_BUDDY_ORDERS - 1); - if (order < 0) { - ret = order; - goto out; - } - - spin_lock(&sbi->file_alloc_lock); - - if (sbi->file_alloc_count == 0) { - sbi->file_alloc_blkno = alloc_blkno; - sbi->file_alloc_count = 1 << order; - order = -1; - } - } - - if (sbi->file_alloc_count) { - *blkno = sbi->file_alloc_blkno; - sbi->file_alloc_blkno++; - sbi->file_alloc_count--; - ret = 0; - } else { - ret = -ENOSPC; - } - - spin_unlock(&sbi->file_alloc_lock); - - if (order > 0) - scoutfs_buddy_free(sb, sbi->super.hdr.seq, alloc_blkno, order); - -out: - trace_printk("allocated blkno %llu ret %d\n", *blkno, ret); - return ret; -} - -/* - * The caller didn't need an allocated file block after all. We return - * it to the pool. This has to succeed because it's called after we've - * done things that would be annoying to revert. - */ -static void return_file_block(struct super_block *sb, u64 blkno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - spin_lock(&sbi->file_alloc_lock); - - BUG_ON(sbi->file_alloc_count && - sbi->file_alloc_blkno != (blkno + 1)); - - if (sbi->file_alloc_count == 0) - sbi->file_alloc_blkno = blkno + 1; - - sbi->file_alloc_blkno--; - sbi->file_alloc_count++; - - spin_unlock(&sbi->file_alloc_lock); -} - -/* - * Free extents whose blocks fall inside the specified blocks. The - * caller holds a transaction. - * - * If 'release' is given then blocks are freed inside i_size but the - * extent items are left behind and their _OFFLINE flag is set. - * - * This is the low level extent item truncate code. Callers manage - * higher order truncation and orphan cleanup. - */ -int scoutfs_truncate_extent_items(struct super_block *sb, u64 ino, u64 iblock, - u64 len, bool offline) -{ - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_extent extent; - struct scoutfs_btree_val val; - struct scoutfs_key key; - struct scoutfs_key first; - u64 seq; - int ret; - - scoutfs_set_key(&first, ino, SCOUTFS_EXTENT_KEY, iblock); - scoutfs_set_key(&key, ino, SCOUTFS_EXTENT_KEY, iblock + len - 1); - - trace_printk("iblock %llu\n", iblock); - - scoutfs_btree_init_val(&val, &extent, sizeof(extent)); - val.check_size_eq = 1; - - for (;;) { - ret = scoutfs_btree_prev(sb, meta, &first, &key, &key, &seq, - &val); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - break; - } - - len = le64_to_cpu(extent.len); - if (WARN_ON_ONCE(len != 1)) { - ret = -EIO; - break; - } - - /* XXX corruption: offline and allocation are exclusive */ - if (!!extent.blkno == - !!(extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE)) { - ret = -EIO; - break; - } - - if (offline && (extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE)) - continue; - - /* make sure we can delete the extent after freeing */ - if (extent.blkno) { - ret = scoutfs_btree_dirty(sb, meta, &key); - if (ret) - break; - - ret = scoutfs_buddy_free(sb, cpu_to_le64(seq), - le64_to_cpu(extent.blkno), 0); - if (ret) - break; - } - - if (offline) { - extent.blkno = 0; - extent.flags |= SCOUTFS_EXTENT_FLAG_OFFLINE; - scoutfs_btree_update(sb, meta, &key, &val); - } else { - ret = scoutfs_btree_delete(sb, meta, &key); - if (ret) - break; - } - - /* XXX sync transaction if it's enormous */ - scoutfs_dec_key(&key); - } - - return ret; -} - -/* - * The caller ensures that this is serialized against all other callers - * and writers. - */ -void scoutfs_filerw_free_alloc(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - trace_printk("blkno %llu count %llu\n", sbi->file_alloc_blkno, - sbi->file_alloc_count); - - if (sbi->file_alloc_count) - scoutfs_buddy_free_extent(sb, sbi->file_alloc_blkno, - sbi->file_alloc_count); - - sbi->file_alloc_blkno = 0; - sbi->file_alloc_count = 0; -} - -/* - * Return the number of contiguously mapped blocks starting from the - * given logical block in the inode. - */ -static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno) -{ - struct super_block *sb = inode->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_btree_val val; - struct scoutfs_extent extent; - struct scoutfs_key key; - int ret; - - *blkno = 0; - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_EXTENT_KEY, iblock); - scoutfs_btree_init_val(&val, &extent, sizeof(extent)); - - ret = scoutfs_btree_lookup(sb, meta, &key, &val); - if (ret == sizeof(extent)) { - if (extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE) { - ret = 0; - } else { - *blkno = le64_to_cpu(extent.blkno); - ret = min_t(u64, le64_to_cpu(extent.len), INT_MAX); - } - } else if (ret >= 0) { - /* XXX corruption */ - ret = -EIO; - } else if (ret == -ENOENT) { - ret = 0; - } - - trace_printk("ino %llu iblock %llu blkno %llu ret %d\n", - scoutfs_ino(inode), iblock, *blkno, ret); - - return ret; -} - -/* - * Make sure that the mapped block at the given logical block number is - * writable in this transaction. If it's not we allocate and reference - * a new block. If there was a previous stable block we free it. We - * give the caller the writable block number. - * - * Writeback is allowed during a transaction so we can get here with - * buffer heads that are newly allocated and being written to but for - * blocks that were allocated in the current transacation. In that - * case we re-use the existing mapping. None of it will be stable until - * there's a sync that writes all the referencing metadata. - */ -static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) -{ - struct scoutfs_inode_info *si = SCOUTFS_I(inode); - struct super_block *sb = inode->i_sb; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->stable_super; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_extent extent; - struct scoutfs_btree_val val; - struct scoutfs_key first; - struct scoutfs_key key; - bool inserted = false; - u64 old_blkno = 0; - u64 new_blkno = 0; - u64 seq; - int ret; - int err; - - scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_EXTENT_KEY, 0); - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_EXTENT_KEY, iblock); - scoutfs_btree_init_val(&val, &extent, sizeof(extent)); - val.check_size_eq = 1; - - /* see if there's an existing mapping */ - ret = scoutfs_btree_prev(sb, meta, &first, &key, &key, &seq, &val); - if (ret == 0 && ((le64_to_cpu(key.offset) + - le64_to_cpu(extent.len)) <= iblock)) - ret = -ENOENT; - if (ret < 0 && ret != -ENOENT) - goto out; - - /* make sure that updating the extent item won't fail */ - if (ret == -ENOENT) { - memset(&extent, 0, sizeof(extent)); - ret = scoutfs_btree_insert(sb, meta, &key, &val); - if (ret) - goto out; - inserted = true; - } else { - if ((extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE) && - !si->staging) { - ret = -EINVAL; - goto out; - } - - ret = scoutfs_btree_dirty(sb, meta, &key); - if (ret) - goto out; - } - - old_blkno = le64_to_cpu(extent.blkno); - - /* If the existing block is dirty then we can use it */ - if (old_blkno && cpu_to_le64(seq) == super->hdr.seq) { - *blkno_ret = old_blkno; - ret = 0; - goto out; - } - - ret = alloc_file_block(sb, &new_blkno); - if (ret < 0) - goto out; - - if (old_blkno) { - ret = scoutfs_buddy_free(sb, cpu_to_le64(seq), old_blkno, 0); - if (ret) - goto out; - } - - extent.blkno = cpu_to_le64(new_blkno); - extent.len = cpu_to_le64(1); - extent.flags &= ~SCOUTFS_EXTENT_FLAG_OFFLINE; - - /* dirtying guarantees success */ - err = scoutfs_btree_update(sb, meta, &key, &val); - BUG_ON(err); - - *blkno_ret = new_blkno; - new_blkno = 0; - ret = 0; -out: - if (ret) { - if (new_blkno) - return_file_block(sb, new_blkno); - if (inserted) { - err = scoutfs_btree_delete(sb, meta, &key); - BUG_ON(err); /* always succeeds */ - } - } - - return ret; -} - -static int scoutfs_readpage_get_block(struct inode *inode, sector_t iblock, - struct buffer_head *bh, int create) -{ - u64 blkno; - int ret; - - if (WARN_ON_ONCE(create)) - return -EINVAL; - - ret = contig_mapped_blocks(inode, iblock, &blkno); - if (ret > 0) { - map_bh(bh, inode->i_sb, blkno); - bh->b_size = min_t(u64, bh->b_size, - (u64)ret << inode->i_blkbits); - ret = 0; - } - - trace_printk("ino %llu iblock %llu create %d "BHF"\n", - scoutfs_ino(inode), (u64)iblock, create, BHA(bh)); - - return ret; -} - -static int scoutfs_readpage(struct file *file, struct page *page) -{ - trace_printk(PGF"\n", PGA(page)); - - return mpage_readpage(page, scoutfs_readpage_get_block); -} - -static int scoutfs_readpages(struct file *file, struct address_space *mapping, - struct list_head *pages, unsigned nr_pages) -{ - return mpage_readpages(mapping, pages, nr_pages, - scoutfs_readpage_get_block); -} - -/* - * For now we don't know what to do if unmapped blocks make it to - * writepage (mmap?). - */ -static int scoutfs_writepage_get_block(struct inode *inode, sector_t iblock, - struct buffer_head *bh, int create) -{ - trace_printk("ino %llu iblock %llu create %d "BHF"\n", - scoutfs_ino(inode), (u64)iblock, create, BHA(bh)); - - return WARN_ON_ONCE(-EINVAL); -} - -/* - * Dirty file pages can be written to their newly allocated free extents - * at any time. They won't be referenced by metadata until the current - * transaction is committed. They can be re-read and re-dirtied at - * their free block number in this transaction. - */ -static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) -{ - trace_printk(PGF"\n", PGA(page)); - - return block_write_full_page(page, scoutfs_writepage_get_block, wbc); -} - -static int scoutfs_writepages(struct address_space *mapping, - struct writeback_control *wbc) -{ - trace_printk("mapping %p\n", mapping); - - return mpage_writepages(mapping, wbc, scoutfs_writepage_get_block); -} - -/* - * Extent allocation during buffered writes needs to make sure that the - * dirty blocks will be written to free space. - */ -static int scoutfs_write_begin_get_block(struct inode *inode, sector_t iblock, - struct buffer_head *bh, int create) -{ - u64 blkno = 0; - int ret; - - if (WARN_ON_ONCE(!create)) - return -EINVAL; - - ret = map_writable_block(inode, iblock, &blkno); - if (ret == 0) { - map_bh(bh, inode->i_sb, blkno); - bh->b_size = SCOUTFS_BLOCK_SIZE; - ret = 0; - } - - trace_printk("ino %llu iblock %llu create %d ret %d "BHF"\n", - scoutfs_ino(inode), (u64)iblock, create, ret, BHA(bh)); - return ret; -} - -/* XXX could make a for_each wrapper if we get a few of these */ -static inline void clear_mapped_page_buffers(struct page *page) -{ - struct buffer_head *head; - struct buffer_head *bh; - - if (!page_has_buffers(page)) - return; - - head = page_buffers(page); - bh = head; - do { - if (buffer_mapped(bh)) { - trace_printk(BHF"\n", BHA(bh)); - clear_buffer_mapped(bh); - } - - bh = bh->b_this_page; - } while (bh != head); -} - -/* - * Dirty blocks have to be mapped to be written out to free space so - * that we don't overwrite live data. We're relying on - * block_write_begin() to call get_block(). There are two problems with - * this. - * - * First, if it's going to be trying to read a partial block before writing - * then we can't give it the location to read. It'll just mark the - * block dirty and write to that same location. We use readpage to make - * the page uptodate if it's going to be satisfying a partial overwrite. - * - * Second, we can't let it use mappings that were used by readpage to - * read the current stable data. We need to have get_block be called - * for existing clean uptodate pages so that we can reallocate them to - * free space. We do this by clearing the buffer mappings for every buffer - * on the page for every call. This is probably unnecessarily expensive - * because we don't need to do it for clean buffers. That optimization - * would need to be done very carefully. - */ -static int scoutfs_write_begin(struct file *file, - struct address_space *mapping, loff_t pos, - unsigned len, unsigned flags, - struct page **pagep, void **fsdata) -{ - struct inode *inode = mapping->host; - struct super_block *sb = inode->i_sb; - pgoff_t index = pos >> PAGE_SHIFT; - struct page *page; - int ret; - - ret = scoutfs_hold_trans(sb); - if (ret) - return ret; - - /* can't re-enter fs, have trans */ - flags |= AOP_FLAG_NOFS; - - /* generic write_end updates i_size and calls dirty_inode */ - ret = scoutfs_dirty_inode_item(inode); - if (ret) - goto out; - -retry: - page = grab_cache_page_write_begin(mapping, index, flags); - if (!page) { - ret = -ENOMEM; - goto out; - } - - /* - * read in the page if we're going to be dirtying part of the - * page. readpage catches when this is a read past i_size or - * from a hole and zeros the buffer. We try to grab the page - * again to let it deal with locking and races. - */ - if (!PageUptodate(page) && !IS_ALIGNED(pos | len, SCOUTFS_BLOCK_SIZE)) { - ClearPageError(page); - ret = scoutfs_readpage(file, page); - if (!ret) { - wait_on_page_locked(page); - if (!PageUptodate(page)) - ret = -EIO; - } - page_cache_release(page); - if (ret) - goto out; - goto retry; - } - - /* make sure our get_block gets a chance to alloc */ - clear_mapped_page_buffers(page); - - ret = __block_write_begin(page, pos, len, - scoutfs_write_begin_get_block); - if (ret < 0) { - /* XXX handle truncating? */ - unlock_page(page); - put_page(page); - page = NULL; - } - - *pagep = page; -out: - if (ret) - scoutfs_release_trans(sb); - return ret; -} - -static int scoutfs_write_end(struct file *file, struct address_space *mapping, - loff_t pos, unsigned len, unsigned copied, - struct page *page, void *fsdata) -{ - struct inode *inode = mapping->host; - struct super_block *sb = inode->i_sb; - int ret; - - trace_printk("ino %llu "PGF" pos %llu len %u copied %d\n", - scoutfs_ino(inode), PGA(page), (u64)pos, len, copied); - - ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata); - if (ret > 0) { - scoutfs_inode_inc_data_version(inode); - /* XXX kind of a big hammer, inode life cycle needs work */ - scoutfs_update_inode_item(inode); - } - scoutfs_release_trans(sb); - return ret; -} - -const struct address_space_operations scoutfs_file_aops = { - .readpage = scoutfs_readpage, - .readpages = scoutfs_readpages, - .writepage = scoutfs_writepage, - .writepages = scoutfs_writepages, - .write_begin = scoutfs_write_begin, - .write_end = scoutfs_write_end, -}; - -const struct file_operations scoutfs_file_fops = { - .read = do_sync_read, - .write = do_sync_write, - .aio_read = generic_file_aio_read, - .aio_write = generic_file_aio_write, - .unlocked_ioctl = scoutfs_ioctl, - .fsync = scoutfs_file_fsync, -}; diff --git a/kmod/src/filerw.h b/kmod/src/filerw.h deleted file mode 100644 index f5924d71..00000000 --- a/kmod/src/filerw.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef _SCOUTFS_FILERW_H_ -#define _SCOUTFS_FILERW_H_ - -extern const struct address_space_operations scoutfs_file_aops; -extern const struct file_operations scoutfs_file_fops; - -void scoutfs_filerw_free_alloc(struct super_block *sb); -int scoutfs_truncate_extent_items(struct super_block *sb, u64 ino, u64 iblock, - u64 len, bool offline); - -#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index f2480fc3..2ba9cbd8 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -238,6 +238,7 @@ struct scoutfs_key { #define SCOUTFS_SYMLINK_KEY 8 #define SCOUTFS_EXTENT_KEY 9 #define SCOUTFS_ORPHAN_KEY 10 +#define SCOUTFS_DATA_KEY 11 #define SCOUTFS_MAX_UNUSED_KEY 255 #define SCOUTFS_MAX_ITEM_LEN 512 @@ -268,6 +269,13 @@ struct scoutfs_orphan_key { __be64 ino; } __packed; +/* value is data payload bytes */ +struct scoutfs_data_key { + __u8 type; + __be64 ino; + __be64 block; +} __packed; + struct scoutfs_btree_root { u8 height; struct scoutfs_block_ref ref; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 87e6e063..b773f0ee 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -23,7 +24,7 @@ #include "inode.h" #include "btree.h" #include "dir.h" -#include "filerw.h" +#include "data.h" #include "scoutfs_trace.h" #include "xattr.h" #include "trans.h" @@ -103,6 +104,9 @@ static void set_inode_ops(struct inode *inode) init_special_inode(inode, inode->i_mode, inode->i_rdev); break; } + + /* ephemeral data items avoid kmap for pointers to page contents */ + mapping_set_gfp_mask(inode->i_mapping, GFP_USER); } static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index d40cfae1..4c1b039a 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -30,7 +30,7 @@ #include "super.h" #include "inode.h" #include "trans.h" -#include "filerw.h" +#include "data.h" /* * Find all the inodes that have had keys of a given type modified since @@ -365,8 +365,8 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) if (ret) goto out; - ret = scoutfs_truncate_extent_items(sb, scoutfs_ino(inode), - iblock, len, true); + ret = scoutfs_data_truncate_items(sb, scoutfs_ino(inode), iblock, len, + true); scoutfs_release_trans(sb); out: mutex_unlock(&inode->i_mutex); diff --git a/kmod/src/super.c b/kmod/src/super.c index 2fd01d8c..c96bbbb4 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -35,6 +35,7 @@ #include "alloc.h" #include "treap.h" #include "compact.h" +#include "data.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -212,7 +213,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) spin_lock_init(&sbi->trans_write_lock); INIT_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); init_waitqueue_head(&sbi->trans_write_wq); - spin_lock_init(&sbi->file_alloc_lock); sbi->block_shrinker.shrink = scoutfs_block_shrink; sbi->block_shrinker.seeks = DEFAULT_SEEKS; @@ -228,6 +228,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_seg_setup(sb) ?: scoutfs_manifest_setup(sb) ?: scoutfs_item_setup(sb) ?: + scoutfs_data_setup(sb) ?: scoutfs_alloc_setup(sb) ?: scoutfs_treap_setup(sb) ?: // scoutfs_buddy_setup(sb) ?: @@ -268,6 +269,7 @@ static void scoutfs_kill_sb(struct super_block *sb) scoutfs_buddy_destroy(sb); if (sbi->block_shrinker.shrink == scoutfs_block_shrink) unregister_shrinker(&sbi->block_shrinker); + scoutfs_data_destroy(sb); scoutfs_item_destroy(sb); scoutfs_alloc_destroy(sb); scoutfs_manifest_destroy(sb); diff --git a/kmod/src/super.h b/kmod/src/super.h index d93a296b..82eb6bba 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -14,6 +14,7 @@ struct manifest; struct segment_cache; struct treap_info; struct compact_info; +struct data_info; struct scoutfs_sb_info { struct super_block *sb; @@ -39,6 +40,7 @@ struct scoutfs_sb_info { struct seg_alloc *seg_alloc; struct treap_info *treap_info; struct compact_info *compact_info; + struct data_info *data_info; struct buddy_info *buddy_info; @@ -59,11 +61,6 @@ struct scoutfs_sb_info { struct kset *kset; struct scoutfs_counters *counters; - - /* XXX we'd like this to be per task, not per super */ - spinlock_t file_alloc_lock; - u64 file_alloc_blkno; - u64 file_alloc_count; }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 65db65ec..26e30367 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -21,7 +21,7 @@ #include "block.h" #include "trans.h" #include "buddy.h" -#include "filerw.h" +#include "data.h" #include "bio.h" #include "item.h" #include "manifest.h" @@ -93,11 +93,6 @@ void scoutfs_trans_write_func(struct work_struct *work) wait_event(sbi->trans_hold_wq, atomic_cmpxchg(&sbi->trans_holds, 0, -1) == 0); - /* XXX file data needs to be updated to the new item api */ -#if 0 - scoutfs_filerw_free_alloc(sb); -#endif - trace_printk("items dirty %d manifest dirty %d alloc dirty %d\n", scoutfs_item_has_dirty(sb), scoutfs_manifest_has_dirty(sb), @@ -137,6 +132,9 @@ out: /* XXX this all needs serious work for dealing with errors */ WARN_ON_ONCE(ret); + /* must be done before waking waiting trans holders who might dirty */ + scoutfs_data_end_writeback(sb, ret); + spin_lock(&sbi->trans_write_lock); if (advance) scoutfs_advance_dirty_super(sb);