diff --git a/kmod/src/Makefile b/kmod/src/Makefile index fa632aa1..7fd73191 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -25,6 +25,7 @@ scoutfs-y += \ forest.o \ inode.o \ ioctl.o \ + iomap.o \ item.o \ kernelcompat.o \ lock.o \ diff --git a/kmod/src/data.c b/kmod/src/data.c index 1d3168f6..0bf0e5c7 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -21,9 +21,11 @@ #include #include #include +#include #include #include #include +#include #include "format.h" #include "super.h" @@ -42,6 +44,7 @@ #include "msg.h" #include "ext.h" #include "util.h" +#include "iomap.h" /* * We want to amortize work done after dirtying the shared transaction @@ -63,12 +66,6 @@ struct data_info { #define DECLARE_DATA_INFO(sb, name) \ struct data_info *name = SCOUTFS_SB(sb)->data_info -struct data_ext_args { - u64 ino; - struct inode *inode; - struct scoutfs_lock *lock; -}; - static void item_from_extent(struct scoutfs_key *key, struct scoutfs_data_extent_val *dv, u64 ino, u64 start, u64 len, u64 map, u8 flags) @@ -181,7 +178,7 @@ static int data_ext_remove(struct super_block *sb, void *arg, u64 start, return ret; } -static struct scoutfs_ext_ops data_ext_ops = { +struct scoutfs_ext_ops data_ext_ops = { .next = data_ext_next, .insert = data_ext_insert, .remove = data_ext_remove, @@ -386,9 +383,9 @@ static inline u64 ext_last(struct scoutfs_extent *ext) * reasonable when a file population is known to be large and dense but * known to be written with non-streaming write patterns. */ -static int alloc_block(struct super_block *sb, struct inode *inode, - struct scoutfs_extent *ext, u64 iblock, - struct scoutfs_lock *lock) +int scoutfs_data_alloc_block(struct super_block *sb, struct inode *inode, + struct scoutfs_extent *ext, u64 iblock, + struct scoutfs_lock *lock) { DECLARE_DATA_INFO(sb, datinf); struct scoutfs_mount_options opts; @@ -611,6 +608,7 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, un.len = 1; un.map = ext.map + (iblock - ext.start); un.flags = ext.flags & ~(SEF_OFFLINE|SEF_UNWRITTEN); + ret = scoutfs_ext_set(sb, &data_ext_ops, &args, un.start, un.len, un.map, un.flags); if (ret == 0) { @@ -622,7 +620,7 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, /* allocate and map blocks containing our logical block */ if (create && !ext.map) { - ret = alloc_block(sb, inode, &ext, iblock, lock); + ret = scoutfs_data_alloc_block(sb, inode, &ext, iblock, lock); if (ret == 0) set_buffer_new(bh); } else { @@ -653,11 +651,17 @@ static int scoutfs_get_block_read(struct inode *inode, sector_t iblock, struct buffer_head *bh, int create) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); + void *extent_sem; /* Just a pointer */ int ret; - down_read(&si->extent_sem); + extent_sem = scoutfs_per_task_get(&si->pt_extent_sem); + if (extent_sem == NULL) + down_read(&si->extent_sem); + ret = scoutfs_get_block(inode, iblock, bh, create); - up_read(&si->extent_sem); + + if (extent_sem == NULL) + up_read(&si->extent_sem); return ret; } @@ -706,44 +710,55 @@ static int scoutfs_readpage(struct file *file, struct page *page) struct scoutfs_lock *inode_lock = NULL; SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); DECLARE_DATA_WAIT(dw); + bool locked; int flags; int ret; - flags = SCOUTFS_LKF_REFRESH_INODE | SCOUTFS_LKF_NONBLOCK; - ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, flags, inode, - &inode_lock); - if (ret < 0) { - unlock_page(page); - if (ret == -EAGAIN) { - flags &= ~SCOUTFS_LKF_NONBLOCK; - ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, flags, - inode, &inode_lock); - if (ret == 0) { - scoutfs_unlock(sb, inode_lock, - SCOUTFS_LOCK_READ); - ret = AOP_TRUNCATED_PAGE; - } - } - return ret; - } - - if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, inode_lock)) { - ret = scoutfs_data_wait_check(inode, page_offset(page), - PAGE_SIZE, SEF_OFFLINE, - SCOUTFS_IOC_DWO_READ, &dw, - inode_lock); - if (ret != 0) { + inode_lock = scoutfs_per_task_get(&si->pt_data_lock); + if (!inode_lock) { + flags = SCOUTFS_LKF_REFRESH_INODE | SCOUTFS_LKF_NONBLOCK; + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, flags, inode, + &inode_lock); + if (ret < 0) { unlock_page(page); - scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); - scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); - } - if (ret > 0) { - ret = scoutfs_data_wait(inode, &dw); - if (ret == 0) - ret = AOP_TRUNCATED_PAGE; - } - if (ret != 0) + if (ret == -EAGAIN) { + flags &= ~SCOUTFS_LKF_NONBLOCK; + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, flags, + inode, &inode_lock); + if (ret == 0) { + scoutfs_unlock(sb, inode_lock, + SCOUTFS_LOCK_READ); + ret = AOP_TRUNCATED_PAGE; + } + } return ret; + } + + locked = true; + + if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, inode_lock)) { + ret = scoutfs_data_wait_check(inode, page_offset(page), + PAGE_SIZE, SEF_OFFLINE, + SCOUTFS_IOC_DWO_READ, &dw, + inode_lock); + if (ret != 0) { + unlock_page(page); + scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + locked = false; + } + if (ret > 0) { + ret = scoutfs_data_wait(inode, &dw); + if (ret == 0) + ret = AOP_TRUNCATED_PAGE; + } + if (ret != 0) + return ret; + } else { + WARN_ON_ONCE(true); + } + } else { + locked = false; } #ifdef KC_MPAGE_READ_FOLIO @@ -752,8 +767,10 @@ static int scoutfs_readpage(struct file *file, struct page *page) ret = mpage_readpage(page, scoutfs_get_block_read); #endif - scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); - scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); + if (locked) { + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); + } return ret; } @@ -761,14 +778,22 @@ static int scoutfs_readpage(struct file *file, struct page *page) static void scoutfs_readahead(struct readahead_control *rac) { struct inode *inode = rac->file->f_inode; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; struct scoutfs_lock *inode_lock = NULL; + bool found_lock; int ret; - ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, - SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); - if (ret) - return; + inode_lock = scoutfs_per_task_get(&si->pt_data_lock); + if (!inode_lock) { + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); + if (ret) + return; + found_lock = false; + } else { + found_lock = true; + } ret = scoutfs_data_wait_check(inode, readahead_pos(rac), readahead_length(rac), SEF_OFFLINE, @@ -777,7 +802,8 @@ static void scoutfs_readahead(struct readahead_control *rac) if (ret == 0) mpage_readahead(rac, scoutfs_get_block_read); - scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + if (!found_lock) + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); } static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) @@ -866,8 +892,8 @@ out: } /* kinda like __filemap_fdatawrite_range! :P */ -static int writepages_sync_none(struct address_space *mapping, loff_t start, - loff_t end) +int scoutfs_writepages_sync_none(struct address_space *mapping, + loff_t start, loff_t end) { struct writeback_control wbc = { .sync_mode = WB_SYNC_NONE, @@ -879,12 +905,26 @@ static int writepages_sync_none(struct address_space *mapping, loff_t start, return mapping->a_ops->writepages(mapping, &wbc); } +void scoutfs_do_write_end(struct inode *inode, struct scoutfs_lock *data_lock, + struct list_head *ind_locks) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + if (!si->staging) { + scoutfs_inode_set_data_seq(inode); + scoutfs_inode_inc_data_version(inode); + } + + inode_inc_iversion(inode); + scoutfs_update_inode_item(inode, data_lock, ind_locks); + scoutfs_inode_queue_writeback(inode); +} + static int scoutfs_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata) { struct inode *inode = mapping->host; - struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; struct write_begin_data *wbd = fsdata; int ret; @@ -893,16 +933,9 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, len, copied); ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata); - if (ret > 0) { - if (!si->staging) { - scoutfs_inode_set_data_seq(inode); - scoutfs_inode_inc_data_version(inode); - } + if (ret > 0) + scoutfs_do_write_end(inode, wbd->lock, &wbd->ind_locks); - inode_inc_iversion(inode); - scoutfs_update_inode_item(inode, wbd->lock, &wbd->ind_locks); - scoutfs_inode_queue_writeback(inode); - } scoutfs_release_trans(sb); scoutfs_inode_index_unlock(sb, &wbd->ind_locks); kfree(wbd); @@ -914,18 +947,16 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, * to very long commit latencies with lots of dirty file data. * * This hack tries to minimize these writeback latencies while - * keeping concurrent large file strreaming writes from + * keeping concurrent large file streaming writes from * suffering too terribly. Every N bytes we kick off background - * writbeack on the previous N bytes. By the time transaction + * writeback on the previous N bytes. By the time transaction * commit comes along it will find that dirty file blocks have * already been written. */ -#define BACKGROUND_WRITEBACK_BYTES (16 * 1024 * 1024) -#define BACKGROUND_WRITEBACK_MASK (BACKGROUND_WRITEBACK_BYTES - 1) if (ret > 0 && ((pos + ret) & BACKGROUND_WRITEBACK_MASK) == 0) - writepages_sync_none(mapping, - pos + ret - BACKGROUND_WRITEBACK_BYTES, - pos + ret - 1); + scoutfs_writepages_sync_none(mapping, + pos + ret - BACKGROUND_WRITEBACK_BYTES, + pos + ret - 1); return ret; } @@ -1142,6 +1173,37 @@ out: return ret; } +static int scoutfs_fadvise(struct file *file, loff_t start, loff_t end, int advice) +{ + struct inode *inode = file_inode(file); + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_extent_ent); + bool locked = false; + int ret; + + /* + * We need to get the extent_sem now, or we'll be out of order with the + * mapping.invalidate_lock. + */ + if (advice == POSIX_FADV_WILLNEED) { + down_read(&si->extent_sem); + locked = true; + + if (!scoutfs_per_task_add_excl(&si->pt_extent_sem, &pt_extent_ent, + &pt_extent_ent)) + WARN_ON_ONCE(true); + } + + ret = generic_fadvise(file, start, end, advice); + + if (locked) { + scoutfs_per_task_del(&si->pt_extent_sem, &pt_extent_ent); + up_read(&si->extent_sem); + } + + return ret; +} + /* * A special case of initializing a single large offline extent. This * chooses not to deal with any existing extents. It can only be used @@ -1439,7 +1501,6 @@ int scoutfs_data_move_blocks(struct inode *from, u64 from_off, from_start += len; } - up_write(&from_si->extent_sem); up_write(&to_si->extent_sem); @@ -1572,163 +1633,47 @@ out: return ret; } -/* - * This copies to userspace :/ - */ -static int fill_extent(struct fiemap_extent_info *fieinfo, - struct scoutfs_extent *ext, u32 fiemap_flags) -{ - u32 flags; - - if (ext->len == 0) - return 0; - - flags = fiemap_flags; - if (ext->flags & SEF_OFFLINE) - flags |= FIEMAP_EXTENT_UNKNOWN; - else if (ext->flags & SEF_UNWRITTEN) - flags |= FIEMAP_EXTENT_UNWRITTEN; - - return fiemap_fill_next_extent(fieinfo, - ext->start << SCOUTFS_BLOCK_SM_SHIFT, - ext->map << SCOUTFS_BLOCK_SM_SHIFT, - ext->len << SCOUTFS_BLOCK_SM_SHIFT, - flags); -} - -/* - * Return all the file's extents whose blocks overlap with the caller's - * byte region. We set _LAST on the last extent and _UNKNOWN on offline - * extents. - */ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; - const u64 ino = scoutfs_ino(inode); struct scoutfs_lock *lock = NULL; - struct scoutfs_extent *info = NULL; - struct page *page = NULL; - struct scoutfs_extent ext; - struct scoutfs_extent cur; - struct data_ext_args args; - u32 last_flags; - u64 iblock; - u64 last; - int entries = 0; + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); int ret; - int complete = 0; if (len == 0) { - ret = 0; + ret = -EINVAL; goto out; } - ret = fiemap_prep(inode, fieinfo, start, &len, FIEMAP_FLAG_SYNC); + /* + * We don't support xattrs stored in extents. The magic incantation + * to communicate this to the caller is to reset fi_flags to + * FIEMAP_FLAG_XATTR and return -EBADR. + */ + if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { + fieinfo->fi_flags = FIEMAP_FLAG_XATTR; + ret = -EBADR; + goto out; + } + + inode_lock(inode); + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &lock); if (ret) goto out; - page = alloc_page(GFP_KERNEL); - if (!page) { - ret = -ENOMEM; - goto out; - } + scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, lock); - /* use a dummy extent to track */ - memset(&cur, 0, sizeof(cur)); - last_flags = 0; + ret = iomap_fiemap(inode, fieinfo, start, len, &scoutfs_iomap_report_ops); - iblock = start >> SCOUTFS_BLOCK_SM_SHIFT; - last = (start + len - 1) >> SCOUTFS_BLOCK_SM_SHIFT; + scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); - args.ino = ino; - args.inode = inode; - - /* outer loop */ - while (iblock <= last) { - /* lock */ - inode_lock(inode); - down_read(&si->extent_sem); - - ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &lock); - if (ret) { - up_read(&si->extent_sem); - inode_unlock(inode); - break; - } - - args.lock = lock; - - /* collect entries */ - info = page_address(page); - memset(info, 0, PAGE_SIZE); - while (entries < (PAGE_SIZE / sizeof(struct fiemap_extent)) - 1) { - ret = scoutfs_ext_next(sb, &data_ext_ops, &args, - iblock, 1, &ext); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - complete = 1; - last_flags = FIEMAP_EXTENT_LAST; - break; - } - - trace_scoutfs_data_fiemap_extent(sb, ino, &ext); - - if (ext.start > last) { - /* not setting _LAST, it's for end of file */ - ret = 0; - complete = 1; - break; - } - - if (scoutfs_ext_can_merge(&cur, &ext)) { - /* merged extents could be greater than input len */ - cur.len += ext.len; - } else { - /* fill it */ - memcpy(info, &cur, sizeof(cur)); - - entries++; - info++; - - cur = ext; - } - - iblock = ext.start + ext.len; - } - - /* unlock */ - scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); - up_read(&si->extent_sem); - inode_unlock(inode); - - if (ret) - break; - - /* emit entries */ - info = page_address(page); - for (; entries > 0; entries--) { - ret = fill_extent(fieinfo, info, 0); - if (ret != 0) - goto out; - info++; - } - - if (complete) - break; - } - - /* still one left, it's in cur */ - if (cur.len) - ret = fill_extent(fieinfo, &cur, last_flags); + inode_unlock(inode); out: - if (ret == 1) - ret = 0; - if (page) - __free_page(page); trace_scoutfs_data_fiemap(sb, start, len, ret); return ret; @@ -1832,6 +1777,7 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, u64 on; u64 off; loff_t tmp; + void *semlock = NULL; int ret = 0; if (len == 0) @@ -1845,7 +1791,7 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, goto out; } - if ((sef & SEF_OFFLINE)) { + if (sef & SEF_OFFLINE) { scoutfs_inode_get_onoff(inode, &on, &off); if (off == 0) { ret = 0; @@ -1853,12 +1799,14 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, } } - down_read(&si->extent_sem); + semlock = scoutfs_per_task_get(&si->pt_extent_sem); + if (semlock == NULL) + down_read(&si->extent_sem); iblock = pos >> SCOUTFS_BLOCK_SM_SHIFT; last_block = (pos + len - 1) >> SCOUTFS_BLOCK_SM_SHIFT; - while(iblock <= last_block) { + while (iblock <= last_block) { ret = scoutfs_ext_next(sb, &data_ext_ops, &args, iblock, 1, &ext); if (ret < 0) { @@ -1891,7 +1839,8 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, iblock = ext.start + ext.len; } - up_read(&si->extent_sem); + if (semlock == NULL) + up_read(&si->extent_sem); out: trace_scoutfs_data_wait_check(sb, ino, pos, len, sef, op, &ext, ret); @@ -2168,40 +2117,70 @@ static vm_fault_t scoutfs_data_filemap_fault(struct vm_fault *vmf) struct super_block *sb = inode->i_sb; struct scoutfs_lock *inode_lock = NULL; SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_sem); DECLARE_DATA_WAIT(dw); loff_t pos; int err; vm_fault_t ret = VM_FAULT_SIGBUS; + void *semlock; + bool found_lock; pos = vmf->pgoff; pos <<= PAGE_SHIFT; retry: - err = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, - SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); - if (err < 0) - return vmf_error(err); + inode_lock = scoutfs_per_task_get(&si->pt_data_lock); + if (!inode_lock) { + found_lock = false; - if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, inode_lock)) { - /* protect checked extents from stage/release */ - atomic_inc(&inode->i_dio_count); + err = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); + if (err < 0) + return vmf_error(err); - err = scoutfs_data_wait_check(inode, pos, PAGE_SIZE, - SEF_OFFLINE, SCOUTFS_IOC_DWO_READ, - &dw, inode_lock); - if (err != 0) { - if (err < 0) - ret = vmf_error(err); - goto out; + if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, inode_lock)) { + /* protect checked extents from stage/release */ + atomic_inc(&inode->i_dio_count); + + err = scoutfs_data_wait_check(inode, pos, PAGE_SIZE, + SEF_OFFLINE, SCOUTFS_IOC_DWO_READ, + &dw, inode_lock); + if (err != 0) { + if (err < 0) + ret = vmf_error(err); + goto out; + } + } else { + WARN_ON_ONCE(true); } + } else { + found_lock = true; + } + + semlock = scoutfs_per_task_get(&si->pt_extent_sem); + if (semlock == NULL) { + down_read(&si->extent_sem); + if (!scoutfs_per_task_add_excl(&si->pt_extent_sem, &pt_sem, &semlock)) + WARN_ON_ONCE(true); } ret = filemap_fault(vmf); + if (semlock == NULL) { + up_read(&si->extent_sem); + scoutfs_per_task_del(&si->pt_extent_sem, &pt_sem); + } + out: - if (scoutfs_per_task_del(&si->pt_data_lock, &pt_ent)) - inode_dio_end(inode); - scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + if (!found_lock) { + if (scoutfs_per_task_del(&si->pt_data_lock, &pt_ent)) + inode_dio_end(inode); + else + WARN_ON_ONCE(true); + + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + } + if (scoutfs_data_wait_found(&dw)) { err = scoutfs_data_wait(inode, &dw); if (err == 0) @@ -2229,6 +2208,7 @@ static int scoutfs_file_mmap(struct file *file, struct vm_area_struct *vma) const struct address_space_operations scoutfs_file_aops = { #ifdef KC_MPAGE_READ_FOLIO + .direct_IO = noop_direct_IO, .dirty_folio = block_dirty_folio, .invalidate_folio = block_invalidate_folio, .read_folio = scoutfs_read_folio, @@ -2252,6 +2232,7 @@ const struct file_operations scoutfs_file_fops = { .fsync = scoutfs_file_fsync, .llseek = scoutfs_file_llseek, .fallocate = scoutfs_fallocate, + .fadvise = scoutfs_fadvise, }; void scoutfs_data_init_btrees(struct super_block *sb, diff --git a/kmod/src/data.h b/kmod/src/data.h index 8b2f5f60..d1e701ec 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -38,10 +38,30 @@ struct scoutfs_data_wait { .err = 0, \ } +struct data_ext_args { + u64 ino; + struct inode *inode; + struct scoutfs_lock *lock; +}; + extern const struct address_space_operations scoutfs_file_aops; extern const struct file_operations scoutfs_file_fops; +extern struct scoutfs_ext_ops data_ext_ops; + struct scoutfs_alloc; struct scoutfs_block_writer; +struct scoutfs_extent; + +#define BACKGROUND_WRITEBACK_BYTES (16 * 1024 * 1024) +#define BACKGROUND_WRITEBACK_MASK (BACKGROUND_WRITEBACK_BYTES - 1) +int scoutfs_writepages_sync_none(struct address_space *mapping, loff_t start, loff_t end); + +void scoutfs_do_write_end(struct inode *inode, struct scoutfs_lock *data_lock, + struct list_head *ind_locks); + +int scoutfs_data_alloc_block(struct super_block *sb, struct inode *inode, + struct scoutfs_extent *ext, u64 iblock, + struct scoutfs_lock *lock); int scoutfs_get_block_write(struct inode *inode, sector_t iblock, struct buffer_head *bh, int create); diff --git a/kmod/src/file.c b/kmod/src/file.c index 0f2e7b83..76825cdf 100644 --- a/kmod/src/file.c +++ b/kmod/src/file.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -29,6 +30,651 @@ #include "per_task.h" #include "omap.h" #include "quota.h" +#include "iomap.h" +#include "trans.h" +#include "msg.h" + +static int lock_for_iomap_read(struct inode *inode, bool nowait, + struct scoutfs_per_task_entry *pt_extent_ent) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + if (nowait) { + if (!down_read_trylock(&si->extent_sem)) + return -EAGAIN; + } else { + down_read(&si->extent_sem); + } + + if (!scoutfs_per_task_add_excl(&si->pt_extent_sem, pt_extent_ent, pt_extent_ent)) + WARN_ON_ONCE(true); + + return 0; +} + +static void unlock_for_iomap_read(struct inode *inode, + struct scoutfs_per_task_entry *pt_extent_ent) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + scoutfs_per_task_del(&si->pt_extent_sem, pt_extent_ent); + up_read(&si->extent_sem); +} + +#ifdef KC_USE_IOMAP_FOR_IO +static bool scoutfs_should_use_dio(struct kiocb *iocb, struct iov_iter *iter) +{ + /* Current offset must be aligned */ + if (iocb->ki_pos & SCOUTFS_BLOCK_SM_MASK) + return false; + + if (iov_iter_alignment(iter) & SCOUTFS_BLOCK_SM_MASK) + return false; + + return true; +} + +/* copied from fs/gfs2/file.c */ +static inline bool should_fault_in_pages(struct iov_iter *i, + struct kiocb *iocb, + size_t *prev_count, + size_t *window_size) +{ + size_t count = iov_iter_count(i); + size_t size, offs; + + if (!count) + return false; + if (!user_backed_iter(i)) + return false; + + size = PAGE_SIZE; + offs = offset_in_page(iocb->ki_pos); + if (*prev_count != count || !*window_size) { + size_t nr_dirtied; + + nr_dirtied = max(current->nr_dirtied_pause - + current->nr_dirtied, 8); + size = min_t(size_t, SZ_1M, nr_dirtied << PAGE_SHIFT); + } + + *prev_count = count; + *window_size = size - offs; + return true; +} + +static ssize_t scoutfs_file_buffered_read(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_extent_ent); + size_t prev_count = 0; + size_t window_size = 0; + size_t read = 0; + bool locked = false; + ssize_t ret; + + pagefault_disable(); + iocb->ki_flags |= IOCB_NOIO; + ret = generic_file_read_iter(iocb, to); + iocb->ki_flags &= ~IOCB_NOIO; + pagefault_enable(); + + if (ret >= 0) { + if (iov_iter_count(to) == 0) + return ret; + read = ret; + } else if (ret != -EFAULT) { + if (ret != -EAGAIN) + return ret; + } + +retry: + ret = lock_for_iomap_read(inode, false, &pt_extent_ent); + if (ret) + return ret; + locked = true; + + pagefault_disable(); + ret = generic_file_read_iter(iocb, to); + pagefault_enable(); + if (ret <= 0 && ret != -EFAULT) + goto out; + if (ret > 0) + read += ret; + + if (should_fault_in_pages(to, iocb, &prev_count, &window_size)) { + unlock_for_iomap_read(inode, &pt_extent_ent); + locked = false; + window_size -= fault_in_iov_iter_writeable(to, window_size); + if (window_size != 0) + goto retry; + } + +out: + if (locked) + unlock_for_iomap_read(inode, &pt_extent_ent); + + return read ? read : ret; +} + +static ssize_t scoutfs_file_direct_read(struct kiocb *iocb, struct iov_iter *to, + bool nowait) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_extent_ent); + size_t prev_count = 0; + size_t window_size = 0; + size_t read = 0; + bool locked = false; + ssize_t ret; + + if (!scoutfs_should_use_dio(iocb, to)) { + iocb->ki_flags &= ~IOCB_DIRECT; + return scoutfs_file_buffered_read(iocb, to); + } + +retry: + ret = lock_for_iomap_read(inode, nowait, &pt_extent_ent); + if (ret) + return ret; + + locked = true; + + pagefault_disable(); + to->nofault = true; + ret = iomap_dio_rw(iocb, to, &scoutfs_iomap_ops, NULL, IOMAP_DIO_PARTIAL, 0); + to->nofault = false; + pagefault_enable(); + + if (ret <= 0 && ret != -EFAULT) + goto out_unlock; + if (ret > 0) + read = ret; + + if (should_fault_in_pages(to, iocb, &prev_count, &window_size)) { + unlock_for_iomap_read(inode, &pt_extent_ent); + locked = false; + window_size -= fault_in_iov_iter_writeable(to, window_size); + if (window_size != 0) + goto retry; + } + +out_unlock: + if (locked) + unlock_for_iomap_read(inode, &pt_extent_ent); + + file_accessed(file); + + if (ret < 0) + return ret; + + return read; +} + +ssize_t scoutfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *scoutfs_inode_lock; + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_data_ent); + DECLARE_DATA_WAIT(dw); + int lock_flags = SCOUTFS_LKF_REFRESH_INODE; + bool is_dio = (iocb->ki_flags & IOCB_DIRECT); + bool nowait = (iocb->ki_flags & IOCB_NOWAIT); + ssize_t ret; + + /* IOCB_NOWAIT is only for direct I/O */ + if (!is_dio && nowait) + return -EOPNOTSUPP; + +retry: + scoutfs_inode_lock = NULL; + + /* protect checked extents from release */ + inode_lock(inode); + atomic_inc(&inode->i_dio_count); + inode_unlock(inode); + + if (is_dio) { + if (nowait) { + if (!inode_trylock_shared(inode)) { + ret = -EAGAIN; + goto out; + } + lock_flags |= SCOUTFS_LKF_NONBLOCK; + } else { + inode_lock_shared(inode); + } + } + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + lock_flags, inode, &scoutfs_inode_lock); + if (ret) + goto out; + + if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_data_ent, + scoutfs_inode_lock)) { + ret = scoutfs_data_wait_check(inode, iocb->ki_pos, iov_iter_count(to), + SEF_OFFLINE, SCOUTFS_IOC_DWO_READ, &dw, + scoutfs_inode_lock); + if (ret != 0) + goto out; + } else { + WARN_ON_ONCE(true); + } + + if (is_dio) + ret = scoutfs_file_direct_read(iocb, to, nowait); + else + ret = scoutfs_file_buffered_read(iocb, to); + +out: + inode_dio_end(inode); + + if (scoutfs_inode_lock) { + scoutfs_per_task_del(&si->pt_data_lock, &pt_data_ent); + scoutfs_unlock(sb, scoutfs_inode_lock, SCOUTFS_LOCK_READ); + } + + if (is_dio) + inode_unlock_shared(inode); + + if (scoutfs_data_wait_found(&dw)) { + if (!nowait) { + ret = scoutfs_data_wait(inode, &dw); + if (ret == 0) + goto retry; + } else { + ret = -EAGAIN; + } + } + + return ret; +} + +/* + * We need to complete all of our locking before we get into the iomap iterator. + * The lock hierarchy isn't well defined, and some paths reverse the ordering of + * the inode index lock and the extent_sem because of how the get_block functions + * are called. So we have to use trylocks and retry as needed, unless nonblocking + * mode was requested. + */ +static int lock_for_iomap_write(struct inode *inode, struct list_head *ind_locks, + struct scoutfs_per_task_entry *pt_inode_ent, + struct scoutfs_per_task_entry *pt_extent_ent, + struct scoutfs_lock *scoutfs_inode_lock, + bool nowait) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + u64 seq; + int ret; + + do { + ret = scoutfs_inode_index_start(sb, &seq) ?: + scoutfs_inode_index_prepare(sb, ind_locks, inode, true) ?: + scoutfs_inode_index_try_lock_hold(sb, ind_locks, seq, true); + if (ret < 0) + return ret; + + /* A return value > 0 means the seq number changed */ + if (ret > 0) { + if (nowait) + return -EAGAIN; + continue; + } + + if (!down_write_trylock(&si->extent_sem)) { + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, ind_locks); + if (nowait) + return -EAGAIN; + continue; + } + + scoutfs_per_task_add_excl(&si->pt_inode_locks, pt_inode_ent, ind_locks); + + /* + * We need to assign a pointer value to the per-task var. Since this + * one is just a marker that we hold the extent_sem, it can be anything + * that's non-NULL. So just use the entry itself. + */ + scoutfs_per_task_add_excl(&si->pt_extent_sem, pt_extent_ent, + pt_extent_ent); + + ret = scoutfs_dirty_inode_item(inode, scoutfs_inode_lock); + + break; + } while (true); + + return ret; +} + +static void unlock_for_iomap_write(struct inode *inode, + struct scoutfs_lock *scoutfs_inode_lock, + struct list_head *ind_locks, + struct scoutfs_per_task_entry *pt_inode_ent, + struct scoutfs_per_task_entry *pt_extent_ent, + size_t written) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + bool t; + + if (written > 0) + scoutfs_do_write_end(inode, scoutfs_inode_lock, ind_locks); + + scoutfs_release_trans(sb); + + t = scoutfs_per_task_del(&si->pt_inode_locks, pt_inode_ent); + BUG_ON(!t); + + scoutfs_inode_index_unlock(sb, ind_locks); + + t = scoutfs_per_task_del(&si->pt_extent_sem, pt_extent_ent); + BUG_ON(!t); + + up_write(&si->extent_sem); +} + +static ssize_t scoutfs_file_direct_write(struct kiocb *iocb, struct iov_iter *from, + struct scoutfs_lock *scoutfs_inode_lock) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + bool nowait = (iocb->ki_flags & IOCB_NOWAIT); + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_inode_ent); + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_extent_ent); + LIST_HEAD(ind_locks); + size_t prev_count = 0; + size_t window_size = 0; + size_t written = 0; + ssize_t ret = 0; + bool locked = false; + + if (!scoutfs_should_use_dio(iocb, from)) { + iocb->ki_flags &= ~IOCB_DIRECT; + goto out; + } + +retry: + ret = lock_for_iomap_write(inode, &ind_locks, &pt_inode_ent, &pt_extent_ent, + scoutfs_inode_lock, nowait); + if (ret < 0) + goto out; + + locked = true; + + /* + * Due to lock ordering issues, we need to disable page faults while we're + * doing the iomap iterations. Pass IOMAP_DIO_PARTIAL so that the iomap + * code knows it's OK to return a partial result. We will try to fault in + * any needed pages later on. + */ + from->nofault = true; + ret = iomap_dio_rw(iocb, from, &scoutfs_iomap_ops, NULL, IOMAP_DIO_PARTIAL, + written); + from->nofault = false; + + if (ret <= 0) { + if (ret == -ENOTBLK) + ret = 0; + if (ret != -EFAULT) + goto out; + } + + /* No increment (+=) because iomap returns a cumulative value. */ + if (ret > 0) + written = ret; + + /* + * We might have skipped some pages that needed to be faulted in. If so, drop + * our locks to avoid deadlock and try to fault them in. Then we can relock + * and try the remaining DIO writes. + */ + if (should_fault_in_pages(from, iocb, &prev_count, &window_size)) { + unlock_for_iomap_write(inode, scoutfs_inode_lock, &ind_locks, + &pt_inode_ent, &pt_extent_ent, written); + locked = false; + window_size -= fault_in_iov_iter_readable(from, window_size); + if (window_size != 0) + goto retry; + } + +out: + if (locked) { + unlock_for_iomap_write(inode, scoutfs_inode_lock, &ind_locks, + &pt_inode_ent, &pt_extent_ent, written); + } + + return ret < 0 ? ret : written; +} + +static ssize_t scoutfs_file_buffered_write(struct kiocb *iocb, struct iov_iter *from, + struct scoutfs_lock *scoutfs_inode_lock) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_inode_ent); + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_extent_ent); + LIST_HEAD(ind_locks); + size_t prev_count = 0; + size_t window_size = 0; + size_t orig_count = iov_iter_count(from); + size_t written = 0; + bool locked; + ssize_t ret; + +retry: + locked = false; + + /* + * Because of lock ordering issues with the page fault code, we need to + * try to manually fault in any pages before acquiring locks. Then we + * disable page faults while doing the iomap iterations. + */ + if (should_fault_in_pages(from, iocb, &prev_count, &window_size)) { + window_size -= fault_in_iov_iter_readable(from, window_size); + if (window_size == 0) { + ret = -EFAULT; + goto out; + } + from->count = min(from->count, window_size); + } + + ret = lock_for_iomap_write(inode, &ind_locks, &pt_inode_ent, &pt_extent_ent, + scoutfs_inode_lock, false); + if (ret < 0) + goto out; + + locked = true; + + pagefault_disable(); + ret = iomap_file_buffered_write(iocb, from, &scoutfs_iomap_ops); + pagefault_enable(); + + /* Accumlate the count of what's been written so far */ + if (ret > 0) + written += ret; + + if (ret <= 0 && ret != -EFAULT) + goto out; + + from->count = orig_count - written; + if (should_fault_in_pages(from, iocb, &prev_count, &window_size)) { + /* + * There are still some pages to be faulted in. Drop our locks and + * try another pass. + */ + unlock_for_iomap_write(inode, scoutfs_inode_lock, &ind_locks, + &pt_inode_ent, &pt_extent_ent, written); + locked = false; + goto retry; + } + +out: + if (locked) { + unlock_for_iomap_write(inode, scoutfs_inode_lock, &ind_locks, + &pt_inode_ent, &pt_extent_ent, written); + } + + from->count = orig_count - written; + + return written ? written : ret; +} + +ssize_t scoutfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *scoutfs_inode_lock = NULL; + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_data_ent); + DECLARE_DATA_WAIT(dw); + int lock_flags = SCOUTFS_LKF_REFRESH_INODE; + bool added_pt_data; + bool is_dio = (iocb->ki_flags & IOCB_DIRECT); + bool nowait = (iocb->ki_flags & IOCB_NOWAIT); + bool is_sync = (iocb->ki_flags & IOCB_DSYNC); + ssize_t written = 0; + ssize_t written_buffered; + loff_t endbyte; + loff_t pos; + loff_t start = iocb->ki_pos; + ssize_t ret; + + /* We don't support O_DSYNC */ + iocb->ki_flags &= ~IOCB_DSYNC; + + /* IOCB_NOWAIT is only for direct I/O */ + if (!is_dio && nowait) + return -EOPNOTSUPP; + + if (nowait) + lock_flags |= SCOUTFS_LKF_NONBLOCK; + +retry: + + added_pt_data = false; + + if (nowait) { + if (!inode_trylock(inode)) { + return -EAGAIN; + } + } else { + inode_lock(inode); + } + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, lock_flags, + inode, &scoutfs_inode_lock); + if (ret == -EAGAIN) + goto out; + if (ret < 0) + goto out; + + ret = generic_write_checks(iocb, from); + if (ret <= 0) + goto out; + + ret = scoutfs_inode_check_retention(inode); + if (ret < 0) + goto out; + + ret = scoutfs_complete_truncate(inode, scoutfs_inode_lock); + if (ret) + goto out; + + ret = scoutfs_quota_check_data(sb, inode); + if (ret) + goto out; + + if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_data_ent, + scoutfs_inode_lock)) { + /* data_version is per inode, whole file must be online */ + /* @@@ need to handle nowait here too? */ + added_pt_data = true; + + ret = scoutfs_data_wait_check(inode, 0, i_size_read(inode), SEF_OFFLINE, + SCOUTFS_IOC_DWO_WRITE, &dw, + scoutfs_inode_lock); + if (ret != 0) + goto out; + } + + /* XXX: remove SUID bit */ + + if (is_dio) { + ret = scoutfs_file_direct_write(iocb, from, scoutfs_inode_lock); + if (ret < 0 || !iov_iter_count(from)) + goto out; + + written = ret; + + pos = iocb->ki_pos; + + written_buffered = scoutfs_file_buffered_write(iocb, from, + scoutfs_inode_lock); + if (written_buffered < 0) { + ret = written_buffered; + goto out; + } + + /* + * Ensure all data is persisted. We want the next direct IO read to be + * able to read what was just written. + */ + endbyte = pos + written_buffered - 1; + ret = filemap_fdatawait_range(file->f_mapping, pos, endbyte); + if (ret) + goto out; + written += written_buffered; + iocb->ki_pos = pos + written_buffered; + invalidate_mapping_pages(file->f_mapping, pos >> PAGE_SHIFT, + endbyte >> PAGE_SHIFT); + } else { + ret = scoutfs_file_buffered_write(iocb, from, scoutfs_inode_lock); + } + +out: + if (added_pt_data) { + scoutfs_per_task_del(&si->pt_data_lock, &pt_data_ent); + added_pt_data = false; + } + + if (written > 0 && ((start + written) & BACKGROUND_WRITEBACK_MASK) == 0) { + scoutfs_writepages_sync_none(inode->i_mapping, + start + written - BACKGROUND_WRITEBACK_BYTES, + start + written - 1); + } + + scoutfs_unlock(sb, scoutfs_inode_lock, SCOUTFS_LOCK_WRITE); + + inode_unlock(inode); + + if (scoutfs_data_wait_found(&dw)) { + if (!nowait) { + ret = scoutfs_data_wait(inode, &dw); + if (ret == 0) + goto retry; + } else { + ret = -EAGAIN; + } + } + + if (ret > 0) + ret = generic_write_sync(iocb, ret); + + if (is_sync) + iocb->ki_flags |= IOCB_DSYNC; + + return written ? written : ret; +} + +#else ssize_t scoutfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) { @@ -38,10 +684,14 @@ ssize_t scoutfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) struct super_block *sb = inode->i_sb; struct scoutfs_lock *scoutfs_inode_lock = NULL; SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_extent_ent); DECLARE_DATA_WAIT(dw); + bool locked; int ret; retry: + locked = false; + /* protect checked extents from release */ inode_lock(inode); atomic_inc(&inode->i_dio_count); @@ -52,6 +702,12 @@ retry: if (ret) goto out; + ret = lock_for_iomap_read(inode, false, &pt_extent_ent); + if (ret) + return ret; + + locked = true; + if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, scoutfs_inode_lock)) { ret = scoutfs_data_wait_check(inode, iocb->ki_pos, iov_iter_count(to), SEF_OFFLINE, SCOUTFS_IOC_DWO_READ, &dw, scoutfs_inode_lock); @@ -64,6 +720,11 @@ retry: ret = generic_file_read_iter(iocb, to); out: + if (locked) { + unlock_for_iomap_read(inode, &pt_extent_ent); + locked = false; + } + inode_dio_end(inode); scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); scoutfs_unlock(sb, scoutfs_inode_lock, SCOUTFS_LOCK_READ); @@ -139,6 +800,71 @@ out: return ret; } +#endif + +loff_t scoutfs_file_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *lock = NULL; + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); + bool ilocked = false; + int ret = 0; + + switch (whence) { + case SEEK_END: + case SEEK_DATA: + case SEEK_HOLE: + /* + * These require a lock and inode refresh as they reference i_size. + */ + inode_lock(inode); + ilocked = true; + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, + &lock); + if (ret == 0) { + if (!scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, lock)) + WARN_ON_ONCE(true); + } + case SEEK_SET: + case SEEK_CUR: + /* No lock required */ + break; + default: + ret = -EINVAL; + break; + } + + if (ret == 0) { + if (whence == SEEK_DATA) { + offset = iomap_seek_data(inode, offset, + &scoutfs_iomap_report_ops); + } else if (whence == SEEK_HOLE) { + offset = iomap_seek_hole(inode, offset, + &scoutfs_iomap_report_ops); + } else { + offset = generic_file_llseek(file, offset, whence); + } + } + + if (ilocked) + inode_unlock(inode); + + if (lock) { + scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); + } + + /* The iomap functions don't update the file pointer */ + if (ret == 0 && offset >= 0) + offset = vfs_setpos(file, offset, sb->s_maxbytes); + + return ret ? ret : offset; +} + int scoutfs_permission(KC_VFS_NS_DEF struct inode *inode, int mask) { @@ -161,41 +887,3 @@ int scoutfs_permission(KC_VFS_NS_DEF return ret; } - -loff_t scoutfs_file_llseek(struct file *file, loff_t offset, int whence) -{ - struct inode *inode = file->f_mapping->host; - struct super_block *sb = inode->i_sb; - struct scoutfs_lock *lock = NULL; - int ret = 0; - - switch (whence) { - case SEEK_END: - case SEEK_DATA: - case SEEK_HOLE: - /* - * These require a lock and inode refresh as they - * reference i_size. - * - * XXX: SEEK_DATA/SEEK_HOLE can search our extent - * items instead of relying on generic_file_llseek() - * trickery. - */ - ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, - SCOUTFS_LKF_REFRESH_INODE, inode, - &lock); - case SEEK_SET: - case SEEK_CUR: - /* No lock required, fall through to the generic helper */ - break; - default: - ret = -EINVAL; - } - - if (ret == 0) - offset = generic_file_llseek(file, offset, whence); - - scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); - - return ret ? ret : offset; -} diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 9f6b55f7..8bc4568d 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -95,6 +95,8 @@ static void scoutfs_inode_ctor(void *obj) seqlock_init(&si->seqlock); si->staging = false; scoutfs_per_task_init(&si->pt_data_lock); + scoutfs_per_task_init(&si->pt_inode_locks); + scoutfs_per_task_init(&si->pt_extent_sem); atomic64_set(&si->data_waitq.changed, 0); init_waitqueue_head(&si->data_waitq.waitq); init_rwsem(&si->xattr_rwsem); @@ -398,7 +400,9 @@ static int set_inode_size(struct inode *inode, struct scoutfs_lock *lock, if (ret) return ret; - scoutfs_per_task_add(&si->pt_data_lock, &pt_ent, lock); + if (!scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, lock)) + WARN_ON_ONCE(true); + ret = block_truncate_page(inode->i_mapping, new_size, scoutfs_get_block_write); scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); if (ret < 0) diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 2c390908..dc9b7129 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -51,6 +51,8 @@ struct scoutfs_inode_info { seqlock_t seqlock; bool staging; /* holder of i_mutex is staging */ struct scoutfs_per_task pt_data_lock; + struct scoutfs_per_task pt_inode_locks; + struct scoutfs_per_task pt_extent_sem; struct scoutfs_data_waitq data_waitq; struct rw_semaphore xattr_rwsem; struct list_head writeback_entry; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 903033be..6da71d95 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -49,6 +49,7 @@ #include "quota.h" #include "scoutfs_trace.h" #include "util.h" +#include "msg.h" /* * We make inode index items coherent by locking fixed size regions of @@ -529,7 +530,8 @@ static long scoutfs_ioc_stage(struct file *file, unsigned long arg) if (ret) goto out; - scoutfs_per_task_add(&si->pt_data_lock, &pt_ent, lock); + if (!scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, lock)) + WARN_ON_ONCE(true); isize = i_size_read(inode); diff --git a/kmod/src/iomap.c b/kmod/src/iomap.c new file mode 100644 index 00000000..1a1e5fe7 --- /dev/null +++ b/kmod/src/iomap.c @@ -0,0 +1,271 @@ +/* + * Copyright (C) 2026 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include "format.h" +#include "super.h" +#include "inode.h" +#include "key.h" +#include "counters.h" +#include "scoutfs_trace.h" +#include "item.h" +#include "btree.h" +#include "lock.h" +#include "ext.h" +#include "iomap.h" +#include "msg.h" +#include "trans.h" + +static void scoutfs_set_iomap(struct inode *inode, struct iomap *iomap, + struct scoutfs_extent *ext, loff_t offset, + u64 iblock, loff_t length) +{ + struct super_block *sb = inode->i_sb; + + iomap->flags |= IOMAP_F_BUFFER_HEAD; + iomap->bdev = sb->s_bdev; + + if (offset + length > i_size_read(inode)) + iomap->flags |= IOMAP_F_DIRTY; + + if (ext->len == 0 || ext->start > iblock) { + iomap->type = IOMAP_HOLE; + iomap->addr = IOMAP_NULL_ADDR; + iomap->offset = offset; + + if (ext->len > 0) { + /* There's a hole at the starting offset */ + iomap->length = min_t(loff_t, + (ext->start - iblock) << SCOUTFS_BLOCK_SM_SHIFT, + length); + } else { + /* There's an implicit hole at EOF */ + iomap->length = length; + } + + goto out; + } + + if (ext->flags & SEF_OFFLINE) { + iomap->type = IOMAP_DELALLOC; + iomap->addr = IOMAP_NULL_ADDR; + } else if (ext->flags & SEF_UNWRITTEN) { + iomap->type = IOMAP_UNWRITTEN; + iomap->addr = (u64) ext->map << SCOUTFS_BLOCK_SM_SHIFT; + iomap->flags |= IOMAP_F_NEW; + } else if (ext->map) { + iomap->type = IOMAP_MAPPED; + iomap->addr = (u64) ext->map << SCOUTFS_BLOCK_SM_SHIFT; + } else { + WARN_ON(true); /* holes should've been handled above */ + } + + iomap->offset = (u64) ext->start << SCOUTFS_BLOCK_SM_SHIFT; + iomap->length = (u64) ext->len << SCOUTFS_BLOCK_SM_SHIFT; + +out: + trace_scoutfs_set_iomap(sb, scoutfs_ino(inode), iblock, length, iomap->type, + iomap->flags, iomap->offset, iomap->length, ext->map); +} + +static int scoutfs_iomap_begin_report(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, + struct iomap *srcmap) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); + struct scoutfs_lock *lock = NULL; + struct scoutfs_extent ext; + struct data_ext_args args; + u64 iblock; + int ret; + + WARN_ON_ONCE(flags & ~(IOMAP_REPORT|IOMAP_DIRECT|IOMAP_WRITE|IOMAP_NOWAIT)); + WARN_ON(!inode_is_locked(inode)); + + iblock = offset >> SCOUTFS_BLOCK_SM_SHIFT; + + /* make sure caller holds a cluster lock */ + lock = scoutfs_per_task_get(&si->pt_data_lock); + BUG_ON(!lock); + + args.ino = ino; + args.inode = inode; + args.lock = lock; + + memset(&ext, 0, sizeof(ext)); + + down_read(&si->extent_sem); + + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, iblock, 1, &ext); + + up_read(&si->extent_sem); + + if (ret == -ENOENT) + ret = 0; + + if (ret == 0) + scoutfs_set_iomap(inode, iomap, &ext, offset, iblock, length); + + return ret; +} + +static int scoutfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, + struct iomap *srcmap) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); + struct scoutfs_lock *data_lock = NULL; + struct scoutfs_extent ext; + struct data_ext_args args; + u64 iblock; + int ret; + + WARN_ON_ONCE(flags & ~(IOMAP_REPORT|IOMAP_DIRECT|IOMAP_WRITE|IOMAP_NOWAIT)); + WARN_ON_ONCE(!rwsem_is_locked(&si->extent_sem)); + + iblock = offset >> SCOUTFS_BLOCK_SM_SHIFT; + + /* make sure caller holds a cluster lock */ + data_lock = scoutfs_per_task_get(&si->pt_data_lock); + BUG_ON(!data_lock); + + args.ino = ino; + args.inode = inode; + args.lock = data_lock; + + memset(&ext, 0, sizeof(ext)); + + if (flags & IOMAP_WRITE) { + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, iblock, 1, &ext); + + if (ret == -ENOENT) + ret = 0; + if (ret < 0) + goto out; + + /* non-staging callers should have waited on offline blocks */ + if (WARN_ON_ONCE(ext.map && (ext.flags & SEF_OFFLINE) && !si->staging)){ + ret = -EIO; + goto out; + } + + if (!si->staging) { + ret = scoutfs_inode_check_retention(inode); + if (ret < 0) + goto out; + } + + if (ext.start > iblock) + memset(&ext, 0, sizeof(ext)); + + /* No need to allocate space */ + if (ext.map) { + trace_scoutfs_data_get_block_found(sb, ino, &ext); + goto out; + } + + ret = scoutfs_data_alloc_block(sb, inode, &ext, iblock, data_lock); + if (ret == 0) + iomap->flags |= IOMAP_F_NEW; + } else { + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, iblock, 1, &ext); + + if (ret == -ENOENT) + ret = 0; + + if (ext.len) + trace_scoutfs_data_get_block_found(sb, ino, &ext); + } + +out: + if (ret == 0) + scoutfs_set_iomap(inode, iomap, &ext, offset, iblock, length); + + return ret; +} + +static int scoutfs_iomap_end(struct inode *inode, loff_t offset, loff_t length, + ssize_t written, unsigned flags, struct iomap *iomap) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *data_lock = NULL; + struct list_head *inode_locks = NULL; + struct scoutfs_extent un; + struct data_ext_args args; + u64 end_blk; + int ret; + + if (!(flags & IOMAP_WRITE)) + return 0; + + /* + * If we hit an error during a direct I/O write, tell the iomap layer to fall + * back to buffered I/O by returning the magic value -ENOTBLK. + */ + if (flags & (IOMAP_WRITE | IOMAP_DIRECT) && written == 0) + return -ENOTBLK; + + data_lock = scoutfs_per_task_get(&si->pt_data_lock); + BUG_ON(!data_lock); + + inode_locks = scoutfs_per_task_get(&si->pt_inode_locks); + BUG_ON(!inode_locks); + + /* convert unwritten to written, could be staging */ + if (iomap->type == IOMAP_UNWRITTEN) { + args.ino = scoutfs_ino(inode); + args.inode = inode; + args.lock = data_lock; + + /* This is the file block where this write began */ + un.start = offset >> SCOUTFS_BLOCK_SM_SHIFT; + + /* This is the file block where this write ended */ + end_blk = (offset + written - 1) >> SCOUTFS_BLOCK_SM_SHIFT; + + /* We wrote this many blocks (rounded up) */ + un.len = (end_blk - un.start) + 1; + + /* Move the phys map start forward to reflect the start of this write */ + un.map = (iomap->addr >> SCOUTFS_BLOCK_SM_SHIFT) + + (un.start - (iomap->offset >> SCOUTFS_BLOCK_SM_SHIFT)); + + un.flags = 0; + + ret = scoutfs_ext_set(sb, &data_ext_ops, &args, un.start, un.len, + un.map, un.flags); + BUG_ON(ret); /* uh-oh, we already wrote those blocks */ + } + + if (offset + written > i_size_read(inode)) + i_size_write(inode, offset + written); + + return ret; +} + +const struct iomap_ops scoutfs_iomap_report_ops = { + .iomap_begin = scoutfs_iomap_begin_report, +}; + +const struct iomap_ops scoutfs_iomap_ops = { + .iomap_begin = scoutfs_iomap_begin, + .iomap_end = scoutfs_iomap_end, +}; diff --git a/kmod/src/iomap.h b/kmod/src/iomap.h new file mode 100644 index 00000000..27c2a971 --- /dev/null +++ b/kmod/src/iomap.h @@ -0,0 +1,7 @@ +#ifndef _SCOUTFS_IOMAP_H +#define _SCOUTFS_IOMAP_H + +extern const struct iomap_ops scoutfs_iomap_report_ops; +extern const struct iomap_ops scoutfs_iomap_ops; + +#endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index c7157352..90fbc0aa 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -3269,6 +3269,40 @@ TRACE_EVENT(scoutfs_trigger_fired, TP_printk(SCSBF" %s", SCSB_TRACE_ARGS, __entry->name) ); +TRACE_EVENT(scoutfs_set_iomap, + TP_PROTO(struct super_block *sb, u64 ino, u64 iblock, loff_t length, + unsigned int type, unsigned int iflags, loff_t ioffset, loff_t ilength, + u64 map), + + TP_ARGS(sb, ino, iblock, length, type, iflags, ioffset, ilength, map), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(u64, ino) + __field(u64, iblock) + __field(loff_t, length) + __field(unsigned int, type) + __field(loff_t, ioffset) + __field(loff_t, ilength) + __field(u64, map) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->iblock = iblock; + __entry->length = length; + __entry->type = type; + __entry->ioffset = ioffset; + __entry->ilength = ilength; + __entry->map = map; + ), + + TP_printk(SCSBF" ino %llu iblock %llu len %llu type %d ioff %llu ilen %llu map %llu", SCSB_TRACE_ARGS, + __entry->ino, __entry->iblock, __entry->length, __entry->type, + __entry->ioffset, __entry->ilength, __entry->map) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 4ac83cad..6e8c890d 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -517,14 +517,22 @@ unlock: static int scoutfs_xattr_get(struct dentry *dentry, const char *name, void *buffer, size_t size) { struct inode *inode = dentry->d_inode; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; struct scoutfs_lock *lock = NULL; - int ret; + bool found_lock = false; + int ret = 0; - ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &lock); + lock = scoutfs_per_task_get(&si->pt_data_lock); + if (lock) { + found_lock = true; + } else { + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &lock); + } if (ret == 0) { ret = scoutfs_xattr_get_locked(inode, name, buffer, size, lock); - scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); + if (!found_lock) + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); } return ret; diff --git a/tests/extra/xfstests/expected-results b/tests/extra/xfstests/expected-results index c4032ca9..83153603 100644 --- a/tests/extra/xfstests/expected-results +++ b/tests/extra/xfstests/expected-results @@ -65,7 +65,6 @@ generic/088 generic/090 generic/091 generic/092 -generic/094 generic/096 generic/097 generic/098 @@ -168,7 +167,6 @@ generic/220 generic/221 generic/222 generic/223 -generic/225 generic/227 generic/228 generic/229 @@ -430,7 +428,6 @@ generic/584 generic/586 generic/587 generic/588 -generic/591 generic/592 generic/593 generic/594 @@ -537,8 +534,6 @@ generic/078 generic/079 generic/081 generic/082 -generic/091 -generic/094 generic/096 generic/110 generic/111 @@ -552,9 +547,7 @@ generic/121 generic/122 generic/123 generic/128 -generic/130 generic/134 -generic/135 generic/136 generic/138 generic/139 @@ -614,7 +607,6 @@ generic/207 generic/210 generic/211 generic/212 -generic/214 generic/216 generic/217 generic/218 @@ -622,7 +614,6 @@ generic/219 generic/220 generic/222 generic/223 -generic/225 generic/227 generic/229 generic/230 @@ -640,7 +631,6 @@ generic/259 generic/260 generic/261 generic/262 -generic/263 generic/264 generic/265 generic/266 @@ -709,7 +699,6 @@ generic/383 generic/384 generic/385 generic/386 -generic/391 generic/392 generic/395 generic/396 @@ -718,17 +707,14 @@ generic/398 generic/400 generic/402 generic/404 -generic/406 generic/407 generic/408 -generic/412 generic/413 generic/414 generic/417 generic/419 generic/420 generic/421 -generic/422 generic/424 generic/425 generic/427 @@ -736,7 +722,6 @@ generic/439 generic/440 generic/446 generic/449 -generic/450 generic/451 generic/453 generic/454 @@ -788,7 +773,6 @@ generic/546 generic/548 generic/549 generic/550 -generic/552 generic/553 generic/555 generic/556 @@ -809,7 +793,6 @@ generic/584 generic/586 generic/587 generic/588 -generic/591 generic/592 generic/593 generic/594 @@ -826,7 +809,6 @@ generic/605 generic/606 generic/607 generic/608 -generic/609 generic/610 generic/612 generic/613 @@ -842,7 +824,6 @@ generic/635 generic/644 generic/645 generic/646 -generic/647 generic/651 generic/652 generic/653 @@ -864,7 +845,6 @@ generic/669 generic/673 generic/674 generic/675 -generic/677 generic/678 generic/679 generic/680 @@ -879,4 +859,4 @@ generic/688 generic/689 shared/002 shared/032 -Passed all 512 tests +Passed all 509 tests diff --git a/tests/extra/xfstests/local.exclude b/tests/extra/xfstests/local.exclude index fa274227..f71dc604 100644 --- a/tests/extra/xfstests/local.exclude +++ b/tests/extra/xfstests/local.exclude @@ -1,9 +1,11 @@ generic/003 # missing atime update in buffered read generic/075 # file content mismatch failures (fds, etc) +generic/094 # preallocation generates an unwritten block where a hole is expected generic/103 # enospc causes trans commit failures generic/108 # mount fails on failing device? generic/112 # file content mismatch failures (fds, etc) generic/213 # enospc causes trans commit failures +generic/225 # preallocation generates an unwritten block where a hole is expected generic/318 # can't support user namespaces until v5.11 generic/321 # requires selinux enabled for '+' in ls? generic/338 # BUG_ON update inode error handling @@ -37,6 +39,7 @@ generic/565 # xfs_io copy_range missing in el7 generic/568 # falloc not resulting in block count increase generic/569 # swap generic/570 # swap +generic/591 # do we care if splice() and O_DIRECT works on pipes? generic/620 # dm-hugedisk generic/633 # id-mapped mounts missing in el7 generic/636 # swap